VM builds: Use 9p/virtfs instead of CIFS

9p (with caching enabled) is much faster than CIFS and doesn't require
Samba or virtual networking.  For instance, building GNU Hello with
CIFS takes ~323s on my laptop, but with 9p it takes 54s.

More measurements will be needed to see if "cache=fscache" is really
faster than "cache=loose" (the former seems to be a little bit
faster).
This commit is contained in:
Eelco Dolstra 2013-07-05 00:06:08 +02:00
parent 9efe759dd8
commit 0e386d0c13
3 changed files with 21 additions and 64 deletions

View File

@ -1,21 +1,14 @@
{ pkgs { pkgs
, linuxKernel ? pkgs.linux_3_9 , kernel ? pkgs.linux_3_9
, img ? "bzImage" , img ? "bzImage"
, rootModules ? , rootModules ?
[ "cifs" "virtio_net" "virtio_pci" "virtio_blk" "virtio_balloon" "nls_utf8" [ "virtio_pci" "virtio_blk" "virtio_balloon" "ext4" "unix" "9p" "9pnet_virtio" ]
"ext4" "unix" "hmac" "md4" "ecb" "des_generic" "sha256"
]
}: }:
with pkgs; with pkgs;
rec { rec {
# The 15 second CIFS timeout is too short if the host if heavily
# loaded (e.g., in the Hydra build farm when it's running many jobs
# in parallel). So apply a patch to increase the timeout to 120s.
kernel = assert pkgs.linux.features.cifsTimeout; linuxKernel;
kvm = pkgs.qemu; kvm = pkgs.qemu;
qemuProg = "${kvm}/bin/qemu-system-" + (if stdenv.system == "x86_64-linux" then "x86_64" else "i386"); qemuProg = "${kvm}/bin/qemu-system-" + (if stdenv.system == "x86_64-linux" then "x86_64" else "i386");
@ -99,21 +92,13 @@ rec {
done done
for i in $(cat ${modulesClosure}/insmod-list); do for i in $(cat ${modulesClosure}/insmod-list); do
args=
case $i in
*/cifs.ko)
args="CIFSMaxBufSize=4194304"
;;
esac
echo "loading module $(basename $i .ko)" echo "loading module $(basename $i .ko)"
insmod $i $args insmod $i
done done
mount -t tmpfs none /dev mount -t tmpfs none /dev
${createDeviceNodes "/dev"} ${createDeviceNodes "/dev"}
ifconfig eth0 up 10.0.2.15
mkdir /fs mkdir /fs
if test -z "$mountDisk"; then if test -z "$mountDisk"; then
@ -127,14 +112,14 @@ rec {
echo "mounting Nix store..." echo "mounting Nix store..."
mkdir -p /fs/nix/store mkdir -p /fs/nix/store
mount -t cifs //10.0.2.4/store /fs/nix/store -o guest,sec=none,sec=ntlm mount -t 9p store /fs/nix/store -o trans=virtio,version=9p2000.L,msize=262144,cache=fscache
mkdir -p /fs/tmp mkdir -p /fs/tmp
mount -t tmpfs -o "mode=755" none /fs/tmp mount -t tmpfs -o "mode=755" none /fs/tmp
echo "mounting host's temporary directory..." echo "mounting host's temporary directory..."
mkdir -p /fs/tmp/xchg mkdir -p /fs/tmp/xchg
mount -t cifs //10.0.2.4/xchg /fs/tmp/xchg -o guest,sec=none,sec=ntlm mount -t 9p xchg /fs/tmp/xchg -o trans=virtio,version=9p2000.L,msize=262144,cache=fscache
mkdir -p /fs/proc mkdir -p /fs/proc
mount -t proc none /fs/proc mount -t proc none /fs/proc
@ -203,9 +188,8 @@ rec {
-enable-kvm \ -enable-kvm \
${lib.optionalString (pkgs.stdenv.system == "x86_64-linux") "-cpu kvm64"} \ ${lib.optionalString (pkgs.stdenv.system == "x86_64-linux") "-cpu kvm64"} \
-nographic -no-reboot \ -nographic -no-reboot \
-net nic,model=virtio \ -virtfs local,path=/nix/store,security_model=none,mount_tag=store \
-chardev socket,id=samba,path=./samba \ -virtfs local,path=$TMPDIR/xchg,security_model=none,mount_tag=xchg \
-net user,guestfwd=tcp:10.0.2.4:445-chardev:samba \
-drive file=$diskImage,if=virtio,cache=writeback,werror=report \ -drive file=$diskImage,if=virtio,cache=writeback,werror=report \
-kernel ${kernel}/${img} \ -kernel ${kernel}/${img} \
-initrd ${initrd}/initrd \ -initrd ${initrd}/initrd \
@ -214,40 +198,6 @@ rec {
''; '';
startSamba =
''
export WHO=`whoami`
mkdir -p $TMPDIR/xchg
cat > $TMPDIR/smb.conf <<SMB
[global]
private dir = $TMPDIR
smb ports = 0
socket address = 127.0.0.1
pid directory = $TMPDIR
lock directory = $TMPDIR
log file = $TMPDIR/log.smbd
smb passwd file = $TMPDIR/smbpasswd
security = share
[store]
force user = $WHO
path = /nix/store
read only = no
guest ok = yes
[xchg]
force user = $WHO
path = $TMPDIR/xchg
read only = no
guest ok = yes
$EXTRA_SAMBA_CONF
SMB
rm -f ./samba
${socat}/bin/socat unix-listen:./samba exec:"${utillinux}/bin/setsid ${samba}/sbin/smbd -s $TMPDIR/smb.conf",nofork > /dev/null 2>&1 &
while [ ! -e ./samba ]; do sleep 0.1; done # ugly
'';
vmRunCommand = qemuCommand: writeText "vm-run" '' vmRunCommand = qemuCommand: writeText "vm-run" ''
export > saved-env export > saved-env
@ -267,7 +217,6 @@ rec {
diskImage=$diskImage diskImage=$diskImage
TMPDIR=$TMPDIR TMPDIR=$TMPDIR
cd $TMPDIR cd $TMPDIR
${startSamba}
${qemuCommand} ${qemuCommand}
EOF EOF
@ -314,9 +263,9 @@ rec {
/* Run a derivation in a Linux virtual machine (using Qemu/KVM). By /* Run a derivation in a Linux virtual machine (using Qemu/KVM). By
default, there is no disk image; the root filesystem is a tmpfs, default, there is no disk image; the root filesystem is a tmpfs,
and /nix/store is shared with the host (via the CIFS protocol to and /nix/store is shared with the host (via the 9P protocol).
a Samba instance automatically started by Qemu). Thus, any pure Thus, any pure Nix derivation should run unmodified, e.g. the
Nix derivation should run unmodified, e.g. the call call
runInLinuxVM patchelf runInLinuxVM patchelf

View File

@ -7,6 +7,10 @@ rec {
# Run the PatchELF derivation in a VM. # Run the PatchELF derivation in a VM.
buildPatchelfInVM = runInLinuxVM patchelf; buildPatchelfInVM = runInLinuxVM patchelf;
buildHelloInVM = runInLinuxVM hello;
buildPanInVM = runInLinuxVM pan;
testRPMImage = makeImageTestScript diskImages.fedora16x86_64; testRPMImage = makeImageTestScript diskImages.fedora16x86_64;
@ -17,10 +21,10 @@ rec {
diskImage = diskImages.fedora16x86_64; diskImage = diskImages.fedora16x86_64;
}; };
testUbuntuImage = makeImageTestScript diskImages.ubuntu810i386; testUbuntuImage = makeImageTestScript diskImages.ubuntu810i386;
buildInDebian = runInLinuxImage (stdenv.mkDerivation { buildInDebian = runInLinuxImage (stdenv.mkDerivation {
name = "deb-compile"; name = "deb-compile";
src = patchelf.src; src = patchelf.src;
@ -65,6 +69,6 @@ rec {
make install make install
''; '';
}; };
*/ */
} }

View File

@ -246,6 +246,10 @@ let
# Easier debug of NFS issues # Easier debug of NFS issues
SUNRPC_DEBUG y SUNRPC_DEBUG y
# Enable the 9P cache to speed up NixOS VM tests.
9P_FSCACHE y
9P_FS_POSIX_ACL y
${if kernelPlatform ? kernelExtraConfig then kernelPlatform.kernelExtraConfig else ""} ${if kernelPlatform ? kernelExtraConfig then kernelPlatform.kernelExtraConfig else ""}
${extraConfig} ${extraConfig}
''; '';