diff --git a/pkgs/development/compilers/llvm/3.9/llvm.nix b/pkgs/development/compilers/llvm/3.9/llvm.nix
index 0408b9413c8..6c86ac0c294 100644
--- a/pkgs/development/compilers/llvm/3.9/llvm.nix
+++ b/pkgs/development/compilers/llvm/3.9/llvm.nix
@@ -81,6 +81,12 @@ in stdenv.mkDerivation rec {
   preBuild = ''
     mkdir -p $out/
     ln -sv $PWD/lib $out
+  ''
+  + # This is a good candidate for using the `placeholder` primitive when it's released
+    # This should hopefully be unnecessary once
+    # https://github.com/NixOS/nixpkgs/pull/25047 is merged
+    stdenv.lib.optionalString (buildPlatform != hostPlatform && enableSharedLibraries) ''
+    export NIX_CROSS_LDFLAGS="-rpath $lib/lib -rpath $lib/lib64 $NIX_CROSS_LDFLAGS"
   '';
 
   cmakeFlags = with stdenv; [