R: improve support for CRAN and Bioconductor package sets

- Update the instructions for re-generating each of the package set files.
 - Provide test-evaluation.nix expression to verify that the package sets evaluates.
 - Update list of known broken packages.

pkgs/build-support/fetchurl/mirrors.nix

@@ -293,15 +293,15 @@ rec {
   # The commented-out ones don't seem to allow direct package downloads;
   # they serve error messages that result in hash mismatches instead.
   bioc = [
-    # http://bioc.ism.ac.jp/3.0/bioc/
+    # http://bioc.ism.ac.jp/3.2/bioc/
-    # http://bioc.openanalytics.eu/3.0/bioc/
+    # http://bioc.openanalytics.eu/3.2/bioc/
-    # http://bioconductor.fmrp.usp.br/3.0/bioc/
+    # http://bioconductor.fmrp.usp.br/3.2/bioc/
-    # http://mirror.aarnet.edu.au/pub/bioconductor/3.0/bioc/
+    # http://mirror.aarnet.edu.au/pub/bioconductor/3.2/bioc/
-    # http://watson.nci.nih.gov/bioc_mirror/3.0/bioc/
+    # http://watson.nci.nih.gov/bioc_mirror/3.2/bioc/
-    http://bioconductor.jp/packages/3.0/bioc/
+    http://bioconductor.jp/packages/3.2/bioc/
-    http://bioconductor.statistik.tu-dortmund.de/packages/3.0/bioc/
+    http://bioconductor.statistik.tu-dortmund.de/packages/3.2/bioc/
-    http://mirrors.ebi.ac.uk/bioconductor/packages/3.0/bioc/
+    http://mirrors.ebi.ac.uk/bioconductor/packages/3.2/bioc/
-    http://mirrors.ustc.edu.cn/bioc/3.0/bioc/
+    http://mirrors.ustc.edu.cn/bioc/3.2/bioc/
   ];
 
   # CRAN mirrors (from http://cran.r-project.org/mirrors.html)

pkgs/development/r-modules/generate-r-packages.R

@@ -5,47 +5,54 @@ library(parallel)
 cl <- makeCluster(10)
 
 mirrorType <- commandArgs(trailingOnly=TRUE)[1]
-packagesFile <- paste(mirrorType, 'packages.nix', sep='-')
+stopifnot(mirrorType %in% c("bioc","cran"))
-mirrorUrls <- list(
-  bioc="http://bioconductor.statistik.tu-dortmund.de/packages/3.0/bioc",
-  cran="http://cran.r-project.org"
-)
-mirrorUrl <- paste(mirrorUrls[mirrorType], "/src/contrib/", sep="")
 
+packagesFile <- paste(mirrorType, 'packages.nix', sep='-')
 readFormatted <- as.data.table(read.table(skip=6, sep='"', text=head(readLines(packagesFile), -1)))
 
+mirrorUrls <- list( bioc="http://bioconductor.statistik.tu-dortmund.de/packages/3.2/bioc/src/contrib/"
+                  , cran="http://cran.r-project.org/src/contrib/"
+                  )
+mirrorUrl <- mirrorUrls[mirrorType][[1]]
+knownPackages <- lapply(mirrorUrls, function(url) as.data.table(available.packages(url, filters=c("R_version", "OS_type", "duplicates"))))
+pkgs <- knownPackages[mirrorType][[1]]
+setkey(pkgs, Package)
+knownPackages <- c(unique(do.call("rbind", knownPackages)$Package))
+knownPackages <- sapply(knownPackages, gsub, pattern=".", replacement="_", fixed=TRUE)
+
 nixPrefetch <- function(name, version) {
   prevV <- readFormatted$V2 == name & readFormatted$V4 == version
   if (sum(prevV) == 1) as.character(readFormatted$V6[ prevV ]) else
     system(paste0("nix-prefetch-url --type sha256 ", mirrorUrl, name, "_", version, ".tar.gz"), intern=TRUE)
 }
 
-formatPackage <- function(name, version, sha256, depends, imports, linkingTo, knownPackages) {
+formatPackage <- function(name, version, sha256, depends, imports, linkingTo) {
     attr <- gsub(".", "_", name, fixed=TRUE)
-    if (is.na(depends)) depends <- "";
+    depends <- paste( if (is.na(depends)) "" else gsub("[ \t\n]+", "", depends)
-    depends <- unlist(strsplit(depends, split="[ \t\n]*,[ \t\n]*", fixed=FALSE))
+                    , if (is.na(imports)) "" else gsub("[ \t\n]+", "", imports)
-    depends <- c(depends, unlist(strsplit(imports, split="[ \t\n]*,[ \t\n]*", fixed=FALSE)))
+                    , if (is.na(linkingTo)) "" else gsub("[ \t\n]+", "", linkingTo)
-    depends <- c(depends, unlist(strsplit(linkingTo, split="[ \t\n]*,[ \t\n]*", fixed=FALSE)))
+                    , sep=","
+                    )
+    depends <- unlist(strsplit(depends, split=",", fixed=TRUE))
     depends <- sapply(depends, gsub, pattern="([^ \t\n(]+).*", replacement="\\1")
-    depends <- depends[depends %in% knownPackages]
     depends <- sapply(depends, gsub, pattern=".", replacement="_", fixed=TRUE)
-    depends <- paste(depends, collapse=" ")
+    depends <- depends[depends %in% knownPackages]
+    depends <- paste(sort(unique(depends)), collapse=" ")
     paste0(attr, " = derive { name=\"", name, "\"; version=\"", version, "\"; sha256=\"", sha256, "\"; depends=[", depends, "]; };")
 }
 
-clusterExport(cl, c("nixPrefetch","readFormatted", "mirrorUrl"))
+clusterExport(cl, c("nixPrefetch","readFormatted", "mirrorUrl", "knownPackages"))
 
 pkgs <- as.data.table(available.packages(mirrorUrl, filters=c("R_version", "OS_type", "duplicates")))
 pkgs <- pkgs[order(Package)]
 pkgs$sha256 <- parApply(cl, pkgs, 1, function(p) nixPrefetch(p[1], p[2]))
-knownPackages <- unique(pkgs$Package)
 
-nix <- apply(pkgs, 1, function(p) formatPackage(p[1], p[2], p[18], p[4], p[5], p[6], knownPackages))
+nix <- apply(pkgs, 1, function(p) formatPackage(p[1], p[2], p[18], p[4], p[5], p[6]))
 
 cat("# This file is generated from generate-r-packages.R. DO NOT EDIT.\n")
 cat("# Execute the following command to update the file.\n")
 cat("#\n")
-cat(paste("# Rscript generate-r-packages.R", mirrorType, ">", packagesFile))
+cat(paste("# Rscript generate-r-packages.R", mirrorType, ">new && mv new", packagesFile))
 cat("\n\n")
 cat("{ self, derive }: with self; {\n")
 cat(paste(nix, collapse="\n"), "\n")

pkgs/development/r-modules/test-evaluation.nix

@@ -0,0 +1,21 @@
+# Run
+#
+#   nix-build test-evaluation.nix --dry-run
+#
+# to test whether the R package set evaluates properly.
+
+let
+
+  config = {
+    allowBroken = true;
+    allowUnfree = true;
+  };
+
+  inherit (import ../../.. { inherit config; }) pkgs;
+
+  rWrapper = pkgs.rWrapper.override {
+    packages = pkgs.lib.filter pkgs.lib.isDerivation (pkgs.lib.attrValues pkgs.rPackages);
+  };
+
+in
+  rWrapper