R: improve support for CRAN and Bioconductor package sets

- Update the instructions for re-generating each of the package set files.
 - Provide test-evaluation.nix expression to verify that the package sets evaluates.
 - Update list of known broken packages.
This commit is contained in:
Peter Simons 2015-06-16 12:58:58 +02:00
parent 93a742fd28
commit bb42c215e2
6 changed files with 4661 additions and 4235 deletions

View file

@ -293,15 +293,15 @@ rec {
# The commented-out ones don't seem to allow direct package downloads;
# they serve error messages that result in hash mismatches instead.
bioc = [
# http://bioc.ism.ac.jp/3.0/bioc/
# http://bioc.openanalytics.eu/3.0/bioc/
# http://bioconductor.fmrp.usp.br/3.0/bioc/
# http://mirror.aarnet.edu.au/pub/bioconductor/3.0/bioc/
# http://watson.nci.nih.gov/bioc_mirror/3.0/bioc/
http://bioconductor.jp/packages/3.0/bioc/
http://bioconductor.statistik.tu-dortmund.de/packages/3.0/bioc/
http://mirrors.ebi.ac.uk/bioconductor/packages/3.0/bioc/
http://mirrors.ustc.edu.cn/bioc/3.0/bioc/
# http://bioc.ism.ac.jp/3.2/bioc/
# http://bioc.openanalytics.eu/3.2/bioc/
# http://bioconductor.fmrp.usp.br/3.2/bioc/
# http://mirror.aarnet.edu.au/pub/bioconductor/3.2/bioc/
# http://watson.nci.nih.gov/bioc_mirror/3.2/bioc/
http://bioconductor.jp/packages/3.2/bioc/
http://bioconductor.statistik.tu-dortmund.de/packages/3.2/bioc/
http://mirrors.ebi.ac.uk/bioconductor/packages/3.2/bioc/
http://mirrors.ustc.edu.cn/bioc/3.2/bioc/
];
# CRAN mirrors (from http://cran.r-project.org/mirrors.html)

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -5,47 +5,54 @@ library(parallel)
cl <- makeCluster(10)
mirrorType <- commandArgs(trailingOnly=TRUE)[1]
packagesFile <- paste(mirrorType, 'packages.nix', sep='-')
mirrorUrls <- list(
bioc="http://bioconductor.statistik.tu-dortmund.de/packages/3.0/bioc",
cran="http://cran.r-project.org"
)
mirrorUrl <- paste(mirrorUrls[mirrorType], "/src/contrib/", sep="")
stopifnot(mirrorType %in% c("bioc","cran"))
packagesFile <- paste(mirrorType, 'packages.nix', sep='-')
readFormatted <- as.data.table(read.table(skip=6, sep='"', text=head(readLines(packagesFile), -1)))
mirrorUrls <- list( bioc="http://bioconductor.statistik.tu-dortmund.de/packages/3.2/bioc/src/contrib/"
, cran="http://cran.r-project.org/src/contrib/"
)
mirrorUrl <- mirrorUrls[mirrorType][[1]]
knownPackages <- lapply(mirrorUrls, function(url) as.data.table(available.packages(url, filters=c("R_version", "OS_type", "duplicates"))))
pkgs <- knownPackages[mirrorType][[1]]
setkey(pkgs, Package)
knownPackages <- c(unique(do.call("rbind", knownPackages)$Package))
knownPackages <- sapply(knownPackages, gsub, pattern=".", replacement="_", fixed=TRUE)
nixPrefetch <- function(name, version) {
prevV <- readFormatted$V2 == name & readFormatted$V4 == version
if (sum(prevV) == 1) as.character(readFormatted$V6[ prevV ]) else
system(paste0("nix-prefetch-url --type sha256 ", mirrorUrl, name, "_", version, ".tar.gz"), intern=TRUE)
}
formatPackage <- function(name, version, sha256, depends, imports, linkingTo, knownPackages) {
formatPackage <- function(name, version, sha256, depends, imports, linkingTo) {
attr <- gsub(".", "_", name, fixed=TRUE)
if (is.na(depends)) depends <- "";
depends <- unlist(strsplit(depends, split="[ \t\n]*,[ \t\n]*", fixed=FALSE))
depends <- c(depends, unlist(strsplit(imports, split="[ \t\n]*,[ \t\n]*", fixed=FALSE)))
depends <- c(depends, unlist(strsplit(linkingTo, split="[ \t\n]*,[ \t\n]*", fixed=FALSE)))
depends <- paste( if (is.na(depends)) "" else gsub("[ \t\n]+", "", depends)
, if (is.na(imports)) "" else gsub("[ \t\n]+", "", imports)
, if (is.na(linkingTo)) "" else gsub("[ \t\n]+", "", linkingTo)
, sep=","
)
depends <- unlist(strsplit(depends, split=",", fixed=TRUE))
depends <- sapply(depends, gsub, pattern="([^ \t\n(]+).*", replacement="\\1")
depends <- depends[depends %in% knownPackages]
depends <- sapply(depends, gsub, pattern=".", replacement="_", fixed=TRUE)
depends <- paste(depends, collapse=" ")
depends <- depends[depends %in% knownPackages]
depends <- paste(sort(unique(depends)), collapse=" ")
paste0(attr, " = derive { name=\"", name, "\"; version=\"", version, "\"; sha256=\"", sha256, "\"; depends=[", depends, "]; };")
}
clusterExport(cl, c("nixPrefetch","readFormatted", "mirrorUrl"))
clusterExport(cl, c("nixPrefetch","readFormatted", "mirrorUrl", "knownPackages"))
pkgs <- as.data.table(available.packages(mirrorUrl, filters=c("R_version", "OS_type", "duplicates")))
pkgs <- pkgs[order(Package)]
pkgs$sha256 <- parApply(cl, pkgs, 1, function(p) nixPrefetch(p[1], p[2]))
knownPackages <- unique(pkgs$Package)
nix <- apply(pkgs, 1, function(p) formatPackage(p[1], p[2], p[18], p[4], p[5], p[6], knownPackages))
nix <- apply(pkgs, 1, function(p) formatPackage(p[1], p[2], p[18], p[4], p[5], p[6]))
cat("# This file is generated from generate-r-packages.R. DO NOT EDIT.\n")
cat("# Execute the following command to update the file.\n")
cat("#\n")
cat(paste("# Rscript generate-r-packages.R", mirrorType, ">", packagesFile))
cat(paste("# Rscript generate-r-packages.R", mirrorType, ">new && mv new", packagesFile))
cat("\n\n")
cat("{ self, derive }: with self; {\n")
cat(paste(nix, collapse="\n"), "\n")

View file

@ -0,0 +1,21 @@
# Run
#
# nix-build test-evaluation.nix --dry-run
#
# to test whether the R package set evaluates properly.
let
config = {
allowBroken = true;
allowUnfree = true;
};
inherit (import ../../.. { inherit config; }) pkgs;
rWrapper = pkgs.rWrapper.override {
packages = pkgs.lib.filter pkgs.lib.isDerivation (pkgs.lib.attrValues pkgs.rPackages);
};
in
rWrapper