Fix updating of the tarball mirror
This commit is contained in:
Eelco Dolstra
parent
aea043aec0
commit
7a51e17775
@ -1,45 +0,0 @@
|
|||||||
#! /bin/sh -e
|
|
||||||
|
|
||||||
distDir=${NIX_TARBALLS_CACHE:-/tarballs}
|
|
||||||
|
|
||||||
url="$1"
|
|
||||||
file="$2"
|
|
||||||
if [ -z "$url" ]; then echo "syntax: $0 URL"; exit 0; fi
|
|
||||||
|
|
||||||
base="$(basename "$url")"
|
|
||||||
if [ -z "$base" ]; then echo "bad URL"; exit 1; fi
|
|
||||||
dstPath="$distDir/$base"
|
|
||||||
|
|
||||||
if [ -e "$dstPath" ]; then if [ -n "$VERBOSE" ]; then echo "$dstPath already exists"; fi; exit 0; fi
|
|
||||||
|
|
||||||
if [ -z "$file" ]; then
|
|
||||||
|
|
||||||
echo "downloading $url to $dstPath"
|
|
||||||
|
|
||||||
if [ -n "$DRY_RUN" ]; then exit 0; fi
|
|
||||||
|
|
||||||
declare -a res
|
|
||||||
if ! res=($(PRINT_PATH=1 nix-prefetch-url "$url")); then
|
|
||||||
exit
|
|
||||||
fi
|
|
||||||
|
|
||||||
storePath=${res[1]}
|
|
||||||
|
|
||||||
else
|
|
||||||
storePath="$file"
|
|
||||||
fi
|
|
||||||
|
|
||||||
cp $storePath "$dstPath.tmp.$$"
|
|
||||||
mv -f "$dstPath.tmp.$$" "$dstPath"
|
|
||||||
|
|
||||||
echo "hashing $dstPath"
|
|
||||||
|
|
||||||
md5=$(nix-hash --flat --type md5 "$dstPath")
|
|
||||||
ln -sfn "../$base" $distDir/md5/$md5
|
|
||||||
|
|
||||||
sha1=$(nix-hash --flat --type sha1 "$dstPath")
|
|
||||||
ln -sfn "../$base" $distDir/sha1/$sha1
|
|
||||||
|
|
||||||
sha256=$(nix-hash --flat --type sha256 "$dstPath")
|
|
||||||
ln -sfn "../$base" $distDir/sha256/$sha256
|
|
||||||
ln -sfn "../$base" $distDir/sha256/$(nix-hash --type sha256 --to-base32 "$sha256")
|
|
||||||
95
maintainers/scripts/copy-tarballs.pl
Executable file
95
maintainers/scripts/copy-tarballs.pl
Executable file
@ -0,0 +1,95 @@
|
|||||||
|
#! /run/current-system/sw/bin/perl -w
|
||||||
|
|
||||||
|
use strict;
|
||||||
|
use XML::Simple;
|
||||||
|
use File::Basename;
|
||||||
|
use File::Path;
|
||||||
|
use File::Copy 'cp';
|
||||||
|
use IPC::Open2;
|
||||||
|
use Nix::Store;
|
||||||
|
|
||||||
|
my $myDir = dirname($0);
|
||||||
|
|
||||||
|
my $tarballsCache = $ENV{'NIX_TARBALLS_CACHE'} // "/tarballs";
|
||||||
|
|
||||||
|
my $xml = `nix-instantiate --eval-only --xml --strict '<nixpkgs/maintainers/scripts/find-tarballs.nix>'`;
|
||||||
|
die "$0: evaluation failed\n" if $? != 0;
|
||||||
|
|
||||||
|
my $data = XMLin($xml) or die;
|
||||||
|
|
||||||
|
mkpath($tarballsCache);
|
||||||
|
mkpath("$tarballsCache/md5");
|
||||||
|
mkpath("$tarballsCache/sha1");
|
||||||
|
mkpath("$tarballsCache/sha256");
|
||||||
|
|
||||||
|
foreach my $file (@{$data->{list}->{attrs}}) {
|
||||||
|
my $url = $file->{attr}->{url}->{string}->{value};
|
||||||
|
my $algo = $file->{attr}->{type}->{string}->{value};
|
||||||
|
my $hash = $file->{attr}->{hash}->{string}->{value};
|
||||||
|
|
||||||
|
if ($url !~ /^http:/ && $url !~ /^https:/ && $url !~ /^ftp:/ && $url !~ /^mirror:/) {
|
||||||
|
print STDERR "skipping $url (unsupported scheme)\n";
|
||||||
|
next;
|
||||||
|
}
|
||||||
|
|
||||||
|
$url =~ /([^\/]+)$/;
|
||||||
|
my $fn = $1;
|
||||||
|
|
||||||
|
if (!defined $fn) {
|
||||||
|
print STDERR "skipping $url (no file name)\n";
|
||||||
|
next;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($fn =~ /[&?=%]/ || $fn =~ /^\./) {
|
||||||
|
print STDERR "skipping $url (bad character in file name)\n";
|
||||||
|
next;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($fn !~ /[a-zA-Z]/) {
|
||||||
|
print STDERR "skipping $url (no letter in file name)\n";
|
||||||
|
next;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($fn !~ /[0-9]/) {
|
||||||
|
print STDERR "skipping $url (no digit in file name)\n";
|
||||||
|
next;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($fn !~ /[-_\.]/) {
|
||||||
|
print STDERR "skipping $url (no dash/dot/underscore in file name)\n";
|
||||||
|
next;
|
||||||
|
}
|
||||||
|
|
||||||
|
my $dstPath = "$tarballsCache/$fn";
|
||||||
|
|
||||||
|
next if -e $dstPath;
|
||||||
|
|
||||||
|
print "downloading $url to $dstPath...\n";
|
||||||
|
|
||||||
|
next if $ENV{DRY_RUN};
|
||||||
|
|
||||||
|
$ENV{QUIET} = 1;
|
||||||
|
$ENV{PRINT_PATH} = 1;
|
||||||
|
my $fh;
|
||||||
|
my $pid = open($fh, "-|", "nix-prefetch-url", "--type", $algo, $url, $hash) or die;
|
||||||
|
waitpid($pid, 0) or die;
|
||||||
|
if ($? != 0) {
|
||||||
|
print STDERR "failed to fetch $url: $?\n";
|
||||||
|
last if $? >> 8 == 255;
|
||||||
|
next;
|
||||||
|
}
|
||||||
|
<$fh>; my $storePath = <$fh>; chomp $storePath;
|
||||||
|
|
||||||
|
die unless -e $storePath;
|
||||||
|
|
||||||
|
cp($storePath, $dstPath) or die;
|
||||||
|
|
||||||
|
my $md5 = hashFile("md5", 0, $storePath) or die;
|
||||||
|
symlink("../$fn", "$tarballsCache/md5/$md5");
|
||||||
|
|
||||||
|
my $sha1 = hashFile("sha1", 0, $storePath) or die;
|
||||||
|
symlink("../$fn", "$tarballsCache/sha1/$sha1");
|
||||||
|
|
||||||
|
my $sha256 = hashFile("sha256", 0, $storePath) or die;
|
||||||
|
symlink("../$fn", "$tarballsCache/sha256/$sha256");
|
||||||
|
}
|
||||||
@ -1,27 +0,0 @@
|
|||||||
#! /bin/sh -e
|
|
||||||
|
|
||||||
urls=$(nix-instantiate --eval-only --xml --strict '<nixpkgs/maintainers/scripts/eval-release.nix>' \
|
|
||||||
| grep -A2 'name="urls"' \
|
|
||||||
| grep '<string value=' \
|
|
||||||
| sed 's/.*"\(.*\)".*/\1/' \
|
|
||||||
| sort | uniq)
|
|
||||||
|
|
||||||
for url in $urls; do
|
|
||||||
if echo "$url" | grep -q -E "www.cs.uu.nl|nixos.org|.stratego-language.org|java.sun.com|ut2004|linuxq3a|RealPlayer|Adbe|belastingdienst|microsoft|armijn/.nix|sun.com|archive.eclipse.org"; then continue; fi
|
|
||||||
|
|
||||||
# Check the URL scheme.
|
|
||||||
if ! echo "$url" | grep -q -E "^[a-z]+://"; then echo "skipping $url (no URL scheme)"; continue; fi
|
|
||||||
|
|
||||||
# Check the basename. It should include something resembling a version.
|
|
||||||
base="$(basename "$url")"
|
|
||||||
#if ! echo "$base" | grep -q -E "[-_].*[0-9].*"; then echo "skipping $url (no version)"; continue; fi
|
|
||||||
if ! echo "$base" | grep -q -E "[a-zA-Z]"; then echo "skipping $url (no letter in name)"; continue; fi
|
|
||||||
if ! echo "$base" | grep -q -E "[0-9]"; then echo "skipping $url (no digit in name)"; continue; fi
|
|
||||||
if ! echo "$base" | grep -q -E "[-_\.]"; then echo "skipping $url (no dot/underscore in name)"; continue; fi
|
|
||||||
if echo "$base" | grep -q -E "[&?=%]"; then echo "skipping $url (bad character in name)"; continue; fi
|
|
||||||
if [ "${base:0:1}" = "." ]; then echo "skipping $url (starts with a dot)"; continue; fi
|
|
||||||
|
|
||||||
$(dirname $0)/copy-tarball.sh "$url"
|
|
||||||
done
|
|
||||||
|
|
||||||
echo DONE
|
|
||||||
45
maintainers/scripts/find-tarballs.nix
Normal file
45
maintainers/scripts/find-tarballs.nix
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
# This expression returns a list of all fetchurl calls used by all
|
||||||
|
# packages reachable from release.nix.
|
||||||
|
|
||||||
|
with import ../.. { };
|
||||||
|
with lib;
|
||||||
|
|
||||||
|
let
|
||||||
|
|
||||||
|
root = removeAttrs (import ../../pkgs/top-level/release.nix { }) [ "tarball" "unstable" ];
|
||||||
|
|
||||||
|
uniqueUrls = map (x: x.file) (genericClosure {
|
||||||
|
startSet = map (file: { key = file.url; inherit file; }) urls;
|
||||||
|
operator = const [ ];
|
||||||
|
});
|
||||||
|
|
||||||
|
urls = map (drv: { url = head drv.urls; hash = drv.outputHash; type = drv.outputHashAlgo; }) fetchurlDependencies;
|
||||||
|
|
||||||
|
fetchurlDependencies = filter (drv: drv.outputHash or "" != "" && drv ? urls) dependencies;
|
||||||
|
|
||||||
|
dependencies = map (x: x.value) (genericClosure {
|
||||||
|
startSet = map keyDrv (derivationsIn' root);
|
||||||
|
operator = { key, value }: map keyDrv (immediateDependenciesOf value);
|
||||||
|
});
|
||||||
|
|
||||||
|
derivationsIn' = x:
|
||||||
|
if !canEval x then []
|
||||||
|
else if isDerivation x then optional (canEval x.drvPath) x
|
||||||
|
else if isList x then concatLists (map derivationsIn' x)
|
||||||
|
else if isAttrs x then concatLists (mapAttrsToList (n: v: derivationsIn' v) x)
|
||||||
|
else [ ];
|
||||||
|
|
||||||
|
keyDrv = drv: if canEval drv.drvPath then { key = drv.drvPath; value = drv; } else { };
|
||||||
|
|
||||||
|
immediateDependenciesOf = drv:
|
||||||
|
concatLists (mapAttrsToList (n: v: derivationsIn v) (removeAttrs drv ["meta" "passthru"]));
|
||||||
|
|
||||||
|
derivationsIn = x:
|
||||||
|
if !canEval x then []
|
||||||
|
else if isDerivation x then optional (canEval x.drvPath) x
|
||||||
|
else if isList x then concatLists (map derivationsIn x)
|
||||||
|
else [ ];
|
||||||
|
|
||||||
|
canEval = val: (builtins.tryEval val).success;
|
||||||
|
|
||||||
|
in uniqueUrls
|
||||||
@ -65,6 +65,10 @@ releaseTools.sourceTarball rec {
|
|||||||
header "checking eval-release.nix"
|
header "checking eval-release.nix"
|
||||||
nix-instantiate --eval-only --strict --show-trace ./maintainers/scripts/eval-release.nix > /dev/null
|
nix-instantiate --eval-only --strict --show-trace ./maintainers/scripts/eval-release.nix > /dev/null
|
||||||
stopNest
|
stopNest
|
||||||
|
|
||||||
|
header "checking find-tarballs.nix"
|
||||||
|
nix-instantiate --eval-only --strict --show-trace ./maintainers/scripts/find-tarballs.nix > /dev/null
|
||||||
|
stopNest
|
||||||
'';
|
'';
|
||||||
|
|
||||||
distPhase = ''
|
distPhase = ''
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user