Merge branch 'master' into patch-3
This commit is contained in:
Graham Christensen
@@ -14,5 +14,5 @@ removeAttrs (import ../../pkgs/top-level/release.nix
|
||||
supportedSystems = [ "x86_64-linux" ];
|
||||
})
|
||||
[ # Remove jobs whose evaluation depends on a writable Nix store.
|
||||
"tarball" "unstable"
|
||||
"tarball" "unstable" "darwin-tested"
|
||||
]
|
||||
|
||||
@@ -6,7 +6,7 @@ GNOME_FTP=ftp.gnome.org/pub/GNOME/sources
|
||||
|
||||
# projects that don't follow the GNOME major versioning, or that we don't want to
|
||||
# programmatically update
|
||||
NO_GNOME_MAJOR='gtkhtml gdm'
|
||||
NO_GNOME_MAJOR="ghex gtkhtml gdm"
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 gnome_dir <show project>|<update project>|<update-all> [major.minor]" >&2
|
||||
|
||||
@@ -48,8 +48,8 @@ def get_maintainers(attr_name):
|
||||
@click.command()
|
||||
@click.option(
|
||||
'--jobset',
|
||||
default="nixos/release-16.09",
|
||||
help='Hydra project like nixos/release-16.09')
|
||||
default="nixos/release-17.03",
|
||||
help='Hydra project like nixos/release-17.03')
|
||||
def cli(jobset):
|
||||
"""
|
||||
Given a Hydra project, inspect latest evaluation
|
||||
@@ -74,13 +74,13 @@ def cli(jobset):
|
||||
# TODO: dependency failed without propagated builds
|
||||
for tr in d('img[alt="Failed"]').parents('tr'):
|
||||
a = pq(tr)('a')[1]
|
||||
print "- [ ] [{}]({})".format(a.text, a.get('href'))
|
||||
print("- [ ] [{}]({})".format(a.text, a.get('href')))
|
||||
|
||||
sys.stdout.flush()
|
||||
|
||||
maintainers = get_maintainers(a.text)
|
||||
if maintainers:
|
||||
print " - maintainers: {}".format(", ".join(map(lambda u: '@' + u, maintainers)))
|
||||
print(" - maintainers: {}".format(", ".join(map(lambda u: '@' + u, maintainers))))
|
||||
# TODO: print last three persons that touched this file
|
||||
# TODO: pinpoint the diff that broke this build, or maybe it's transient or maybe it never worked?
|
||||
|
||||
|
||||
277
maintainers/scripts/nix-diff.sh
Executable file
277
maintainers/scripts/nix-diff.sh
Executable file
@@ -0,0 +1,277 @@
|
||||
#!/usr/bin/env nix-shell
|
||||
#! nix-shell -i bash -p coreutils gnugrep gnused
|
||||
|
||||
################################################################################
|
||||
# nix-diff.sh #
|
||||
################################################################################
|
||||
# This script "diffs" Nix profile generations. #
|
||||
# #
|
||||
# Example: #
|
||||
################################################################################
|
||||
# > nix-diff.sh 90 92 #
|
||||
# + gnumake-4.2.1 #
|
||||
# + gnumake-4.2.1-doc #
|
||||
# - htmldoc-1.8.29 #
|
||||
################################################################################
|
||||
# The example shows that as of generation 92 and since generation 90, #
|
||||
# gnumake-4.2.1 and gnumake-4.2.1-doc have been installed, while #
|
||||
# htmldoc-1.8.29 has been removed. #
|
||||
# #
|
||||
# The example above shows the default, minimal output mode of this script. #
|
||||
# For more features, run `nix-diff.sh -h` for usage instructions. #
|
||||
################################################################################
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
usage: nix-diff.sh [-h | [-p profile | -s] [-q] [-l] [range]]
|
||||
-h: print this message before exiting
|
||||
-q: list the derivations installed in the parent generation
|
||||
-l: diff every available intermediate generation between parent and
|
||||
child
|
||||
-p profile: specify the Nix profile to use
|
||||
* defaults to ~/.nix-profile
|
||||
-s: use the system profile
|
||||
* equivalent to: -p /nix/var/nix/profiles/system
|
||||
profile: * should be something like /nix/var/nix/profiles/default, not a
|
||||
generation link like /nix/var/nix/profiles/default-2-link
|
||||
range: the range of generations to diff
|
||||
* the following patterns are allowed, where A, B, and N are positive
|
||||
integers, and G is the currently active generation:
|
||||
A..B => diffs from generation A to generation B
|
||||
~N => diffs from the Nth newest generation (older than G) to G
|
||||
A => diffs from generation A to G
|
||||
* defaults to ~1
|
||||
EOF
|
||||
}
|
||||
|
||||
usage_tip() {
|
||||
echo 'run `nix-diff.sh -h` for usage instructions' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
while getopts :hqlp:s opt; do
|
||||
case $opt in
|
||||
h)
|
||||
usage
|
||||
exit
|
||||
;;
|
||||
q)
|
||||
opt_query=1
|
||||
;;
|
||||
l)
|
||||
opt_log=1
|
||||
;;
|
||||
p)
|
||||
opt_profile=$OPTARG
|
||||
;;
|
||||
s)
|
||||
opt_profile=/nix/var/nix/profiles/system
|
||||
;;
|
||||
\?)
|
||||
echo "error: invalid option -$OPTARG" >&2
|
||||
usage_tip
|
||||
;;
|
||||
esac
|
||||
done
|
||||
shift $((OPTIND-1))
|
||||
|
||||
if [ -n "$opt_profile" ]; then
|
||||
if ! [ -L "$opt_profile" ]; then
|
||||
echo "error: expecting \`$opt_profile\` to be a symbolic link" >&2
|
||||
usage_tip
|
||||
fi
|
||||
else
|
||||
opt_profile=$(readlink ~/.nix-profile)
|
||||
if (( $? != 0 )); then
|
||||
echo 'error: unable to dereference `~/.nix-profile`' >&2
|
||||
echo 'specify the profile manually with the `-p` flag' >&2
|
||||
usage_tip
|
||||
fi
|
||||
fi
|
||||
|
||||
list_gens() {
|
||||
nix-env -p "$opt_profile" --list-generations \
|
||||
| sed -r 's:^\s*::' \
|
||||
| cut -d' ' -f1
|
||||
}
|
||||
|
||||
current_gen() {
|
||||
nix-env -p "$opt_profile" --list-generations \
|
||||
| grep -E '\(current\)\s*$' \
|
||||
| sed -r 's:^\s*::' \
|
||||
| cut -d' ' -f1
|
||||
}
|
||||
|
||||
neg_gen() {
|
||||
local i=0 from=$1 n=$2 tmp
|
||||
for gen in $(list_gens | sort -rn); do
|
||||
if ((gen < from)); then
|
||||
tmp=$gen
|
||||
((i++))
|
||||
((i == n)) && break
|
||||
fi
|
||||
done
|
||||
if ((i < n)); then
|
||||
echo -n "error: there aren't $n generation(s) older than" >&2
|
||||
echo " generation $from" >&2
|
||||
return 1
|
||||
fi
|
||||
echo $tmp
|
||||
}
|
||||
|
||||
match() {
|
||||
argv=("$@")
|
||||
for i in $(seq $(($#-1))); do
|
||||
if grep -E "^${argv[$i]}\$" <(echo "$1") >/dev/null; then
|
||||
echo $i
|
||||
return
|
||||
fi
|
||||
done
|
||||
echo 0
|
||||
}
|
||||
|
||||
case $(match "$1" '' '[0-9]+' '[0-9]+\.\.[0-9]+' '~[0-9]+') in
|
||||
1)
|
||||
diffTo=$(current_gen)
|
||||
diffFrom=$(neg_gen $diffTo 1)
|
||||
(($? == 1)) && usage_tip
|
||||
;;
|
||||
2)
|
||||
diffFrom=$1
|
||||
diffTo=$(current_gen)
|
||||
;;
|
||||
3)
|
||||
diffFrom=${1%%.*}
|
||||
diffTo=${1##*.}
|
||||
;;
|
||||
4)
|
||||
diffTo=$(current_gen)
|
||||
diffFrom=$(neg_gen $diffTo ${1#*~})
|
||||
(($? == 1)) && usage_tip
|
||||
;;
|
||||
0)
|
||||
echo 'error: invalid invocation' >&2
|
||||
usage_tip
|
||||
;;
|
||||
esac
|
||||
|
||||
dirA="${opt_profile}-${diffFrom}-link"
|
||||
dirB="${opt_profile}-${diffTo}-link"
|
||||
|
||||
declare -a temp_files
|
||||
temp_length() {
|
||||
echo -n ${#temp_files[@]}
|
||||
}
|
||||
temp_make() {
|
||||
temp_files[$(temp_length)]=$(mktemp)
|
||||
}
|
||||
temp_clean() {
|
||||
rm -f ${temp_files[@]}
|
||||
}
|
||||
temp_name() {
|
||||
echo -n "${temp_files[$(($(temp_length)-1))]}"
|
||||
}
|
||||
trap 'temp_clean' EXIT
|
||||
|
||||
temp_make
|
||||
versA=$(temp_name)
|
||||
refs=$(nix-store -q --references "$dirA")
|
||||
(( $? != 0 )) && exit 1
|
||||
echo "$refs" \
|
||||
| grep -v env-manifest.nix \
|
||||
| sort \
|
||||
> "$versA"
|
||||
|
||||
print_tag() {
|
||||
local gen=$1
|
||||
nix-env -p "$opt_profile" --list-generations \
|
||||
| grep -E "^\s*${gen}" \
|
||||
| sed -r 's:^\s*::' \
|
||||
| sed -r 's:\s*$::'
|
||||
}
|
||||
|
||||
if [ -n "$opt_query" ]; then
|
||||
print_tag $diffFrom
|
||||
cat "$versA" \
|
||||
| sed -r 's:^[^-]+-(.*)$: \1:'
|
||||
|
||||
print_line=1
|
||||
fi
|
||||
|
||||
if [ -n "$opt_log" ]; then
|
||||
gens=$(for gen in $(list_gens); do
|
||||
((diffFrom < gen && gen < diffTo)) && echo $gen
|
||||
done)
|
||||
# Force the $diffTo generation to be included in this list, instead of using
|
||||
# `gen <= diffTo` in the preceding loop, so we encounter an error upon the
|
||||
# event of its nonexistence.
|
||||
gens=$(echo "$gens"
|
||||
echo $diffTo)
|
||||
else
|
||||
gens=$diffTo
|
||||
fi
|
||||
|
||||
temp_make
|
||||
add=$(temp_name)
|
||||
temp_make
|
||||
rem=$(temp_name)
|
||||
temp_make
|
||||
out=$(temp_name)
|
||||
|
||||
for gen in $gens; do
|
||||
|
||||
[ -n "$print_line" ] && echo
|
||||
|
||||
temp_make
|
||||
versB=$(temp_name)
|
||||
|
||||
dirB="${opt_profile}-${gen}-link"
|
||||
refs=$(nix-store -q --references "$dirB")
|
||||
(( $? != 0 )) && exit 1
|
||||
echo "$refs" \
|
||||
| grep -v env-manifest.nix \
|
||||
| sort \
|
||||
> "$versB"
|
||||
|
||||
in=$(comm -3 -1 "$versA" "$versB")
|
||||
sed -r 's:^[^-]*-(.*)$:\1+:' <(echo "$in") \
|
||||
| sort -f \
|
||||
> "$add"
|
||||
|
||||
un=$(comm -3 -2 "$versA" "$versB")
|
||||
sed -r 's:^[^-]*-(.*)$:\1-:' <(echo "$un") \
|
||||
| sort -f \
|
||||
> "$rem"
|
||||
|
||||
cat "$rem" "$add" \
|
||||
| sort -f \
|
||||
| sed -r 's:(.*)-$:- \1:' \
|
||||
| sed -r 's:(.*)\+$:\+ \1:' \
|
||||
| grep -v '^$' \
|
||||
> "$out"
|
||||
|
||||
if [ -n "$opt_query" -o -n "$opt_log" ]; then
|
||||
|
||||
lines=$(wc -l "$out" | cut -d' ' -f1)
|
||||
tag=$(print_tag "$gen")
|
||||
(( $? != 0 )) && exit 1
|
||||
if [ $lines -eq 0 ]; then
|
||||
echo "$tag (no change)"
|
||||
else
|
||||
echo "$tag"
|
||||
fi
|
||||
cat "$out" \
|
||||
| sed 's:^: :'
|
||||
|
||||
print_line=1
|
||||
|
||||
else
|
||||
echo "diffing from generation $diffFrom to $diffTo"
|
||||
cat "$out"
|
||||
fi
|
||||
|
||||
versA=$versB
|
||||
|
||||
done
|
||||
|
||||
exit 0
|
||||
@@ -1,260 +1,115 @@
|
||||
#!/bin/sh
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
usage () {
|
||||
echo 1>&2 "
|
||||
usage:
|
||||
$0
|
||||
[--git commit..commit | --git commit]
|
||||
[--svn rev:rev | --svn rev]
|
||||
[--path path[:path]*]
|
||||
[--help]
|
||||
|
||||
This program is used to investigate how any changes inside your nixpkgs
|
||||
repository may hurt. With these kind of information you may choose wisely
|
||||
where you should commit your changes.
|
||||
|
||||
This program adapts it-self to your versionning system to avoid too much
|
||||
effort on your Internet bandwidth. If you need to check more than one
|
||||
commits / revisions, you may use the following commands:
|
||||
|
||||
--git remotes/trunk..master
|
||||
--svn 17670:17677
|
||||
|
||||
Check the differences between each commit separating the first and the
|
||||
last commit.
|
||||
|
||||
--path /etc/nixos/nixpkgs:/tmp/nixpkgs_1:/tmp/nixpkgs_2
|
||||
|
||||
Check the differences between multiple directories containing different
|
||||
versions of nixpkgs.
|
||||
|
||||
All these options exist with one commit / revision argument. Such options
|
||||
are used to compare your \$NIXPKGS path with the specified version.
|
||||
|
||||
If you omit to mention any other commit / revision, then your \$NIXPKGS path
|
||||
is compared with its last update. This command is useful to test code from
|
||||
a dirty repository.
|
||||
|
||||
"
|
||||
|
||||
exit 1;
|
||||
}
|
||||
|
||||
#####################
|
||||
# Process Arguments #
|
||||
#####################
|
||||
|
||||
: ${NIXPKGS=/etc/nixos/nixpkgs/}
|
||||
|
||||
vcs=""
|
||||
gitCommits=""
|
||||
svnRevisions=""
|
||||
pathLocations=""
|
||||
verbose=false
|
||||
|
||||
argfun=""
|
||||
for arg; do
|
||||
if test -z "$argfun"; then
|
||||
case $arg in
|
||||
--git) vcs="git"; argfun="set_gitCommits";;
|
||||
--svn) vcs="svn"; argfun="set_svnRevisions";;
|
||||
--path) vcs="path"; argfun="set_pathLocations";;
|
||||
--verbose) verbose=true;;
|
||||
--help) usage;;
|
||||
*) usage;;
|
||||
esac
|
||||
else
|
||||
case $argfun in
|
||||
set_*)
|
||||
var=$(echo $argfun | sed 's,^set_,,')
|
||||
eval $var=$arg
|
||||
;;
|
||||
esac
|
||||
argfun=""
|
||||
fi
|
||||
done
|
||||
|
||||
if $verbose; then
|
||||
set -x
|
||||
else
|
||||
set +x
|
||||
if [ "$#" != 1 ] && [ "$#" != 2 ]; then
|
||||
cat <<-EOF
|
||||
Usage: $0 commit-spec [commit-spec]
|
||||
You need to be in a git-controlled nixpkgs tree.
|
||||
The current state of the tree will be used if the second commit is missing.
|
||||
EOF
|
||||
exit 1
|
||||
fi
|
||||
|
||||
############################
|
||||
# Find the repository type #
|
||||
############################
|
||||
# A slightly hacky way to get the config.
|
||||
parallel="$(echo 'config.rebuild-amount.parallel or false' | nix-repl . 2>/dev/null \
|
||||
| grep -v '^\(nix-repl.*\)\?$' | tail -n 1 || true)"
|
||||
|
||||
if test -z "$vcs"; then
|
||||
if test -x "$NIXPKGS/.git"; then
|
||||
if git --git-dir="$NIXPKGS/.git" branch > /dev/null 2>&1; then
|
||||
vcs="git"
|
||||
gitCommits=$(git --git-dir="$NIXPKGS/.git" log -n 1 --pretty=format:%H 2> /dev/null)
|
||||
fi
|
||||
elif test -x "$NIXPKGS/.svn"; then
|
||||
cd "$NIXPKGS"
|
||||
if svn info > /dev/null 2>&1; then
|
||||
vcs="svn";
|
||||
svnRevisions=$(svn info | sed -n 's,Revision: ,,p')
|
||||
fi
|
||||
cd -
|
||||
else
|
||||
usage
|
||||
fi
|
||||
fi
|
||||
echo "Estimating rebuild amount by counting changed Hydra jobs."
|
||||
|
||||
###############################
|
||||
# Define a storage directory. #
|
||||
###############################
|
||||
toRemove=()
|
||||
|
||||
pkgListDir=""
|
||||
exitCode=1
|
||||
cleanup(){
|
||||
test -e "$pkgListDir" && rm -rf "$pkgListDir"
|
||||
exit $exitCode;
|
||||
cleanup() {
|
||||
rm -rf "${toRemove[@]}"
|
||||
}
|
||||
|
||||
trap cleanup EXIT SIGINT SIGQUIT ERR
|
||||
|
||||
pkgListDir=$(mktemp --tmpdir -d rebuild-amount-XXXXXXXX)
|
||||
vcsDir="$pkgListDir/.vcs"
|
||||
MKTEMP='mktemp --tmpdir nix-rebuild-amount-XXXXXXXX'
|
||||
|
||||
###########################
|
||||
# Versionning for Dummies #
|
||||
###########################
|
||||
nixexpr() {
|
||||
cat <<-EONIX
|
||||
let
|
||||
lib = import $1/lib;
|
||||
hydraJobs = import $1/pkgs/top-level/release.nix
|
||||
# Compromise: accuracy vs. resources needed for evaluation.
|
||||
{ supportedSystems = cfg.systems or [ "x86_64-linux" "x86_64-darwin" ]; };
|
||||
cfg = (import $1 {}).config.rebuild-amount or {};
|
||||
|
||||
path_init() {
|
||||
if test "${pathLocations#*:}" = "$pathLocations"; then
|
||||
pathLocations="$NIXPKGS:$pathLocations"
|
||||
fi
|
||||
pathLocations="${pathLocations}:"
|
||||
recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; };
|
||||
|
||||
# hydraJobs leaves recurseForDerivations as empty attrmaps;
|
||||
# that would break nix-env and we also need to recurse everywhere.
|
||||
tweak = lib.mapAttrs
|
||||
(name: val:
|
||||
if name == "recurseForDerivations" then true
|
||||
else if lib.isAttrs val && val.type or null != "derivation"
|
||||
then recurseIntoAttrs (tweak val)
|
||||
else val
|
||||
);
|
||||
|
||||
# Some of these contain explicit references to platform(s) we want to avoid;
|
||||
# some even (transitively) depend on ~/.nixpkgs/config.nix (!)
|
||||
blacklist = [
|
||||
"tarball" "metrics" "manual"
|
||||
"darwin-tested" "unstable" "stdenvBootstrapTools"
|
||||
"moduleSystem" "lib-tests" # these just confuse the output
|
||||
];
|
||||
|
||||
in
|
||||
tweak (builtins.removeAttrs hydraJobs blacklist)
|
||||
EONIX
|
||||
}
|
||||
|
||||
path_getNext() {
|
||||
pathLoc="${pathLocations%%:*}"
|
||||
pathLocations="${pathLocations#*:}"
|
||||
# Output packages in tree $2 that weren't in $1.
|
||||
# Changing the output hash or name is taken as a change.
|
||||
# Extra nix-env parameters can be in $3
|
||||
newPkgs() {
|
||||
# We use files instead of pipes, as running multiple nix-env processes
|
||||
# could eat too much memory for a standard 4GiB machine.
|
||||
local -a list
|
||||
for i in 1 2; do
|
||||
local l="$($MKTEMP)"
|
||||
list[$i]="$l"
|
||||
toRemove+=("$l")
|
||||
|
||||
local expr="$($MKTEMP)"
|
||||
toRemove+=("$expr")
|
||||
nixexpr "${!i}" > "$expr"
|
||||
|
||||
nix-env -f "$expr" -qaP --no-name --out-path --show-trace $3 \
|
||||
| sort > "${list[$i]}" &
|
||||
|
||||
if [ "$parallel" != "true" ]; then
|
||||
wait
|
||||
fi
|
||||
done
|
||||
|
||||
wait
|
||||
comm -13 "${list[@]}"
|
||||
}
|
||||
|
||||
path_setPath() {
|
||||
path="$pathLoc"
|
||||
}
|
||||
# Prepare nixpkgs trees.
|
||||
declare -a tree
|
||||
for i in 1 2; do
|
||||
if [ -n "${!i}" ]; then # use the given commit
|
||||
dir="$($MKTEMP -d)"
|
||||
tree[$i]="$dir"
|
||||
toRemove+=("$dir")
|
||||
|
||||
path_setName() {
|
||||
name=$(echo "$pathLoc" | tr '/' '_')
|
||||
}
|
||||
|
||||
################
|
||||
# Git Commands #
|
||||
################
|
||||
|
||||
git_init() {
|
||||
git clone "$NIXPKGS/.git" "$vcsDir" > /dev/null 2>&1
|
||||
if echo "gitCommits" | grep -c "\.\." > /dev/null 2>&1; then
|
||||
gitCommits=$(git --git-dir="$vcsDir/.git" log --reverse --pretty=format:%H $gitCommits 2> /dev/null)
|
||||
else
|
||||
pathLocations="$vcsDir:$NIXPKGS"
|
||||
vcs="path"
|
||||
path_init
|
||||
fi
|
||||
}
|
||||
|
||||
git_getNext() {
|
||||
git --git-dir="$vcsDir/.git" checkout $(echo "$gitCommits" | head -n 1) > /dev/null 2>&1
|
||||
gitCommits=$(echo "$gitCommits" | sed '1 d')
|
||||
}
|
||||
|
||||
git_setPath() {
|
||||
path="$vcsDir"
|
||||
}
|
||||
|
||||
git_setName() {
|
||||
name=$(git --git-dir="$vcsDir/.git" log -n 1 --pretty=format:%H 2> /dev/null)
|
||||
}
|
||||
|
||||
#######################
|
||||
# Subversion Commands #
|
||||
#######################
|
||||
|
||||
svn_init() {
|
||||
cp -r "$NIXPKGS" "$vcsDir" > /dev/null 2>&1
|
||||
if echo "svnRevisions" | grep -c ":" > /dev/null 2>&1; then
|
||||
svnRevisions=$(seq ${svnRevisions%:*} ${svnRevisions#*:})
|
||||
else
|
||||
pathLocations="$vcsDir:$NIXPKGS"
|
||||
vcs="path"
|
||||
path_init
|
||||
fi
|
||||
}
|
||||
|
||||
svn_getNext() {
|
||||
cd "$vcsDir"
|
||||
svn checkout $(echo "$svnRevisions" | head -n 1) > /dev/null 2>&1
|
||||
cd -
|
||||
svnRevisions=$(echo "$svnRevisions" | sed '1 d')
|
||||
}
|
||||
|
||||
svn_setPath() {
|
||||
path="$vcsDir"
|
||||
}
|
||||
|
||||
svn_setName() {
|
||||
name=$(svn info 2> /dev/null | sed -n 's,Revision: ,,p')
|
||||
}
|
||||
|
||||
####################
|
||||
# Logical Commands #
|
||||
####################
|
||||
|
||||
init () { ${vcs}_init; }
|
||||
getNext () { ${vcs}_getNext; }
|
||||
setPath () { ${vcs}_setPath; }
|
||||
setName () { ${vcs}_setName; }
|
||||
|
||||
|
||||
#####################
|
||||
# Check for Rebuild #
|
||||
#####################
|
||||
|
||||
# Generate the list of all derivations that could be build from a nixpkgs
|
||||
# respository. This list of derivation hashes is compared with previous
|
||||
# lists and a brief summary is produced on the output.
|
||||
|
||||
compareNames () {
|
||||
nb=$(diff -y --suppress-common-lines --speed-large-files "$pkgListDir/$1.drvs" "$pkgListDir/$2.drvs" 2> /dev/null | wc -l)
|
||||
echo "$1 -> $2: $nb"
|
||||
}
|
||||
|
||||
echo "Please wait, this may take some minutes ..."
|
||||
|
||||
init
|
||||
first=""
|
||||
oldPrev=""
|
||||
|
||||
prev=""
|
||||
curr=""
|
||||
|
||||
while true; do
|
||||
getNext
|
||||
setPath # set path=...
|
||||
setName # set name=...
|
||||
curr="$name"
|
||||
|
||||
test -z "$curr" && break || true
|
||||
|
||||
nix-instantiate "$path" > "$pkgListDir/$curr.drvs" > /dev/null 2>&1 || true
|
||||
|
||||
if test -n "$prev"; then
|
||||
compareNames "$prev" "$curr"
|
||||
else
|
||||
echo "Number of package to rebuild:"
|
||||
first="$curr"
|
||||
fi
|
||||
oldPrev="$prev"
|
||||
prev="$curr"
|
||||
git clone --shared --no-checkout --quiet . "${tree[$i]}"
|
||||
(cd "${tree[$i]}" && git checkout --quiet "${!i}")
|
||||
else #use the current tree
|
||||
tree[$i]="$(pwd)"
|
||||
fi
|
||||
done
|
||||
|
||||
if test "$first" != "$oldPrev"; then
|
||||
echo "Number of package to rebuild (first -> last):"
|
||||
compareNames "$first" "$curr"
|
||||
fi
|
||||
newlist="$($MKTEMP)"
|
||||
toRemove+=("$newlist")
|
||||
# Notes:
|
||||
# - the evaluation is done on x86_64-linux, like on Hydra.
|
||||
# - using $newlist file so that newPkgs() isn't in a sub-shell (because of toRemove)
|
||||
newPkgs "${tree[1]}" "${tree[2]}" '--argstr system "x86_64-linux"' > "$newlist"
|
||||
|
||||
# Hacky: keep only the last word of each attribute path and sort.
|
||||
sed -n 's/\([^. ]*\.\)*\([^. ]*\) .*$/\2/p' < "$newlist" \
|
||||
| sort | uniq -c
|
||||
|
||||
exitCode=0
|
||||
|
||||
243
maintainers/scripts/update-python-libraries
Executable file
243
maintainers/scripts/update-python-libraries
Executable file
@@ -0,0 +1,243 @@
|
||||
#! /usr/bin/env nix-shell
|
||||
#! nix-shell -i python3 -p 'python3.withPackages(ps: with ps; [ requests toolz ])'
|
||||
|
||||
"""
|
||||
Update a Python package expression by passing in the `.nix` file, or the directory containing it.
|
||||
You can pass in multiple files or paths.
|
||||
|
||||
You'll likely want to use
|
||||
``
|
||||
$ ./update-python-libraries ../../pkgs/development/python-modules/*
|
||||
``
|
||||
to update all libraries in that folder.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import requests
|
||||
import toolz
|
||||
|
||||
INDEX = "https://pypi.io/pypi"
|
||||
"""url of PyPI"""
|
||||
|
||||
EXTENSIONS = ['tar.gz', 'tar.bz2', 'tar', 'zip', '.whl']
|
||||
"""Permitted file extensions. These are evaluated from left to right and the first occurance is returned."""
|
||||
|
||||
import logging
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
|
||||
def _get_values(attribute, text):
|
||||
"""Match attribute in text and return all matches.
|
||||
|
||||
:returns: List of matches.
|
||||
"""
|
||||
regex = '{}\s+=\s+"(.*)";'.format(attribute)
|
||||
regex = re.compile(regex)
|
||||
values = regex.findall(text)
|
||||
return values
|
||||
|
||||
def _get_unique_value(attribute, text):
|
||||
"""Match attribute in text and return unique match.
|
||||
|
||||
:returns: Single match.
|
||||
"""
|
||||
values = _get_values(attribute, text)
|
||||
n = len(values)
|
||||
if n > 1:
|
||||
raise ValueError("found too many values for {}".format(attribute))
|
||||
elif n == 1:
|
||||
return values[0]
|
||||
else:
|
||||
raise ValueError("no value found for {}".format(attribute))
|
||||
|
||||
def _get_line_and_value(attribute, text):
|
||||
"""Match attribute in text. Return the line and the value of the attribute."""
|
||||
regex = '({}\s+=\s+"(.*)";)'.format(attribute)
|
||||
regex = re.compile(regex)
|
||||
value = regex.findall(text)
|
||||
n = len(value)
|
||||
if n > 1:
|
||||
raise ValueError("found too many values for {}".format(attribute))
|
||||
elif n == 1:
|
||||
return value[0]
|
||||
else:
|
||||
raise ValueError("no value found for {}".format(attribute))
|
||||
|
||||
|
||||
def _replace_value(attribute, value, text):
|
||||
"""Search and replace value of attribute in text."""
|
||||
old_line, old_value = _get_line_and_value(attribute, text)
|
||||
new_line = old_line.replace(old_value, value)
|
||||
new_text = text.replace(old_line, new_line)
|
||||
return new_text
|
||||
|
||||
def _fetch_page(url):
|
||||
r = requests.get(url)
|
||||
if r.status_code == requests.codes.ok:
|
||||
return r.json()
|
||||
else:
|
||||
raise ValueError("request for {} failed".format(url))
|
||||
|
||||
def _get_latest_version_pypi(package, extension):
|
||||
"""Get latest version and hash from PyPI."""
|
||||
url = "{}/{}/json".format(INDEX, package)
|
||||
json = _fetch_page(url)
|
||||
|
||||
version = json['info']['version']
|
||||
for release in json['releases'][version]:
|
||||
if release['filename'].endswith(extension):
|
||||
# TODO: In case of wheel we need to do further checks!
|
||||
sha256 = release['digests']['sha256']
|
||||
break
|
||||
else:
|
||||
sha256 = None
|
||||
return version, sha256
|
||||
|
||||
|
||||
def _get_latest_version_github(package, extension):
|
||||
raise ValueError("updating from GitHub is not yet supported.")
|
||||
|
||||
|
||||
FETCHERS = {
|
||||
'fetchFromGitHub' : _get_latest_version_github,
|
||||
'fetchPypi' : _get_latest_version_pypi,
|
||||
'fetchurl' : _get_latest_version_pypi,
|
||||
}
|
||||
|
||||
|
||||
DEFAULT_SETUPTOOLS_EXTENSION = 'tar.gz'
|
||||
|
||||
|
||||
FORMATS = {
|
||||
'setuptools' : DEFAULT_SETUPTOOLS_EXTENSION,
|
||||
'wheel' : 'whl'
|
||||
}
|
||||
|
||||
def _determine_fetcher(text):
|
||||
# Count occurences of fetchers.
|
||||
nfetchers = sum(text.count('src = {}'.format(fetcher)) for fetcher in FETCHERS.keys())
|
||||
if nfetchers == 0:
|
||||
raise ValueError("no fetcher.")
|
||||
elif nfetchers > 1:
|
||||
raise ValueError("multiple fetchers.")
|
||||
else:
|
||||
# Then we check which fetcher to use.
|
||||
for fetcher in FETCHERS.keys():
|
||||
if 'src = {}'.format(fetcher) in text:
|
||||
return fetcher
|
||||
|
||||
|
||||
def _determine_extension(text, fetcher):
|
||||
"""Determine what extension is used in the expression.
|
||||
|
||||
If we use:
|
||||
- fetchPypi, we check if format is specified.
|
||||
- fetchurl, we determine the extension from the url.
|
||||
- fetchFromGitHub we simply use `.tar.gz`.
|
||||
"""
|
||||
if fetcher == 'fetchPypi':
|
||||
try:
|
||||
format = _get_unique_value('format', text)
|
||||
except ValueError as e:
|
||||
format = None # format was not given
|
||||
|
||||
try:
|
||||
extension = _get_unique_value('extension', text)
|
||||
except ValueError as e:
|
||||
extension = None # extension was not given
|
||||
|
||||
if extension is None:
|
||||
if format is None:
|
||||
format = 'setuptools'
|
||||
extension = FORMATS[format]
|
||||
|
||||
elif fetcher == 'fetchurl':
|
||||
url = _get_unique_value('url', text)
|
||||
extension = os.path.splitext(url)[1]
|
||||
if 'pypi' not in url:
|
||||
raise ValueError('url does not point to PyPI.')
|
||||
|
||||
elif fetcher == 'fetchFromGitHub':
|
||||
raise ValueError('updating from GitHub is not yet implemented.')
|
||||
|
||||
return extension
|
||||
|
||||
|
||||
def _update_package(path):
|
||||
|
||||
|
||||
|
||||
# Read the expression
|
||||
with open(path, 'r') as f:
|
||||
text = f.read()
|
||||
|
||||
# Determine pname.
|
||||
pname = _get_unique_value('pname', text)
|
||||
|
||||
# Determine version.
|
||||
version = _get_unique_value('version', text)
|
||||
|
||||
# First we check how many fetchers are mentioned.
|
||||
fetcher = _determine_fetcher(text)
|
||||
|
||||
extension = _determine_extension(text, fetcher)
|
||||
|
||||
new_version, new_sha256 = _get_latest_version_pypi(pname, extension)
|
||||
|
||||
if new_version == version:
|
||||
logging.info("Path {}: no update available for {}.".format(path, pname))
|
||||
return False
|
||||
if not new_sha256:
|
||||
raise ValueError("no file available for {}.".format(pname))
|
||||
|
||||
text = _replace_value('version', new_version, text)
|
||||
text = _replace_value('sha256', new_sha256, text)
|
||||
|
||||
with open(path, 'w') as f:
|
||||
f.write(text)
|
||||
|
||||
logging.info("Path {}: updated {} from {} to {}".format(path, pname, version, new_version))
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _update(path):
|
||||
|
||||
# We need to read and modify a Nix expression.
|
||||
if os.path.isdir(path):
|
||||
path = os.path.join(path, 'default.nix')
|
||||
|
||||
# If a default.nix does not exist, we quit.
|
||||
if not os.path.isfile(path):
|
||||
logging.info("Path {}: does not exist.".format(path))
|
||||
return False
|
||||
|
||||
# If file is not a Nix expression, we quit.
|
||||
if not path.endswith(".nix"):
|
||||
logging.info("Path {}: does not end with `.nix`.".format(path))
|
||||
return False
|
||||
|
||||
try:
|
||||
return _update_package(path)
|
||||
except ValueError as e:
|
||||
logging.warning("Path {}: {}".format(path, e))
|
||||
return False
|
||||
|
||||
def main():
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('package', type=str, nargs='+')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
packages = map(os.path.abspath, args.package)
|
||||
|
||||
count = list(map(_update, packages))
|
||||
|
||||
logging.info("{} package(s) updated".format(sum(count)))
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user