public/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #75781 from grahamc/dockertools/remove-implementation-detail-layers

Browse Source 
dockertools.buildLayeredImage: remove implementation detail layers
This commit is contained in:
Graham Christensen 2019-12-17 16:03:11 -05:00 committed by GitHub
commit 64453c8dbd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 25 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
pkgs/build-support/docker/default.nix
View File

@ -287,10 +287,16 @@ rec {
# unless there are more paths than $maxLayers. In that case, create # unless there are more paths than $maxLayers. In that case, create
# $maxLayers-1 for the most popular layers, and smush the remainaing # $maxLayers-1 for the most popular layers, and smush the remainaing
# store paths in to one final layer. # store paths in to one final layer.
#
# NOTE: the `closures` parameter is a list of closures to include.
# The TOP LEVEL store paths themselves will never be present in the
# resulting image. At this time (2019-12-16) none of these layers
# are appropriate to include, as they are all created as
# implementation details of dockerTools.
mkManyPureLayers = { mkManyPureLayers = {
name, name,
# Files to add to the layer. # Files to add to the layer.
closure, closures,
configJson, configJson,
# Docker has a 125-layer maximum, we pick 100 to ensure there is # Docker has a 125-layer maximum, we pick 100 to ensure there is
# plenty of room for extension. # plenty of room for extension.
@ -303,10 +309,12 @@ rec {
isExecutable = true; isExecutable = true;
src = ./store-path-to-layer.sh; src = ./store-path-to-layer.sh;
}; };
overallClosure = writeText "closure" (lib.concatStringsSep " " closures);
in in
runCommand "${name}-granular-docker-layers" { runCommand "${name}-granular-docker-layers" {
inherit maxLayers; inherit maxLayers;
paths = referencesByPopularity closure; paths = referencesByPopularity overallClosure;
nativeBuildInputs = [ jshon rsync tarsum ]; nativeBuildInputs = [ jshon rsync tarsum ];
enableParallelBuilding = true; enableParallelBuilding = true;
} }
@ -317,15 +325,20 @@ rec {
| jshon -d config \ | jshon -d config \
| jshon -s "1970-01-01T00:00:01Z" -i created > generic.json | jshon -s "1970-01-01T00:00:01Z" -i created > generic.json
# WARNING! # WARNING!
# The following code is fiddly w.r.t. ensuring every layer is # The following code is fiddly w.r.t. ensuring every layer is
# created, and that no paths are missed. If you change the # created, and that no paths are missed. If you change the
# following head and tail call lines, double-check that your # following head and tail call lines, double-check that your
# code behaves properly when the number of layers equals: # code behaves properly when the number of layers equals:
# maxLayers-1, maxLayers, and maxLayers+1 # maxLayers-1, maxLayers, and maxLayers+1
head -n $((maxLayers - 1)) $paths | cat -n | xargs -P$NIX_BUILD_CORES -n2 ${storePathToLayer} paths() {
if [ $(cat $paths | wc -l) -ge $maxLayers ]; then cat $paths ${lib.concatMapStringsSep " " (path: "| grep -v ${path}") (closures ++ [ overallClosure ])}
tail -n+$maxLayers $paths | xargs ${storePathToLayer} $maxLayers }
paths | head -n $((maxLayers - 1)) | cat -n | xargs -P$NIX_BUILD_CORES -n2 ${storePathToLayer}
if [ $(paths | wc -l) -ge $maxLayers ]; then
paths | tail -n+$maxLayers | xargs ${storePathToLayer} $maxLayers
fi fi
echo "Finished building layer '$name'" echo "Finished building layer '$name'"
@ -534,7 +547,12 @@ rec {
}: }:
let let
baseName = baseNameOf name; baseName = baseNameOf name;
contentsEnv = symlinkJoin { name = "bulk-layers"; paths = (if builtins.isList contents then contents else [ contents ]); }; contentsEnv = symlinkJoin {
name = "bulk-layers";
paths = if builtins.isList contents
then contents
else [ contents ];
};
configJson = let configJson = let
pure = writeText "${baseName}-config.json" (builtins.toJSON { pure = writeText "${baseName}-config.json" (builtins.toJSON {
@ -551,7 +569,7 @@ rec {
bulkLayers = mkManyPureLayers { bulkLayers = mkManyPureLayers {
name = baseName; name = baseName;
closure = writeText "closure" "${contentsEnv} ${configJson}"; closures = [ contentsEnv configJson ];
# One layer will be taken up by the customisationLayer, so # One layer will be taken up by the customisationLayer, so
# take up one less. # take up one less.
maxLayers = maxLayers - 1; maxLayers = maxLayers - 1;