d3d3eedf7f
* guix/docker.scm (%docker-image-max-layers): New variable. (size-sorted-store-items, create-empty-tar): New procedures. (config, manifest, build-docker-image): Build layered images. Change-Id: I4c8846bff0a3ceccb77e6bdf95d4942e5c3efe41
405 lines
17 KiB
Scheme
405 lines
17 KiB
Scheme
;;; GNU Guix --- Functional package management for GNU
|
|
;;; Copyright © 2017 Ricardo Wurmus <rekado@elephly.net>
|
|
;;; Copyright © 2017, 2018, 2019, 2021 Ludovic Courtès <ludo@gnu.org>
|
|
;;; Copyright © 2018 Chris Marusich <cmmarusich@gmail.com>
|
|
;;; Copyright © 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com>
|
|
;;; Copyright © 2023 Oleg Pykhalov <go.wigust@gmail.com>
|
|
;;;
|
|
;;; This file is part of GNU Guix.
|
|
;;;
|
|
;;; GNU Guix is free software; you can redistribute it and/or modify it
|
|
;;; under the terms of the GNU General Public License as published by
|
|
;;; the Free Software Foundation; either version 3 of the License, or (at
|
|
;;; your option) any later version.
|
|
;;;
|
|
;;; GNU Guix is distributed in the hope that it will be useful, but
|
|
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
;;; GNU General Public License for more details.
|
|
;;;
|
|
;;; You should have received a copy of the GNU General Public License
|
|
;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
(define-module (guix docker)
|
|
#:use-module (gcrypt hash)
|
|
#:use-module (guix base16)
|
|
#:use-module (guix build pack)
|
|
#:use-module ((guix build utils)
|
|
#:select (mkdir-p
|
|
delete-file-recursively
|
|
with-directory-excursion
|
|
invoke))
|
|
#:use-module (gnu build install)
|
|
#:use-module ((guix build store-copy)
|
|
#:select (file-size))
|
|
#:use-module (json) ;guile-json
|
|
#:use-module (srfi srfi-1)
|
|
#:use-module (srfi srfi-19)
|
|
#:use-module (srfi srfi-26)
|
|
#:use-module (srfi srfi-71)
|
|
#:use-module ((texinfo string-utils)
|
|
#:select (escape-special-chars))
|
|
#:use-module (rnrs bytevectors)
|
|
#:use-module (ice-9 ftw)
|
|
#:use-module (ice-9 match)
|
|
#:export (%docker-image-max-layers
|
|
build-docker-image))
|
|
|
|
;; The maximum number of layers allowed in a Docker image is typically around
|
|
;; 128, although it may vary depending on the Docker daemon. However, we
|
|
;; recommend setting the limit to 100 to ensure sufficient room for future
|
|
;; extensions.
|
|
(define %docker-image-max-layers
|
|
#f)
|
|
|
|
;; Generate a 256-bit identifier in hexadecimal encoding for the Docker image.
|
|
(define docker-id
|
|
(compose bytevector->base16-string sha256 string->utf8))
|
|
|
|
(define (layer-diff-id layer)
|
|
"Generate a layer DiffID for the given LAYER archive."
|
|
(string-append "sha256:" (bytevector->base16-string (file-sha256 layer))))
|
|
|
|
;; This is the semantic version of the JSON metadata schema according to
|
|
;; https://github.com/docker/docker/blob/master/image/spec/v1.2.md
|
|
;; It is NOT the version of the image specification.
|
|
(define schema-version "1.0")
|
|
|
|
(define (image-description id time)
|
|
"Generate a simple image description."
|
|
`((id . ,id)
|
|
(created . ,time)
|
|
(container_config . #nil)))
|
|
|
|
(define (canonicalize-repository-name name)
|
|
"\"Repository\" names are restricted to roughly [a-z0-9_.-].
|
|
Return a version of TAG that follows these rules."
|
|
;; Refer to https://docs.docker.com/docker-hub/repos/.
|
|
(define min-length 2)
|
|
(define padding-character #\a)
|
|
(define max-length 255)
|
|
|
|
(define ascii-letters
|
|
(string->char-set "abcdefghijklmnopqrstuvwxyz"))
|
|
|
|
(define separators
|
|
(string->char-set "_-."))
|
|
|
|
(define repo-char-set
|
|
(char-set-union char-set:digit ascii-letters separators))
|
|
|
|
(define normalized-name
|
|
(string-map (lambda (chr)
|
|
(if (char-set-contains? repo-char-set chr)
|
|
chr
|
|
#\.))
|
|
(string-trim (string-downcase name) separators)))
|
|
|
|
(let ((l (string-length normalized-name)))
|
|
(match l
|
|
((? (cut > <> max-length))
|
|
(string-take normalized-name max-length))
|
|
((? (cut < <> min-length))
|
|
(string-append normalized-name
|
|
(make-string (- min-length l) padding-character)))
|
|
(_ normalized-name))))
|
|
|
|
(define* (manifest path layers #:optional (tag "guix"))
|
|
"Generate a simple image manifest."
|
|
(let ((tag (canonicalize-repository-name tag)))
|
|
`#(((Config . "config.json")
|
|
(RepoTags . #(,(string-append tag ":latest")))
|
|
(Layers . ,(list->vector layers))))))
|
|
|
|
;; According to the specifications this is required for backwards
|
|
;; compatibility. It duplicates information provided by the manifest.
|
|
(define* (repositories path id #:optional (tag "guix"))
|
|
"Generate a repositories file referencing PATH and the image ID."
|
|
`((,(canonicalize-repository-name tag) . ((latest . ,id)))))
|
|
|
|
;; See https://github.com/opencontainers/image-spec/blob/master/config.md
|
|
(define* (config layers-diff-ids time arch #:key entry-point (environment '()))
|
|
"Generate a minimal image configuration for the given LAYERS files."
|
|
;; "architecture" must be values matching "platform.arch" in the
|
|
;; runtime-spec at
|
|
;; https://github.com/opencontainers/runtime-spec/blob/v1.0.0-rc2/config.md#platform
|
|
`((architecture . ,arch)
|
|
(comment . "Generated by GNU Guix")
|
|
(created . ,time)
|
|
(config . ,`((env . ,(list->vector
|
|
(map (match-lambda
|
|
((name . value)
|
|
(string-append name "=" value)))
|
|
environment)))
|
|
,@(if entry-point
|
|
`((entrypoint . ,(list->vector entry-point)))
|
|
'())))
|
|
(container_config . #nil)
|
|
(os . "linux")
|
|
(rootfs . ((type . "layers")
|
|
(diff_ids . ,(list->vector layers-diff-ids))))))
|
|
|
|
(define directive-file
|
|
;; Return the file or directory created by a 'evaluate-populate-directive'
|
|
;; directive.
|
|
(match-lambda
|
|
((source '-> target)
|
|
(string-trim source #\/))
|
|
(('directory name _ ...)
|
|
(string-trim name #\/))))
|
|
|
|
(define (size-sorted-store-items items max-layers)
|
|
"Split list of ITEMS at %MAX-LAYERS and sort by disk usage."
|
|
(let* ((items-length (length items))
|
|
(head tail
|
|
(split-at
|
|
(map (match-lambda ((size . item) item))
|
|
(sort (map (lambda (item)
|
|
(cons (file-size item) item))
|
|
items)
|
|
(lambda (item1 item2)
|
|
(< (match item2 ((size . _) size))
|
|
(match item1 ((size . _) size))))))
|
|
(if (>= items-length max-layers)
|
|
(- max-layers 2)
|
|
(1- items-length)))))
|
|
(list head tail)))
|
|
|
|
(define (create-empty-tar file)
|
|
(invoke "tar" "-cf" file "--files-from" "/dev/null"))
|
|
|
|
(define* (build-docker-image image paths prefix
|
|
#:key
|
|
(repository "guix")
|
|
(extra-files '())
|
|
(transformations '())
|
|
(system (utsname:machine (uname)))
|
|
database
|
|
entry-point
|
|
(environment '())
|
|
compressor
|
|
(creation-time (current-time time-utc))
|
|
max-layers
|
|
root-system)
|
|
"Write to IMAGE a layerer Docker image archive containing the given PATHS.
|
|
PREFIX must be a store path that is a prefix of any store paths in PATHS.
|
|
REPOSITORY is a descriptive name that will show up in \"REPOSITORY\" column of
|
|
the output of \"docker images\".
|
|
|
|
When DATABASE is true, copy it to /var/guix/db in the image and create
|
|
/var/guix/gcroots and friends.
|
|
|
|
When ENTRY-POINT is true, it must be a list of strings; it is stored as the
|
|
entry point in the Docker image JSON structure.
|
|
|
|
ENVIRONMENT must be a list of name/value pairs. It specifies the environment
|
|
variables that must be defined in the resulting image.
|
|
|
|
EXTRA-FILES must be a list of directives for 'evaluate-populate-directive'
|
|
describing non-store files that must be created in the image.
|
|
|
|
TRANSFORMATIONS must be a list of (OLD -> NEW) tuples describing how to
|
|
transform the PATHS. Any path in PATHS that begins with OLD will be rewritten
|
|
in the Docker image so that it begins with NEW instead. If a path is a
|
|
non-empty directory, then its contents will be recursively added, as well.
|
|
|
|
SYSTEM is a GNU triplet (or prefix thereof) of the system the binaries in
|
|
PATHS are for; it is used to produce metadata in the image. Use COMPRESSOR, a
|
|
command such as '(\"gzip\" \"-9n\"), to compress IMAGE. Use CREATION-TIME, a
|
|
SRFI-19 time-utc object, as the creation time in metadata.
|
|
|
|
When MAX-LAYERS is not false build layered image, providing a Docker
|
|
image with store paths splitted in their own layers to improve sharing
|
|
between images.
|
|
|
|
ROOT-SYSTEM is a directory with a provisioned root file system, which will be
|
|
added to image as a layer."
|
|
(define (sanitize path-fragment)
|
|
(escape-special-chars
|
|
;; GNU tar strips the leading slash off of absolute paths before applying
|
|
;; the transformations, so we need to do the same, or else our
|
|
;; replacements won't match any paths.
|
|
(string-trim path-fragment #\/)
|
|
;; Escape the basic regexp special characters (see: "(sed) BRE syntax").
|
|
;; We also need to escape "/" because we use it as a delimiter.
|
|
"/*.^$[]\\"
|
|
#\\))
|
|
(define transformation->replacement
|
|
(match-lambda
|
|
((old '-> new)
|
|
;; See "(tar) transform" for details on the expression syntax.
|
|
(string-append "s/^" (sanitize old) "/" (sanitize new) "/"))))
|
|
(define (transformations->expression transformations)
|
|
(let ((replacements (map transformation->replacement transformations)))
|
|
(string-append
|
|
;; Avoid transforming link targets, since that would break some links
|
|
;; (e.g., symlinks that point to an absolute store path).
|
|
"flags=rSH;"
|
|
(string-join replacements ";")
|
|
;; Some paths might still have a leading path delimiter even after tar
|
|
;; transforms them (e.g., "/a/b" might be transformed into "/b"), so
|
|
;; strip any leading path delimiters that remain.
|
|
";s,^//*,,")))
|
|
(define transformation-options
|
|
(if (eq? '() transformations)
|
|
'()
|
|
`("--transform" ,(transformations->expression transformations))))
|
|
(define (seal-layer)
|
|
;; Add 'layer.tar' to 'image.tar' under the right name. Return its hash.
|
|
(let* ((file-hash (layer-diff-id "layer.tar"))
|
|
(file-name (string-append file-hash "/layer.tar")))
|
|
(mkdir file-hash)
|
|
(rename-file "layer.tar" file-name)
|
|
(invoke "tar" "-rf" "image.tar" file-name)
|
|
(delete-file file-name)
|
|
file-hash))
|
|
(define layers-hashes
|
|
;; Generate a tarball that includes container image layers as tarballs,
|
|
;; along with a manifest.json file describing the layer and config file
|
|
;; locations.
|
|
(match-lambda
|
|
(((head ...) (tail ...) id)
|
|
(create-empty-tar "image.tar")
|
|
(let* ((head-layers
|
|
(map
|
|
(lambda (file)
|
|
(invoke "tar" "cf" "layer.tar" file)
|
|
(seal-layer))
|
|
head))
|
|
(tail-layer
|
|
(begin
|
|
(create-empty-tar "layer.tar")
|
|
(for-each (lambda (file)
|
|
(invoke "tar" "-rf" "layer.tar" file))
|
|
tail)
|
|
(let* ((file-hash (layer-diff-id "layer.tar"))
|
|
(file-name (string-append file-hash "/layer.tar")))
|
|
(mkdir file-hash)
|
|
(rename-file "layer.tar" file-name)
|
|
(invoke "tar" "-rf" "image.tar" file-name)
|
|
(delete-file file-name)
|
|
file-hash)))
|
|
(customization-layer
|
|
(let* ((file-id (string-append id "/layer.tar"))
|
|
(file-hash (layer-diff-id file-id))
|
|
(file-name (string-append file-hash "/layer.tar")))
|
|
(mkdir file-hash)
|
|
(rename-file file-id file-name)
|
|
(invoke "tar" "-rf" "image.tar" file-name)
|
|
file-hash))
|
|
(all-layers
|
|
(append head-layers (list tail-layer customization-layer))))
|
|
(with-output-to-file "manifest.json"
|
|
(lambda ()
|
|
(scm->json (manifest prefix
|
|
(map (cut string-append <> "/layer.tar")
|
|
all-layers)
|
|
repository))))
|
|
(invoke "tar" "-rf" "image.tar" "manifest.json")
|
|
all-layers))))
|
|
(let* ((directory "/tmp/docker-image") ;temporary working directory
|
|
(id (docker-id prefix))
|
|
(time (date->string (time-utc->date creation-time) "~4"))
|
|
(arch (let-syntax ((cond* (syntax-rules ()
|
|
((_ (pattern clause) ...)
|
|
(cond ((string-prefix? pattern system)
|
|
clause)
|
|
...
|
|
(else
|
|
(error "unsupported system"
|
|
system)))))))
|
|
(cond* ("x86_64" "amd64")
|
|
("i686" "386")
|
|
("arm" "arm")
|
|
("aarch64" "arm64")
|
|
("mips64" "mips64le")))))
|
|
;; Make sure we start with a fresh, empty working directory.
|
|
(mkdir directory)
|
|
(with-directory-excursion directory
|
|
(mkdir id)
|
|
(with-directory-excursion id
|
|
(with-output-to-file "VERSION"
|
|
(lambda () (display schema-version)))
|
|
(with-output-to-file "json"
|
|
(lambda () (scm->json (image-description id time))))
|
|
|
|
(if root-system
|
|
(let ((directory (getcwd)))
|
|
(with-directory-excursion root-system
|
|
(apply invoke "tar"
|
|
"-cf" (string-append directory "/layer.tar")
|
|
`(,@transformation-options
|
|
,@(tar-base-options)
|
|
,@(scandir "."
|
|
(lambda (file)
|
|
(not (member file '("." "..")))))))))
|
|
(begin
|
|
;; Create a directory for the non-store files that need to go
|
|
;; into the archive.
|
|
(mkdir "extra")
|
|
|
|
(with-directory-excursion "extra"
|
|
;; Create non-store files.
|
|
(for-each (cut evaluate-populate-directive <> "./")
|
|
extra-files)
|
|
|
|
(when database
|
|
;; Initialize /var/guix, assuming PREFIX points to a
|
|
;; profile.
|
|
(install-database-and-gc-roots "." database prefix))
|
|
|
|
(apply invoke "tar" "-cf" "../layer.tar"
|
|
`(,@transformation-options
|
|
,@(tar-base-options)
|
|
,@(if max-layers '() paths)
|
|
,@(scandir "."
|
|
(lambda (file)
|
|
(not (member file '("." ".."))))))))
|
|
(delete-file-recursively "extra")))
|
|
|
|
;; It is possible for "/" to show up in the archive, especially when
|
|
;; applying transformations. For example, the transformation
|
|
;; "s,^/a,," will (perhaps surprisingly) cause GNU tar to transform
|
|
;; the path "/a" into "/". The presence of "/" in the archive is
|
|
;; probably benign, but it is definitely safe to remove it, so let's
|
|
;; do that. This fails when "/" is not in the archive, so use system*
|
|
;; instead of invoke to avoid an exception in that case, and redirect
|
|
;; stderr to the bit bucket to avoid "Exiting with failure status"
|
|
;; error messages.
|
|
(with-error-to-port (%make-void-port "w")
|
|
(lambda ()
|
|
(system* "tar" "--delete" "/" "-f" "layer.tar"))))
|
|
|
|
(with-output-to-file "config.json"
|
|
(lambda ()
|
|
(scm->json
|
|
(config (if max-layers
|
|
(layers-hashes
|
|
(append (size-sorted-store-items paths max-layers)
|
|
(list id)))
|
|
(list (layer-diff-id (string-append id "/layer.tar"))))
|
|
time arch
|
|
#:environment environment
|
|
#:entry-point entry-point))))
|
|
(if max-layers
|
|
(begin
|
|
(invoke "tar" "-rf" "image.tar" "config.json")
|
|
(if compressor
|
|
(begin
|
|
(apply invoke `(,@compressor "image.tar"))
|
|
(copy-file "image.tar.gz" image))
|
|
(copy-file "image.tar" image)))
|
|
(begin
|
|
(with-output-to-file "manifest.json"
|
|
(lambda ()
|
|
(scm->json (manifest prefix
|
|
(list (string-append id "/layer.tar"))
|
|
repository))))
|
|
(with-output-to-file "repositories"
|
|
(lambda ()
|
|
(scm->json (repositories prefix id repository))))
|
|
(apply invoke "tar" "-cf" image
|
|
`(,@(tar-base-options #:compressor compressor)
|
|
".")))))
|
|
(delete-file-recursively directory)))
|