mirror of https://github.com/k3d-io/k3d
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
482 lines
14 KiB
482 lines
14 KiB
// Copyright 2018 Google LLC All Rights Reserved.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package layout
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
|
|
"github.com/google/go-containerregistry/pkg/logs"
|
|
v1 "github.com/google/go-containerregistry/pkg/v1"
|
|
"github.com/google/go-containerregistry/pkg/v1/match"
|
|
"github.com/google/go-containerregistry/pkg/v1/mutate"
|
|
"github.com/google/go-containerregistry/pkg/v1/partial"
|
|
"github.com/google/go-containerregistry/pkg/v1/stream"
|
|
"github.com/google/go-containerregistry/pkg/v1/types"
|
|
"golang.org/x/sync/errgroup"
|
|
)
|
|
|
|
var layoutFile = `{
|
|
"imageLayoutVersion": "1.0.0"
|
|
}`
|
|
|
|
// AppendImage writes a v1.Image to the Path and updates
|
|
// the index.json to reference it.
|
|
func (l Path) AppendImage(img v1.Image, options ...Option) error {
|
|
if err := l.WriteImage(img); err != nil {
|
|
return err
|
|
}
|
|
|
|
desc, err := partial.Descriptor(img)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
o := makeOptions(options...)
|
|
for _, opt := range o.descOpts {
|
|
opt(desc)
|
|
}
|
|
|
|
return l.AppendDescriptor(*desc)
|
|
}
|
|
|
|
// AppendIndex writes a v1.ImageIndex to the Path and updates
|
|
// the index.json to reference it.
|
|
func (l Path) AppendIndex(ii v1.ImageIndex, options ...Option) error {
|
|
if err := l.WriteIndex(ii); err != nil {
|
|
return err
|
|
}
|
|
|
|
desc, err := partial.Descriptor(ii)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
o := makeOptions(options...)
|
|
for _, opt := range o.descOpts {
|
|
opt(desc)
|
|
}
|
|
|
|
return l.AppendDescriptor(*desc)
|
|
}
|
|
|
|
// AppendDescriptor adds a descriptor to the index.json of the Path.
|
|
func (l Path) AppendDescriptor(desc v1.Descriptor) error {
|
|
ii, err := l.ImageIndex()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
index, err := ii.IndexManifest()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
index.Manifests = append(index.Manifests, desc)
|
|
|
|
rawIndex, err := json.MarshalIndent(index, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return l.WriteFile("index.json", rawIndex, os.ModePerm)
|
|
}
|
|
|
|
// ReplaceImage writes a v1.Image to the Path and updates
|
|
// the index.json to reference it, replacing any existing one that matches matcher, if found.
|
|
func (l Path) ReplaceImage(img v1.Image, matcher match.Matcher, options ...Option) error {
|
|
if err := l.WriteImage(img); err != nil {
|
|
return err
|
|
}
|
|
|
|
return l.replaceDescriptor(img, matcher, options...)
|
|
}
|
|
|
|
// ReplaceIndex writes a v1.ImageIndex to the Path and updates
|
|
// the index.json to reference it, replacing any existing one that matches matcher, if found.
|
|
func (l Path) ReplaceIndex(ii v1.ImageIndex, matcher match.Matcher, options ...Option) error {
|
|
if err := l.WriteIndex(ii); err != nil {
|
|
return err
|
|
}
|
|
|
|
return l.replaceDescriptor(ii, matcher, options...)
|
|
}
|
|
|
|
// replaceDescriptor adds a descriptor to the index.json of the Path, replacing
|
|
// any one matching matcher, if found.
|
|
func (l Path) replaceDescriptor(append mutate.Appendable, matcher match.Matcher, options ...Option) error {
|
|
ii, err := l.ImageIndex()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
desc, err := partial.Descriptor(append)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
o := makeOptions(options...)
|
|
for _, opt := range o.descOpts {
|
|
opt(desc)
|
|
}
|
|
|
|
add := mutate.IndexAddendum{
|
|
Add: append,
|
|
Descriptor: *desc,
|
|
}
|
|
ii = mutate.AppendManifests(mutate.RemoveManifests(ii, matcher), add)
|
|
|
|
index, err := ii.IndexManifest()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
rawIndex, err := json.MarshalIndent(index, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return l.WriteFile("index.json", rawIndex, os.ModePerm)
|
|
}
|
|
|
|
// RemoveDescriptors removes any descriptors that match the match.Matcher from the index.json of the Path.
|
|
func (l Path) RemoveDescriptors(matcher match.Matcher) error {
|
|
ii, err := l.ImageIndex()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
ii = mutate.RemoveManifests(ii, matcher)
|
|
|
|
index, err := ii.IndexManifest()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
rawIndex, err := json.MarshalIndent(index, "", " ")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return l.WriteFile("index.json", rawIndex, os.ModePerm)
|
|
}
|
|
|
|
// WriteFile write a file with arbitrary data at an arbitrary location in a v1
|
|
// layout. Used mostly internally to write files like "oci-layout" and
|
|
// "index.json", also can be used to write other arbitrary files. Do *not* use
|
|
// this to write blobs. Use only WriteBlob() for that.
|
|
func (l Path) WriteFile(name string, data []byte, perm os.FileMode) error {
|
|
if err := os.MkdirAll(l.path(), os.ModePerm); err != nil && !os.IsExist(err) {
|
|
return err
|
|
}
|
|
|
|
return os.WriteFile(l.path(name), data, perm)
|
|
}
|
|
|
|
// WriteBlob copies a file to the blobs/ directory in the Path from the given ReadCloser at
|
|
// blobs/{hash.Algorithm}/{hash.Hex}.
|
|
func (l Path) WriteBlob(hash v1.Hash, r io.ReadCloser) error {
|
|
return l.writeBlob(hash, -1, r, nil)
|
|
}
|
|
|
|
func (l Path) writeBlob(hash v1.Hash, size int64, rc io.ReadCloser, renamer func() (v1.Hash, error)) error {
|
|
defer rc.Close()
|
|
if hash.Hex == "" && renamer == nil {
|
|
panic("writeBlob called an invalid hash and no renamer")
|
|
}
|
|
|
|
dir := l.path("blobs", hash.Algorithm)
|
|
if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) {
|
|
return err
|
|
}
|
|
|
|
// Check if blob already exists and is the correct size
|
|
file := filepath.Join(dir, hash.Hex)
|
|
if s, err := os.Stat(file); err == nil && !s.IsDir() && (s.Size() == size || size == -1) {
|
|
return nil
|
|
}
|
|
|
|
// If a renamer func was provided write to a temporary file
|
|
open := func() (*os.File, error) { return os.Create(file) }
|
|
if renamer != nil {
|
|
open = func() (*os.File, error) { return os.CreateTemp(dir, hash.Hex) }
|
|
}
|
|
w, err := open()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if renamer != nil {
|
|
// Delete temp file if an error is encountered before renaming
|
|
defer func() {
|
|
if err := os.Remove(w.Name()); err != nil && !errors.Is(err, os.ErrNotExist) {
|
|
logs.Warn.Printf("error removing temporary file after encountering an error while writing blob: %v", err)
|
|
}
|
|
}()
|
|
}
|
|
defer w.Close()
|
|
|
|
// Write to file and exit if not renaming
|
|
if n, err := io.Copy(w, rc); err != nil || renamer == nil {
|
|
return err
|
|
} else if size != -1 && n != size {
|
|
return fmt.Errorf("expected blob size %d, but only wrote %d", size, n)
|
|
}
|
|
|
|
// Always close reader before renaming, since Close computes the digest in
|
|
// the case of streaming layers. If Close is not called explicitly, it will
|
|
// occur in a goroutine that is not guaranteed to succeed before renamer is
|
|
// called. When renamer is the layer's Digest method, it can return
|
|
// ErrNotComputed.
|
|
if err := rc.Close(); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Always close file before renaming
|
|
if err := w.Close(); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Rename file based on the final hash
|
|
finalHash, err := renamer()
|
|
if err != nil {
|
|
return fmt.Errorf("error getting final digest of layer: %w", err)
|
|
}
|
|
|
|
renamePath := l.path("blobs", finalHash.Algorithm, finalHash.Hex)
|
|
return os.Rename(w.Name(), renamePath)
|
|
}
|
|
|
|
// writeLayer writes the compressed layer to a blob. Unlike WriteBlob it will
|
|
// write to a temporary file (suffixed with .tmp) within the layout until the
|
|
// compressed reader is fully consumed and written to disk. Also unlike
|
|
// WriteBlob, it will not skip writing and exit without error when a blob file
|
|
// exists, but does not have the correct size. (The blob hash is not
|
|
// considered, because it may be expensive to compute.)
|
|
func (l Path) writeLayer(layer v1.Layer) error {
|
|
d, err := layer.Digest()
|
|
if errors.Is(err, stream.ErrNotComputed) {
|
|
// Allow digest errors, since streams may not have calculated the hash
|
|
// yet. Instead, use an empty value, which will be transformed into a
|
|
// random file name with `os.CreateTemp` and the final digest will be
|
|
// calculated after writing to a temp file and before renaming to the
|
|
// final path.
|
|
d = v1.Hash{Algorithm: "sha256", Hex: ""}
|
|
} else if err != nil {
|
|
return err
|
|
}
|
|
|
|
s, err := layer.Size()
|
|
if errors.Is(err, stream.ErrNotComputed) {
|
|
// Allow size errors, since streams may not have calculated the size
|
|
// yet. Instead, use zero as a sentinel value meaning that no size
|
|
// comparison can be done and any sized blob file should be considered
|
|
// valid and not overwritten.
|
|
//
|
|
// TODO: Provide an option to always overwrite blobs.
|
|
s = -1
|
|
} else if err != nil {
|
|
return err
|
|
}
|
|
|
|
r, err := layer.Compressed()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := l.writeBlob(d, s, r, layer.Digest); err != nil {
|
|
return fmt.Errorf("error writing layer: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// RemoveBlob removes a file from the blobs directory in the Path
|
|
// at blobs/{hash.Algorithm}/{hash.Hex}
|
|
// It does *not* remove any reference to it from other manifests or indexes, or
|
|
// from the root index.json.
|
|
func (l Path) RemoveBlob(hash v1.Hash) error {
|
|
dir := l.path("blobs", hash.Algorithm)
|
|
err := os.Remove(filepath.Join(dir, hash.Hex))
|
|
if err != nil && !os.IsNotExist(err) {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// WriteImage writes an image, including its manifest, config and all of its
|
|
// layers, to the blobs directory. If any blob already exists, as determined by
|
|
// the hash filename, does not write it.
|
|
// This function does *not* update the `index.json` file. If you want to write the
|
|
// image and also update the `index.json`, call AppendImage(), which wraps this
|
|
// and also updates the `index.json`.
|
|
func (l Path) WriteImage(img v1.Image) error {
|
|
layers, err := img.Layers()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Write the layers concurrently.
|
|
var g errgroup.Group
|
|
for _, layer := range layers {
|
|
layer := layer
|
|
g.Go(func() error {
|
|
return l.writeLayer(layer)
|
|
})
|
|
}
|
|
if err := g.Wait(); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Write the config.
|
|
cfgName, err := img.ConfigName()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
cfgBlob, err := img.RawConfigFile()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := l.WriteBlob(cfgName, io.NopCloser(bytes.NewReader(cfgBlob))); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Write the img manifest.
|
|
d, err := img.Digest()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
manifest, err := img.RawManifest()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return l.WriteBlob(d, io.NopCloser(bytes.NewReader(manifest)))
|
|
}
|
|
|
|
type withLayer interface {
|
|
Layer(v1.Hash) (v1.Layer, error)
|
|
}
|
|
|
|
type withBlob interface {
|
|
Blob(v1.Hash) (io.ReadCloser, error)
|
|
}
|
|
|
|
func (l Path) writeIndexToFile(indexFile string, ii v1.ImageIndex) error {
|
|
index, err := ii.IndexManifest()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Walk the descriptors and write any v1.Image or v1.ImageIndex that we find.
|
|
// If we come across something we don't expect, just write it as a blob.
|
|
for _, desc := range index.Manifests {
|
|
switch desc.MediaType {
|
|
case types.OCIImageIndex, types.DockerManifestList:
|
|
ii, err := ii.ImageIndex(desc.Digest)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := l.WriteIndex(ii); err != nil {
|
|
return err
|
|
}
|
|
case types.OCIManifestSchema1, types.DockerManifestSchema2:
|
|
img, err := ii.Image(desc.Digest)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := l.WriteImage(img); err != nil {
|
|
return err
|
|
}
|
|
default:
|
|
// TODO: The layout could reference arbitrary things, which we should
|
|
// probably just pass through.
|
|
|
|
var blob io.ReadCloser
|
|
// Workaround for #819.
|
|
if wl, ok := ii.(withLayer); ok {
|
|
layer, lerr := wl.Layer(desc.Digest)
|
|
if lerr != nil {
|
|
return lerr
|
|
}
|
|
blob, err = layer.Compressed()
|
|
} else if wb, ok := ii.(withBlob); ok {
|
|
blob, err = wb.Blob(desc.Digest)
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := l.WriteBlob(desc.Digest, blob); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
rawIndex, err := ii.RawManifest()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return l.WriteFile(indexFile, rawIndex, os.ModePerm)
|
|
}
|
|
|
|
// WriteIndex writes an index to the blobs directory. Walks down the children,
|
|
// including its children manifests and/or indexes, and down the tree until all of
|
|
// config and all layers, have been written. If any blob already exists, as determined by
|
|
// the hash filename, does not write it.
|
|
// This function does *not* update the `index.json` file. If you want to write the
|
|
// index and also update the `index.json`, call AppendIndex(), which wraps this
|
|
// and also updates the `index.json`.
|
|
func (l Path) WriteIndex(ii v1.ImageIndex) error {
|
|
// Always just write oci-layout file, since it's small.
|
|
if err := l.WriteFile("oci-layout", []byte(layoutFile), os.ModePerm); err != nil {
|
|
return err
|
|
}
|
|
|
|
h, err := ii.Digest()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
indexFile := filepath.Join("blobs", h.Algorithm, h.Hex)
|
|
return l.writeIndexToFile(indexFile, ii)
|
|
}
|
|
|
|
// Write constructs a Path at path from an ImageIndex.
|
|
//
|
|
// The contents are written in the following format:
|
|
// At the top level, there is:
|
|
//
|
|
// One oci-layout file containing the version of this image-layout.
|
|
// One index.json file listing descriptors for the contained images.
|
|
//
|
|
// Under blobs/, there is, for each image:
|
|
//
|
|
// One file for each layer, named after the layer's SHA.
|
|
// One file for each config blob, named after its SHA.
|
|
// One file for each manifest blob, named after its SHA.
|
|
func Write(path string, ii v1.ImageIndex) (Path, error) {
|
|
lp := Path(path)
|
|
// Always just write oci-layout file, since it's small.
|
|
if err := lp.WriteFile("oci-layout", []byte(layoutFile), os.ModePerm); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
// TODO create blobs/ in case there is a blobs file which would prevent the directory from being created
|
|
|
|
return lp, lp.writeIndexToFile("index.json", ii)
|
|
}
|
|
|