|
|
|
// Copyright 2018 Google LLC All Rights Reserved.
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package tarball
|
|
|
|
|
|
|
|
import (
|
|
|
|
"archive/tar"
|
|
|
|
"bytes"
|
|
|
|
"encoding/json"
|
|
|
|
"errors"
|
|
|
|
"fmt"
|
|
|
|
"io"
|
|
|
|
"os"
|
|
|
|
"sort"
|
|
|
|
"strings"
|
|
|
|
|
|
|
|
"github.com/google/go-containerregistry/pkg/name"
|
|
|
|
v1 "github.com/google/go-containerregistry/pkg/v1"
|
|
|
|
"github.com/google/go-containerregistry/pkg/v1/partial"
|
|
|
|
)
|
|
|
|
|
|
|
|
// WriteToFile writes in the compressed format to a tarball, on disk.
|
|
|
|
// This is just syntactic sugar wrapping tarball.Write with a new file.
|
|
|
|
func WriteToFile(p string, ref name.Reference, img v1.Image, opts ...WriteOption) error {
|
|
|
|
w, err := os.Create(p)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer w.Close()
|
|
|
|
|
|
|
|
return Write(ref, img, w, opts...)
|
|
|
|
}
|
|
|
|
|
|
|
|
// MultiWriteToFile writes in the compressed format to a tarball, on disk.
|
|
|
|
// This is just syntactic sugar wrapping tarball.MultiWrite with a new file.
|
|
|
|
func MultiWriteToFile(p string, tagToImage map[name.Tag]v1.Image, opts ...WriteOption) error {
|
|
|
|
refToImage := make(map[name.Reference]v1.Image, len(tagToImage))
|
|
|
|
for i, d := range tagToImage {
|
|
|
|
refToImage[i] = d
|
|
|
|
}
|
|
|
|
return MultiRefWriteToFile(p, refToImage, opts...)
|
|
|
|
}
|
|
|
|
|
|
|
|
// MultiRefWriteToFile writes in the compressed format to a tarball, on disk.
|
|
|
|
// This is just syntactic sugar wrapping tarball.MultiRefWrite with a new file.
|
|
|
|
func MultiRefWriteToFile(p string, refToImage map[name.Reference]v1.Image, opts ...WriteOption) error {
|
|
|
|
w, err := os.Create(p)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
defer w.Close()
|
|
|
|
|
|
|
|
return MultiRefWrite(refToImage, w, opts...)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Write is a wrapper to write a single image and tag to a tarball.
|
|
|
|
func Write(ref name.Reference, img v1.Image, w io.Writer, opts ...WriteOption) error {
|
|
|
|
return MultiRefWrite(map[name.Reference]v1.Image{ref: img}, w, opts...)
|
|
|
|
}
|
|
|
|
|
|
|
|
// MultiWrite writes the contents of each image to the provided writer, in the compressed format.
|
|
|
|
// The contents are written in the following format:
|
|
|
|
// One manifest.json file at the top level containing information about several images.
|
|
|
|
// One file for each layer, named after the layer's SHA.
|
|
|
|
// One file for the config blob, named after its SHA.
|
|
|
|
func MultiWrite(tagToImage map[name.Tag]v1.Image, w io.Writer, opts ...WriteOption) error {
|
|
|
|
refToImage := make(map[name.Reference]v1.Image, len(tagToImage))
|
|
|
|
for i, d := range tagToImage {
|
|
|
|
refToImage[i] = d
|
|
|
|
}
|
|
|
|
return MultiRefWrite(refToImage, w, opts...)
|
|
|
|
}
|
|
|
|
|
|
|
|
// MultiRefWrite writes the contents of each image to the provided writer, in the compressed format.
|
|
|
|
// The contents are written in the following format:
|
|
|
|
// One manifest.json file at the top level containing information about several images.
|
|
|
|
// One file for each layer, named after the layer's SHA.
|
|
|
|
// One file for the config blob, named after its SHA.
|
|
|
|
func MultiRefWrite(refToImage map[name.Reference]v1.Image, w io.Writer, opts ...WriteOption) error {
|
|
|
|
// process options
|
|
|
|
o := &writeOptions{
|
|
|
|
updates: nil,
|
|
|
|
}
|
|
|
|
for _, option := range opts {
|
|
|
|
if err := option(o); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
imageToTags := dedupRefToImage(refToImage)
|
|
|
|
size, mBytes, err := getSizeAndManifest(imageToTags)
|
|
|
|
if err != nil {
|
|
|
|
return sendUpdateReturn(o, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
return writeImagesToTar(imageToTags, mBytes, size, w, o)
|
|
|
|
}
|
|
|
|
|
|
|
|
// sendUpdateReturn return the passed in error message, also sending on update channel, if it exists
|
|
|
|
func sendUpdateReturn(o *writeOptions, err error) error {
|
|
|
|
if o != nil && o.updates != nil {
|
|
|
|
o.updates <- v1.Update{
|
|
|
|
Error: err,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// sendProgressWriterReturn return the passed in error message, also sending on update channel, if it exists, along with downloaded information
|
|
|
|
func sendProgressWriterReturn(pw *progressWriter, err error) error {
|
|
|
|
if pw != nil {
|
|
|
|
return pw.Error(err)
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// writeImagesToTar writes the images to the tarball
|
|
|
|
func writeImagesToTar(imageToTags map[v1.Image][]string, m []byte, size int64, w io.Writer, o *writeOptions) (err error) {
|
|
|
|
if w == nil {
|
|
|
|
return sendUpdateReturn(o, errors.New("must pass valid writer"))
|
|
|
|
}
|
|
|
|
|
|
|
|
tw := w
|
|
|
|
var pw *progressWriter
|
|
|
|
|
|
|
|
// we only calculate the sizes and use a progressWriter if we were provided
|
|
|
|
// an option with a progress channel
|
|
|
|
if o != nil && o.updates != nil {
|
|
|
|
pw = &progressWriter{
|
|
|
|
w: w,
|
|
|
|
updates: o.updates,
|
|
|
|
size: size,
|
|
|
|
}
|
|
|
|
tw = pw
|
|
|
|
}
|
|
|
|
|
|
|
|
tf := tar.NewWriter(tw)
|
|
|
|
defer tf.Close()
|
|
|
|
|
|
|
|
seenLayerDigests := make(map[string]struct{})
|
|
|
|
|
|
|
|
for img := range imageToTags {
|
|
|
|
// Write the config.
|
|
|
|
cfgName, err := img.ConfigName()
|
|
|
|
if err != nil {
|
|
|
|
return sendProgressWriterReturn(pw, err)
|
|
|
|
}
|
|
|
|
cfgBlob, err := img.RawConfigFile()
|
|
|
|
if err != nil {
|
|
|
|
return sendProgressWriterReturn(pw, err)
|
|
|
|
}
|
|
|
|
if err := writeTarEntry(tf, cfgName.String(), bytes.NewReader(cfgBlob), int64(len(cfgBlob))); err != nil {
|
|
|
|
return sendProgressWriterReturn(pw, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Write the layers.
|
|
|
|
layers, err := img.Layers()
|
|
|
|
if err != nil {
|
|
|
|
return sendProgressWriterReturn(pw, err)
|
|
|
|
}
|
|
|
|
layerFiles := make([]string, len(layers))
|
|
|
|
for i, l := range layers {
|
|
|
|
d, err := l.Digest()
|
|
|
|
if err != nil {
|
|
|
|
return sendProgressWriterReturn(pw, err)
|
|
|
|
}
|
|
|
|
// Munge the file name to appease ancient technology.
|
|
|
|
//
|
|
|
|
// tar assumes anything with a colon is a remote tape drive:
|
|
|
|
// https://www.gnu.org/software/tar/manual/html_section/tar_45.html
|
|
|
|
// Drop the algorithm prefix, e.g. "sha256:"
|
|
|
|
hex := d.Hex
|
|
|
|
|
|
|
|
// gunzip expects certain file extensions:
|
|
|
|
// https://www.gnu.org/software/gzip/manual/html_node/Overview.html
|
|
|
|
layerFiles[i] = fmt.Sprintf("%s.tar.gz", hex)
|
|
|
|
|
|
|
|
if _, ok := seenLayerDigests[hex]; ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
seenLayerDigests[hex] = struct{}{}
|
|
|
|
|
|
|
|
r, err := l.Compressed()
|
|
|
|
if err != nil {
|
|
|
|
return sendProgressWriterReturn(pw, err)
|
|
|
|
}
|
|
|
|
blobSize, err := l.Size()
|
|
|
|
if err != nil {
|
|
|
|
return sendProgressWriterReturn(pw, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := writeTarEntry(tf, layerFiles[i], r, blobSize); err != nil {
|
|
|
|
return sendProgressWriterReturn(pw, err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if err := writeTarEntry(tf, "manifest.json", bytes.NewReader(m), int64(len(m))); err != nil {
|
|
|
|
return sendProgressWriterReturn(pw, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// be sure to close the tar writer so everything is flushed out before we send our EOF
|
|
|
|
if err := tf.Close(); err != nil {
|
|
|
|
return sendProgressWriterReturn(pw, err)
|
|
|
|
}
|
|
|
|
// send an EOF to indicate finished on the channel, but nil as our return error
|
|
|
|
_ = sendProgressWriterReturn(pw, io.EOF)
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// calculateManifest calculates the manifest and optionally the size of the tar file
|
|
|
|
func calculateManifest(imageToTags map[v1.Image][]string) (m Manifest, err error) {
|
|
|
|
if len(imageToTags) == 0 {
|
|
|
|
return nil, errors.New("set of images is empty")
|
|
|
|
}
|
|
|
|
|
|
|
|
for img, tags := range imageToTags {
|
|
|
|
cfgName, err := img.ConfigName()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Store foreign layer info.
|
|
|
|
layerSources := make(map[v1.Hash]v1.Descriptor)
|
|
|
|
|
|
|
|
// Write the layers.
|
|
|
|
layers, err := img.Layers()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
layerFiles := make([]string, len(layers))
|
|
|
|
for i, l := range layers {
|
|
|
|
d, err := l.Digest()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
// Munge the file name to appease ancient technology.
|
|
|
|
//
|
|
|
|
// tar assumes anything with a colon is a remote tape drive:
|
|
|
|
// https://www.gnu.org/software/tar/manual/html_section/tar_45.html
|
|
|
|
// Drop the algorithm prefix, e.g. "sha256:"
|
|
|
|
hex := d.Hex
|
|
|
|
|
|
|
|
// gunzip expects certain file extensions:
|
|
|
|
// https://www.gnu.org/software/gzip/manual/html_node/Overview.html
|
|
|
|
layerFiles[i] = fmt.Sprintf("%s.tar.gz", hex)
|
|
|
|
|
|
|
|
// Add to LayerSources if it's a foreign layer.
|
|
|
|
desc, err := partial.BlobDescriptor(img, d)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
if !desc.MediaType.IsDistributable() {
|
|
|
|
diffid, err := partial.BlobToDiffID(img, d)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
layerSources[diffid] = *desc
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Generate the tar descriptor and write it.
|
|
|
|
m = append(m, Descriptor{
|
|
|
|
Config: cfgName.String(),
|
|
|
|
RepoTags: tags,
|
|
|
|
Layers: layerFiles,
|
|
|
|
LayerSources: layerSources,
|
|
|
|
})
|
|
|
|
}
|
|
|
|
// sort by name of the repotags so it is consistent. Alternatively, we could sort by hash of the
|
|
|
|
// descriptor, but that would make it hard for humans to process
|
|
|
|
sort.Slice(m, func(i, j int) bool {
|
|
|
|
return strings.Join(m[i].RepoTags, ",") < strings.Join(m[j].RepoTags, ",")
|
|
|
|
})
|
|
|
|
|
|
|
|
return m, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// CalculateSize calculates the expected complete size of the output tar file
|
|
|
|
func CalculateSize(refToImage map[name.Reference]v1.Image) (size int64, err error) {
|
|
|
|
imageToTags := dedupRefToImage(refToImage)
|
|
|
|
size, _, err = getSizeAndManifest(imageToTags)
|
|
|
|
return size, err
|
|
|
|
}
|
|
|
|
|
|
|
|
func getSizeAndManifest(imageToTags map[v1.Image][]string) (int64, []byte, error) {
|
|
|
|
m, err := calculateManifest(imageToTags)
|
|
|
|
if err != nil {
|
|
|
|
return 0, nil, fmt.Errorf("unable to calculate manifest: %w", err)
|
|
|
|
}
|
|
|
|
mBytes, err := json.Marshal(m)
|
|
|
|
if err != nil {
|
|
|
|
return 0, nil, fmt.Errorf("could not marshall manifest to bytes: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
size, err := calculateTarballSize(imageToTags, mBytes)
|
|
|
|
if err != nil {
|
|
|
|
return 0, nil, fmt.Errorf("error calculating tarball size: %w", err)
|
|
|
|
}
|
|
|
|
return size, mBytes, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// calculateTarballSize calculates the size of the tar file
|
|
|
|
func calculateTarballSize(imageToTags map[v1.Image][]string, mBytes []byte) (size int64, err error) {
|
|
|
|
seenLayerDigests := make(map[string]struct{})
|
|
|
|
for img, name := range imageToTags {
|
|
|
|
manifest, err := img.Manifest()
|
|
|
|
if err != nil {
|
|
|
|
return size, fmt.Errorf("unable to get manifest for img %s: %w", name, err)
|
|
|
|
}
|
|
|
|
size += calculateSingleFileInTarSize(manifest.Config.Size)
|
|
|
|
for _, l := range manifest.Layers {
|
|
|
|
hex := l.Digest.Hex
|
|
|
|
if _, ok := seenLayerDigests[hex]; ok {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
seenLayerDigests[hex] = struct{}{}
|
|
|
|
size += calculateSingleFileInTarSize(l.Size)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// add the manifest
|
|
|
|
size += calculateSingleFileInTarSize(int64(len(mBytes)))
|
|
|
|
|
|
|
|
// add the two padding blocks that indicate end of a tar file
|
|
|
|
size += 1024
|
|
|
|
return size, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func dedupRefToImage(refToImage map[name.Reference]v1.Image) map[v1.Image][]string {
|
|
|
|
imageToTags := make(map[v1.Image][]string)
|
|
|
|
|
|
|
|
for ref, img := range refToImage {
|
|
|
|
if tag, ok := ref.(name.Tag); ok {
|
|
|
|
if tags, ok := imageToTags[img]; !ok || tags == nil {
|
|
|
|
imageToTags[img] = []string{}
|
|
|
|
}
|
|
|
|
// Docker cannot load tarballs without an explicit tag:
|
|
|
|
// https://github.com/google/go-containerregistry/issues/890
|
|
|
|
//
|
|
|
|
// We can't use the fully qualified tag.Name() because of rules_docker:
|
|
|
|
// https://github.com/google/go-containerregistry/issues/527
|
|
|
|
//
|
|
|
|
// If the tag is "latest", but tag.String() doesn't end in ":latest",
|
|
|
|
// just append it. Kind of gross, but should work for now.
|
|
|
|
ts := tag.String()
|
|
|
|
if tag.Identifier() == name.DefaultTag && !strings.HasSuffix(ts, ":"+name.DefaultTag) {
|
|
|
|
ts = fmt.Sprintf("%s:%s", ts, name.DefaultTag)
|
|
|
|
}
|
|
|
|
imageToTags[img] = append(imageToTags[img], ts)
|
|
|
|
} else if _, ok := imageToTags[img]; !ok {
|
|
|
|
imageToTags[img] = nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return imageToTags
|
|
|
|
}
|
|
|
|
|
|
|
|
// writeTarEntry writes a file to the provided writer with a corresponding tar header
|
|
|
|
func writeTarEntry(tf *tar.Writer, path string, r io.Reader, size int64) error {
|
|
|
|
hdr := &tar.Header{
|
|
|
|
Mode: 0644,
|
|
|
|
Typeflag: tar.TypeReg,
|
|
|
|
Size: size,
|
|
|
|
Name: path,
|
|
|
|
}
|
|
|
|
if err := tf.WriteHeader(hdr); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
_, err := io.Copy(tf, r)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// ComputeManifest get the manifest.json that will be written to the tarball
|
|
|
|
// for multiple references
|
|
|
|
func ComputeManifest(refToImage map[name.Reference]v1.Image) (Manifest, error) {
|
|
|
|
imageToTags := dedupRefToImage(refToImage)
|
|
|
|
return calculateManifest(imageToTags)
|
|
|
|
}
|
|
|
|
|
|
|
|
// WriteOption a function option to pass to Write()
|
|
|
|
type WriteOption func(*writeOptions) error
|
|
|
|
type writeOptions struct {
|
|
|
|
updates chan<- v1.Update
|
|
|
|
}
|
|
|
|
|
|
|
|
// WithProgress create a WriteOption for passing to Write() that enables
|
|
|
|
// a channel to receive updates as they are downloaded and written to disk.
|
|
|
|
func WithProgress(updates chan<- v1.Update) WriteOption {
|
|
|
|
return func(o *writeOptions) error {
|
|
|
|
o.updates = updates
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// progressWriter is a writer which will send the download progress
|
|
|
|
type progressWriter struct {
|
|
|
|
w io.Writer
|
|
|
|
updates chan<- v1.Update
|
|
|
|
size, complete int64
|
|
|
|
}
|
|
|
|
|
|
|
|
func (pw *progressWriter) Write(p []byte) (int, error) {
|
|
|
|
n, err := pw.w.Write(p)
|
|
|
|
if err != nil {
|
|
|
|
return n, err
|
|
|
|
}
|
|
|
|
|
|
|
|
pw.complete += int64(n)
|
|
|
|
|
|
|
|
pw.updates <- v1.Update{
|
|
|
|
Total: pw.size,
|
|
|
|
Complete: pw.complete,
|
|
|
|
}
|
|
|
|
|
|
|
|
return n, err
|
|
|
|
}
|
|
|
|
|
|
|
|
func (pw *progressWriter) Error(err error) error {
|
|
|
|
pw.updates <- v1.Update{
|
|
|
|
Total: pw.size,
|
|
|
|
Complete: pw.complete,
|
|
|
|
Error: err,
|
|
|
|
}
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
func (pw *progressWriter) Close() error {
|
|
|
|
pw.updates <- v1.Update{
|
|
|
|
Total: pw.size,
|
|
|
|
Complete: pw.complete,
|
|
|
|
Error: io.EOF,
|
|
|
|
}
|
|
|
|
return io.EOF
|
|
|
|
}
|
|
|
|
|
|
|
|
// calculateSingleFileInTarSize calculate the size a file will take up in a tar archive,
|
|
|
|
// given the input data. Provided by rounding up to nearest whole block (512)
|
|
|
|
// and adding header 512
|
|
|
|
func calculateSingleFileInTarSize(in int64) (out int64) {
|
|
|
|
// doing this manually, because math.Round() works with float64
|
|
|
|
out += in
|
|
|
|
if remainder := out % 512; remainder != 0 {
|
|
|
|
out += (512 - remainder)
|
|
|
|
}
|
|
|
|
out += 512
|
|
|
|
return out
|
|
|
|
}
|