Skip to content

Commit efe1e0b

Browse files
cuisongliucuisongliu
authored
feature(main): add container storage (#2)
Signed-off-by: cuisongliu <[email protected]>
1 parent 2888d8d commit efe1e0b

File tree

11 files changed

+187
-41
lines changed

11 files changed

+187
-41
lines changed

‎.github/workflows/go.yml

+10
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,13 @@ jobs:
3434
cp dist/sreg_linux_amd64_v1/sreg sreg
3535
chmod a+x sreg
3636
./sreg version
37+
- name: Upload linux-amd64
38+
uses: actions/upload-artifact@v3
39+
with:
40+
name: sreg-linux-amd64
41+
path: dist/sreg_linux_amd64_v1/sreg
42+
- name: Upload linux-arm64
43+
uses: actions/upload-artifact@v3
44+
with:
45+
name: sreg-linux-arm64
46+
path: dist/sreg_linux_arm64/sreg

‎.goreleaser.yml

+22-1
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,36 @@ builds:
77
goarch:
88
- amd64
99
- arm64
10+
flags:
11+
- -trimpath
1012
ldflags:
1113
- -s -w
1214
- -X github.com/labring/sreg/pkg/version.gitVersion={{.Version}}
1315
- -X github.com/labring/sreg/pkg/version.gitCommit={{.ShortCommit}}
1416
- -X github.com/labring/sreg/pkg/version.buildDate={{.Date}}
1517
tags:
1618
- containers_image_openpgp
19+
- netgo
1720
- exclude_graphdriver_devicemapper
18-
21+
- static
22+
- osusergo
23+
- exclude_graphdriver_btrfs
24+
overrides:
25+
- goos: linux
26+
goarch: amd64
27+
goamd64: v1
28+
goarm: ""
29+
gomips: ""
30+
env:
31+
- CGO_ENABLED=1
32+
- CC=x86_64-linux-gnu-gcc
33+
- goos: linux
34+
goarch: arm64
35+
goarm: ""
36+
gomips: ""
37+
env:
38+
- CGO_ENABLED=1
39+
- CC=aarch64-linux-gnu-gcc
1940
checksum:
2041
name_template: 'checksums.txt'
2142
snapshot:

‎cmd/root.go

+4
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"fmt"
1919
"github.com/labring/sreg/pkg/registry/commands"
2020
"github.com/labring/sreg/pkg/utils/logger"
21+
"github.com/sirupsen/logrus"
2122
"os"
2223

2324
"github.com/spf13/cobra"
@@ -50,6 +51,9 @@ func Execute() {
5051
func init() {
5152
cobra.OnInitialize(func() {
5253
logger.CfgConsoleLogger(debug, false)
54+
if debug {
55+
logrus.SetLevel(logrus.DebugLevel)
56+
}
5357
})
5458
rootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "enable debug logger")
5559

‎pkg/buildimage/tar_images.go

+3
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ func TarList(dir string) ([]string, error) {
3737
return nil, wrapGetImageErr(err, tarDir)
3838
}
3939
for i, image := range images {
40+
if image == "" {
41+
continue
42+
}
4043
parts := strings.SplitN(image, "@", 2)
4144
if len(parts) != 2 {
4245
return nil, fmt.Errorf("invalid image format: %s", image)

‎pkg/registry/commands/flags.go

+3
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,12 @@ func (opts *registrySaveResults) CheckAuth() (map[string]types.AuthConfig, error
5555
type registrySaveRawResults struct {
5656
*registrySaveResults
5757
images []string
58+
tars []string
5859
}
5960

6061
func (opts *registrySaveRawResults) RegisterFlags(fs *pflag.FlagSet) {
6162
opts.registrySaveResults.RegisterFlags(fs)
6263
fs.StringSliceVar(&opts.images, "images", []string{}, "images list")
64+
fs.StringSliceVar(&opts.tars, "tars", []string{}, "tar list, eg: --tars=docker-archive:/root/config_main.tar@library/config_main")
65+
6366
}

‎pkg/registry/commands/save.go

+33-23
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ import (
3333

3434
func NewRegistryImageSaveCmd(examplePrefix string) *cobra.Command {
3535
var auth map[string]types.AuthConfig
36-
var images []string
36+
var images, tars []string
3737
flagsResults := registrySaveRawResults{
3838
registrySaveResults: new(registrySaveResults),
3939
}
@@ -42,43 +42,53 @@ func NewRegistryImageSaveCmd(examplePrefix string) *cobra.Command {
4242
Short: "save images to local registry dir",
4343
Example: fmt.Sprintf(`
4444
%[1]s save --registry-dir=/tmp/registry .
45+
%[1]s save --registry-dir=/tmp/registry --images=containers-storage:docker.io/labring/coredns:v0.0.1
46+
%[1]s save --registry-dir=/tmp/registry --images=docker-daemon:docker.io/library/nginx:latest
47+
%[1]s save --registry-dir=/tmp/registry --tars=docker-archive:/root/config_main.tar@library/config_main
48+
%[1]s save --registry-dir=/tmp/registry --tars=oci-archive:/root/config_main.tar@library/config_main
4549
%[1]s save --registry-dir=/tmp/registry --images=docker.io/library/busybox:latest`, examplePrefix),
4650
RunE: func(cmd *cobra.Command, args []string) error {
47-
is := save.NewImageSaver(context.Background(), flagsResults.registryPullMaxPullProcs, auth)
48-
outImages, err := is.SaveImages(images, flagsResults.registryPullRegistryDir, v1.Platform{OS: "linux", Architecture: flagsResults.registryPullArch})
49-
if err != nil {
50-
return err
51+
if len(images) > 0 {
52+
is := save.NewImageSaver(context.Background(), flagsResults.registryPullMaxPullProcs, auth)
53+
outImages, err := is.SaveImages(images, flagsResults.registryPullRegistryDir, v1.Platform{OS: "linux", Architecture: flagsResults.registryPullArch})
54+
if err != nil {
55+
return err
56+
}
57+
logger.Info("images pulled: %+v", outImages)
5158
}
52-
logger.Info("images pulled: %+v", outImages)
53-
if args[0] != "" {
59+
60+
if len(tars) > 0 {
5461
tarIs := save.NewImageTarSaver(context.Background(), flagsResults.registryPullMaxPullProcs)
55-
tars, err := buildimage.TarList(args[0])
62+
outTars, err := tarIs.SaveImages(tars, flagsResults.registryPullRegistryDir, v1.Platform{OS: "linux", Architecture: flagsResults.registryPullArch})
5663
if err != nil {
5764
return err
5865
}
59-
if len(tars) != 0 {
60-
outTars, err := tarIs.SaveImages(tars, flagsResults.registryPullRegistryDir, v1.Platform{OS: "linux", Architecture: flagsResults.registryPullArch})
61-
if err != nil {
62-
return err
63-
}
64-
logger.Info("images tar saved: %+v", outTars)
65-
}
66+
logger.Info("images tar saved: %+v", outTars)
6667
}
67-
6868
return nil
6969
},
7070
PreRunE: func(cmd *cobra.Command, args []string) error {
71-
if len(args) == 0 && len(flagsResults.images) == 0 {
72-
return errors.New("'--images' and args cannot be empty at the same time")
71+
if len(args) == 0 && len(flagsResults.images) == 0 && len(flagsResults.tars) == 0 {
72+
return errors.New("'--images' '--tars' and args cannot be empty at the same time")
7373
}
7474
var err error
75-
if len(flagsResults.images) > 0 {
76-
images = flagsResults.images
75+
if len(args) == 0 {
76+
if len(flagsResults.images) > 0 {
77+
images = flagsResults.images
78+
}
79+
if len(flagsResults.tars) > 0 {
80+
tars = flagsResults.tars
81+
}
7782
} else {
7883
images, err = buildimage.List(args[0])
79-
}
80-
if err != nil {
81-
return err
84+
if err != nil {
85+
return err
86+
}
87+
tars, err = buildimage.TarList(args[0])
88+
if err != nil {
89+
return err
90+
}
91+
8292
}
8393
auth, err = flagsResults.CheckAuth()
8494
if err != nil {

‎pkg/registry/save/registry_save.go

+3-12
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ package save
1717
import (
1818
"context"
1919
"fmt"
20-
"github.com/containers/image/v5/transports/alltransports"
2120
"strings"
2221
stdsync "sync"
2322
"time"
@@ -75,17 +74,9 @@ func (is *tmpRegistryImage) SaveImages(images []string, dir string, platform v1.
7574
mu.Unlock()
7675
}()
7776
var srcRef itype.ImageReference
78-
if strings.HasPrefix(img, "docker-daemon") {
79-
logger.Info("Using containers-storage or docker-daemon as image transport")
80-
srcRef, err = alltransports.ParseImageName(img)
81-
if err != nil {
82-
return fmt.Errorf("invalid source name %s: %v", img, err)
83-
}
84-
} else {
85-
srcRef, err = sync.ImageNameToReference(sys, img, is.auths)
86-
if err != nil {
87-
return err
88-
}
77+
srcRef, err = sync.ImageNameToReference(sys, img, is.auths)
78+
if err != nil {
79+
return err
8980
}
9081
err = sync.RegistryToImage(is.ctx, sys, srcRef, ep, copy.CopySystemImage)
9182
if err != nil {

‎pkg/registry/save/registry_tars_save.go

+3
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,9 @@ func (is *tmpTarRegistryImage) SaveImages(images []string, dir string, platform
8787
<-numCh
8888
mu.Unlock()
8989
}()
90+
if strings.TrimSpace(img) == "" {
91+
return nil
92+
}
9093
allImage := strings.Split(img, "@")
9194
srcRef, err := alltransports.ParseImageName(allImage[0])
9295
if err != nil {

‎pkg/registry/sync/sync.go

+50-5
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,12 @@ func ToRegistry(ctx context.Context, opts *Options) error {
6464
if err != nil {
6565
return err
6666
}
67+
var allError error
68+
defer func() {
69+
if err := policyContext.Destroy(); err != nil {
70+
allError = fmt.Errorf("error tearing down policy context: %v", err)
71+
}
72+
}()
6773
repos, err := docker.SearchRegistry(ctx, sys, src, "", 1<<10)
6874
if err != nil {
6975
return err
@@ -115,7 +121,7 @@ func ToRegistry(ctx context.Context, opts *Options) error {
115121
}
116122
}
117123
}
118-
return nil
124+
return allError
119125
}
120126

121127
func getRetryOptions() *retry.RetryOptions {
@@ -131,6 +137,27 @@ func getRetryOptions() *retry.RetryOptions {
131137
}
132138

133139
func ImageNameToReference(sys *types.SystemContext, img string, auth map[string]dtype.AuthConfig) (types.ImageReference, error) {
140+
if err := reexecIfNecessaryForImages(img); err != nil {
141+
return nil, err
142+
}
143+
transport := alltransports.TransportFromImageName(img)
144+
if transport != nil && transport.Name() == "containers-storage" {
145+
logger.Info("Using containers-storage as image transport")
146+
srcRef, err := alltransports.ParseImageName(img)
147+
if err != nil {
148+
return nil, fmt.Errorf("invalid source name %s: %v", img, err)
149+
}
150+
return srcRef, nil
151+
}
152+
if transport != nil && transport.Name() == "docker-daemon" {
153+
logger.Info("Using docker-daemon as image transport")
154+
srcRef, err := alltransports.ParseImageName(img)
155+
if err != nil {
156+
return nil, fmt.Errorf("invalid source name %s: %v", img, err)
157+
}
158+
return srcRef, nil
159+
}
160+
134161
src, err := name.ParseReference(img)
135162
if err != nil {
136163
return nil, fmt.Errorf("ref invalid source name %s: %v", img, err)
@@ -174,15 +201,24 @@ func RegistryToImage(ctx context.Context, sys *types.SystemContext, src types.Im
174201
if err != nil {
175202
return err
176203
}
177-
return retry.RetryIfNecessary(ctx, func() error {
204+
var allError error
205+
defer func() {
206+
if err := policyContext.Destroy(); err != nil {
207+
allError = fmt.Errorf("error tearing down policy context: %v", err)
208+
}
209+
}()
210+
if err = retry.RetryIfNecessary(ctx, func() error {
178211
_, err = copy.Image(ctx, policyContext, destRef, src, &copy.Options{
179212
SourceCtx: sys,
180213
DestinationCtx: sys,
181214
ImageListSelection: selection,
182215
ReportWriter: os.Stdout,
183216
})
184217
return err
185-
}, getRetryOptions())
218+
}, getRetryOptions()); err != nil {
219+
return err
220+
}
221+
return allError
186222
}
187223

188224
func ArchiveToImage(ctx context.Context, sys *types.SystemContext, src types.ImageReference, dst string, selection copy.ImageListSelection) error {
@@ -195,15 +231,24 @@ func ArchiveToImage(ctx context.Context, sys *types.SystemContext, src types.Ima
195231
if err != nil {
196232
return err
197233
}
198-
return retry.RetryIfNecessary(ctx, func() error {
234+
var allError error
235+
defer func() {
236+
if err = policyContext.Destroy(); err != nil {
237+
allError = fmt.Errorf("error tearing down policy context: %v", err)
238+
}
239+
}()
240+
if err = retry.RetryIfNecessary(ctx, func() error {
199241
_, err = copy.Image(ctx, policyContext, destRef, src, &copy.Options{
200242
SourceCtx: sys,
201243
DestinationCtx: sys,
202244
ImageListSelection: selection,
203245
ReportWriter: os.Stdout,
204246
})
205247
return err
206-
}, getRetryOptions())
248+
}, getRetryOptions()); err != nil {
249+
return err
250+
}
251+
return allError
207252
}
208253

209254
func getPolicyContext() (*signature.PolicyContext, error) {

‎pkg/registry/sync/unshare.go

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
//go:build !linux
2+
// +build !linux
3+
4+
package sync
5+
6+
func reexecIfNecessaryForImages(inputImageNames ...string) error {
7+
return nil
8+
}

‎pkg/registry/sync/unshare_linux.go

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
package sync
2+
3+
import (
4+
"github.com/containers/image/v5/transports/alltransports"
5+
"github.com/containers/storage/pkg/unshare"
6+
"github.com/pkg/errors"
7+
"github.com/syndtr/gocapability/capability"
8+
)
9+
10+
var neededCapabilities = []capability.Cap{
11+
capability.CAP_CHOWN,
12+
capability.CAP_DAC_OVERRIDE,
13+
capability.CAP_FOWNER,
14+
capability.CAP_FSETID,
15+
capability.CAP_MKNOD,
16+
capability.CAP_SETFCAP,
17+
}
18+
19+
func maybeReexec() error {
20+
// With Skopeo we need only the subset of the root capabilities necessary
21+
// for pulling an image to the storage. Do not attempt to create a namespace
22+
// if we already have the capabilities we need.
23+
capabilities, err := capability.NewPid(0)
24+
if err != nil {
25+
return errors.Wrapf(err, "error reading the current capabilities sets")
26+
}
27+
for _, cap := range neededCapabilities {
28+
if !capabilities.Get(capability.EFFECTIVE, cap) {
29+
// We miss a capability we need, create a user namespaces
30+
unshare.MaybeReexecUsingUserNamespace(true)
31+
return nil
32+
}
33+
}
34+
return nil
35+
}
36+
37+
func reexecIfNecessaryForImages(imageNames ...string) error {
38+
// Check if container-storage is used before doing unshare
39+
for _, imageName := range imageNames {
40+
transport := alltransports.TransportFromImageName(imageName)
41+
// Hard-code the storage name to avoid a reference on c/image/storage.
42+
// See https://github.com/containers/skopeo/issues/771#issuecomment-563125006.
43+
if transport != nil && transport.Name() == "containers-storage" {
44+
return maybeReexec()
45+
}
46+
}
47+
return nil
48+
}

0 commit comments

Comments
 (0)