diff --git a/cmd/manager/main.go b/cmd/manager/main.go index edda9a85..c7b905b4 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -24,12 +24,17 @@ import ( "net/url" "os" "path/filepath" + "strings" "time" "github.com/containers/image/v5/types" "github.com/go-logr/logr" "github.com/spf13/pflag" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + k8slabels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + k8stypes "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/metadata" @@ -37,7 +42,9 @@ import ( "k8s.io/klog/v2" "k8s.io/klog/v2/textlogger" ctrl "sigs.k8s.io/controller-runtime" + crcache "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/certwatcher" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/metrics" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" @@ -61,8 +68,8 @@ var ( ) const ( - storageDir = "catalogs" - authFilePath = "/etc/catalogd/auth.json" + storageDir = "catalogs" + authFile = "catalogd-global-pull-secret.json" ) func init() { @@ -88,6 +95,7 @@ func main() { keyFile string webhookPort int caCertDir string + globalPullSecret string ) flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") @@ -105,6 +113,7 @@ func main() { flag.StringVar(&keyFile, "tls-key", "", "The key file used for serving catalog contents over HTTPS. Requires tls-cert.") flag.IntVar(&webhookPort, "webhook-server-port", 9443, "The port that the mutating webhook server serves at.") flag.StringVar(&caCertDir, "ca-certs-dir", "", "The directory of CA certificate to use for verifying HTTPS connections to image registries.") + flag.StringVar(&globalPullSecret, "global-pull-secret", "", "The / of the global pull secret that is going to be used to pull bundle images.") klog.InitFlags(flag.CommandLine) @@ -120,6 +129,17 @@ func main() { ctrl.SetLogger(textlogger.NewLogger(textlogger.NewConfig())) + authFilePath := filepath.Join(os.TempDir(), authFile) + var globalPullSecretKey *k8stypes.NamespacedName + if globalPullSecret != "" { + secretParts := strings.Split(globalPullSecret, "/") + if len(secretParts) != 2 { + setupLog.Error(fmt.Errorf("incorrect number of components"), "value of global-pull-secret should be of the format /") + os.Exit(1) + } + globalPullSecretKey = &k8stypes.NamespacedName{Name: secretParts[1], Namespace: secretParts[0]} + } + if (certFile != "" && keyFile == "") || (certFile == "" && keyFile != "") { setupLog.Error(nil, "unable to configure TLS certificates: tls-cert and tls-key flags must be used together") os.Exit(1) @@ -148,6 +168,22 @@ func main() { }, }) + cacheOptions := crcache.Options{ + ByObject: map[client.Object]crcache.ByObject{}, + } + if globalPullSecretKey != nil { + cacheOptions.ByObject[&corev1.Secret{}] = crcache.ByObject{ + Namespaces: map[string]crcache.Config{ + globalPullSecretKey.Namespace: { + LabelSelector: k8slabels.Everything(), + FieldSelector: fields.SelectorFromSet(map[string]string{ + "metadata.name": globalPullSecretKey.Name, + }), + }, + }, + } + } + // Create manager mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Scheme: scheme, @@ -159,6 +195,7 @@ func main() { LeaderElection: enableLeaderElection, LeaderElectionID: "catalogd-operator-lock", WebhookServer: webhookServer, + Cache: cacheOptions, }) if err != nil { setupLog.Error(err, "unable to create manager") @@ -188,10 +225,20 @@ func main() { } unpacker := &source.ContainersImageRegistry{ BaseCachePath: unpackCacheBasePath, - SourceContext: &types.SystemContext{ - OCICertPath: caCertDir, - DockerCertPath: caCertDir, - AuthFilePath: authFilePathIfPresent(setupLog), + SourceContextFunc: func(logger logr.Logger) (*types.SystemContext, error) { + srcContext := &types.SystemContext{ + DockerCertPath: caCertDir, + OCICertPath: caCertDir, + } + if _, err := os.Stat(authFilePath); err == nil { + logger.Info("using available authentication information for pulling image") + srcContext.AuthFilePath = authFilePath + } else if os.IsNotExist(err) { + logger.Info("no authentication information found for pulling image, proceeding without auth") + } else { + return nil, fmt.Errorf("could not stat auth file, error: %w", err) + } + return srcContext, nil }, } @@ -241,6 +288,19 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "ClusterCatalog") os.Exit(1) } + + if globalPullSecretKey != nil { + setupLog.Info("creating SecretSyncer controller for watching secret", "Secret", globalPullSecret) + err := (&corecontrollers.SecretSyncerReconciler{ + Client: mgr.GetClient(), + AuthFilePath: authFilePath, + SecretKey: *globalPullSecretKey, + }).SetupWithManager(mgr) + if err != nil { + setupLog.Error(err, "unable to create controller", "controller", "SecretSyncer") + os.Exit(1) + } + } //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { @@ -290,17 +350,3 @@ func podNamespace() string { } return string(namespace) } - -func authFilePathIfPresent(logger logr.Logger) string { - _, err := os.Stat(authFilePath) - if os.IsNotExist(err) { - logger.Info("auth file not found, skipping configuration of global auth file", "path", authFilePath) - return "" - } - if err != nil { - logger.Error(err, "unable to access auth file path", "path", authFilePath) - os.Exit(1) - } - logger.Info("auth file found, configuring globally for image registry interactions", "path", authFilePath) - return authFilePath -} diff --git a/internal/controllers/core/secretsyncer_controller.go b/internal/controllers/core/secretsyncer_controller.go new file mode 100644 index 00000000..b59de69d --- /dev/null +++ b/internal/controllers/core/secretsyncer_controller.go @@ -0,0 +1,111 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package core + +import ( + "context" + "fmt" + "os" + + "github.com/go-logr/logr" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// SecretSyncerReconciler reconciles a specific secret object +type SecretSyncerReconciler struct { + client.Client + SecretKey types.NamespacedName + AuthFilePath string +} + +func (r *SecretSyncerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := log.FromContext(ctx) + if req.Name != r.SecretKey.Name || req.Namespace != r.SecretKey.Namespace { + logger.Error(fmt.Errorf("received unexpected request for Secret %v/%v", req.Namespace, req.Name), "reconciliation error") + return ctrl.Result{}, nil + } + + secret := &corev1.Secret{} + err := r.Get(ctx, req.NamespacedName, secret) + if err != nil { + if apierrors.IsNotFound(err) { + logger.Info("secret not found") + return r.deleteSecretFile(logger) + } + logger.Error(err, "failed to get Secret") + return ctrl.Result{}, err + } + + return r.writeSecretToFile(logger, secret) +} + +// SetupWithManager sets up the controller with the Manager. +func (r *SecretSyncerReconciler) SetupWithManager(mgr ctrl.Manager) error { + _, err := ctrl.NewControllerManagedBy(mgr). + For(&corev1.Secret{}). + WithEventFilter(newSecretPredicate(r.SecretKey)). + Build(r) + + return err +} + +func newSecretPredicate(key types.NamespacedName) predicate.Predicate { + return predicate.NewPredicateFuncs(func(obj client.Object) bool { + return obj.GetName() == key.Name && obj.GetNamespace() == key.Namespace + }) +} + +// writeSecretToFile writes the secret data to the specified file +func (r *SecretSyncerReconciler) writeSecretToFile(logger logr.Logger, secret *corev1.Secret) (ctrl.Result, error) { + // image registry secrets are always stored with the key .dockerconfigjson + // ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#registry-secret-existing-credentials + dockerConfigJSON, ok := secret.Data[".dockerconfigjson"] + if !ok { + logger.Error(fmt.Errorf("expected secret.Data key not found"), "expected secret Data to contain key .dockerconfigjson") + return ctrl.Result{}, nil + } + // expected format for auth.json + // https://github.com/containers/image/blob/main/docs/containers-auth.json.5.md + err := os.WriteFile(r.AuthFilePath, dockerConfigJSON, 0600) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to write secret data to file: %w", err) + } + logger.Info("saved global pull secret data locally") + return ctrl.Result{}, nil +} + +// deleteSecretFile deletes the auth file if the secret is deleted +func (r *SecretSyncerReconciler) deleteSecretFile(logger logr.Logger) (ctrl.Result, error) { + logger.Info("deleting local auth file", "file", r.AuthFilePath) + if err := os.Remove(r.AuthFilePath); err != nil { + if os.IsNotExist(err) { + logger.Info("auth file does not exist, nothing to delete") + return ctrl.Result{}, nil + } + return ctrl.Result{}, fmt.Errorf("failed to delete secret file: %w", err) + } + logger.Info("auth file deleted successfully") + return ctrl.Result{}, nil +} diff --git a/internal/controllers/core/secretsyncer_controller_test.go b/internal/controllers/core/secretsyncer_controller_test.go new file mode 100644 index 00000000..d653196b --- /dev/null +++ b/internal/controllers/core/secretsyncer_controller_test.go @@ -0,0 +1,96 @@ +package core + +import ( + "context" + "os" + "path/filepath" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/stretchr/testify/require" +) + +func TestSecretSyncerReconciler(t *testing.T) { + secretData := []byte(`{"auths":{"exampleRegistry": "exampledata"}}`) + authFileName := "test-auth.json" + for _, tt := range []struct { + name string + secret *corev1.Secret + addSecret bool + wantErr string + fileShouldExistBefore bool + fileShouldExistAfter bool + }{ + { + name: "secret exists, content gets saved to authFile", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret", + Namespace: "test-secret-namespace", + }, + Data: map[string][]byte{ + ".dockerconfigjson": secretData, + }, + }, + addSecret: true, + fileShouldExistBefore: false, + fileShouldExistAfter: true, + }, + { + name: "secret does not exist, file exists previously, file should get deleted", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-secret", + Namespace: "test-secret-namespace", + }, + Data: map[string][]byte{ + ".dockerconfigjson": secretData, + }, + }, + addSecret: false, + fileShouldExistBefore: true, + fileShouldExistAfter: false, + }, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + tempAuthFile := filepath.Join(t.TempDir(), authFileName) + clientBuilder := fake.NewClientBuilder() + if tt.addSecret { + clientBuilder = clientBuilder.WithObjects(tt.secret) + } + cl := clientBuilder.Build() + + secretKey := types.NamespacedName{Namespace: tt.secret.Namespace, Name: tt.secret.Name} + r := &SecretSyncerReconciler{ + Client: cl, + SecretKey: secretKey, + AuthFilePath: tempAuthFile, + } + if tt.fileShouldExistBefore { + err := os.WriteFile(tempAuthFile, secretData, 0600) + require.NoError(t, err) + } + res, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: secretKey}) + if tt.wantErr == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, tt.wantErr) + } + require.Equal(t, ctrl.Result{}, res) + + if tt.fileShouldExistAfter { + _, err := os.Stat(tempAuthFile) + require.NoError(t, err) + } else { + _, err := os.Stat(tempAuthFile) + require.True(t, os.IsNotExist(err)) + } + }) + } +} diff --git a/internal/source/containers_image.go b/internal/source/containers_image.go index a95c0a2b..2aedbb9a 100644 --- a/internal/source/containers_image.go +++ b/internal/source/containers_image.go @@ -34,8 +34,8 @@ import ( const ConfigDirLabel = "operators.operatorframework.io.index.configs.v1" type ContainersImageRegistry struct { - BaseCachePath string - SourceContext *types.SystemContext + BaseCachePath string + SourceContextFunc func(logger logr.Logger) (*types.SystemContext, error) } func (i *ContainersImageRegistry) Unpack(ctx context.Context, catalog *catalogdv1alpha1.ClusterCatalog) (*Result, error) { @@ -49,12 +49,16 @@ func (i *ContainersImageRegistry) Unpack(ctx context.Context, catalog *catalogdv return nil, reconcile.TerminalError(fmt.Errorf("error parsing catalog, catalog %s has a nil image source", catalog.Name)) } + srcCtx, err := i.SourceContextFunc(l) + if err != nil { + return nil, err + } ////////////////////////////////////////////////////// // // Resolve a canonical reference for the image. // ////////////////////////////////////////////////////// - imgRef, canonicalRef, specIsCanonical, err := resolveReferences(ctx, catalog.Spec.Source.Image.Ref, i.SourceContext) + imgRef, canonicalRef, specIsCanonical, err := resolveReferences(ctx, catalog.Spec.Source.Image.Ref, srcCtx) if err != nil { return nil, err } @@ -110,7 +114,7 @@ func (i *ContainersImageRegistry) Unpack(ctx context.Context, catalog *catalogdv // a policy context for the image pull. // ////////////////////////////////////////////////////// - policyContext, err := loadPolicyContext(i.SourceContext, l) + policyContext, err := loadPolicyContext(srcCtx, l) if err != nil { return nil, fmt.Errorf("error loading policy context: %w", err) } @@ -126,7 +130,7 @@ func (i *ContainersImageRegistry) Unpack(ctx context.Context, catalog *catalogdv // ////////////////////////////////////////////////////// if _, err := copy.Image(ctx, policyContext, layoutRef, dockerRef, ©.Options{ - SourceCtx: i.SourceContext, + SourceCtx: srcCtx, }); err != nil { return nil, fmt.Errorf("error copying image: %w", err) } @@ -137,7 +141,7 @@ func (i *ContainersImageRegistry) Unpack(ctx context.Context, catalog *catalogdv // Mount the image we just pulled // ////////////////////////////////////////////////////// - if err := i.unpackImage(ctx, unpackPath, layoutRef, specIsCanonical); err != nil { + if err := i.unpackImage(ctx, unpackPath, layoutRef, specIsCanonical, srcCtx); err != nil { if cleanupErr := deleteRecursive(unpackPath); cleanupErr != nil { err = errors.Join(err, cleanupErr) } @@ -243,8 +247,8 @@ func loadPolicyContext(sourceContext *types.SystemContext, l logr.Logger) (*sign return signature.NewPolicyContext(policy) } -func (i *ContainersImageRegistry) unpackImage(ctx context.Context, unpackPath string, imageReference types.ImageReference, specIsCanonical bool) error { - img, err := imageReference.NewImage(ctx, i.SourceContext) +func (i *ContainersImageRegistry) unpackImage(ctx context.Context, unpackPath string, imageReference types.ImageReference, specIsCanonical bool, sourceContext *types.SystemContext) error { + img, err := imageReference.NewImage(ctx, sourceContext) if err != nil { return fmt.Errorf("error reading image: %w", err) } @@ -254,7 +258,7 @@ func (i *ContainersImageRegistry) unpackImage(ctx context.Context, unpackPath st } }() - layoutSrc, err := imageReference.NewImageSource(ctx, i.SourceContext) + layoutSrc, err := imageReference.NewImageSource(ctx, sourceContext) if err != nil { return fmt.Errorf("error creating image source: %w", err) } diff --git a/internal/source/containers_image_test.go b/internal/source/containers_image_test.go index d35c2c90..526d534c 100644 --- a/internal/source/containers_image_test.go +++ b/internal/source/containers_image_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/containers/image/v5/types" + "github.com/go-logr/logr" "github.com/go-logr/logr/funcr" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/registry" @@ -313,9 +314,11 @@ func TestImageRegistry(t *testing.T) { testCache := t.TempDir() imgReg := &source.ContainersImageRegistry{ BaseCachePath: testCache, - SourceContext: &types.SystemContext{ - OCIInsecureSkipTLSVerify: true, - DockerInsecureSkipTLSVerify: types.OptionalBoolTrue, + SourceContextFunc: func(logger logr.Logger) (*types.SystemContext, error) { + return &types.SystemContext{ + OCIInsecureSkipTLSVerify: true, + DockerInsecureSkipTLSVerify: types.OptionalBoolTrue, + }, nil }, } @@ -431,6 +434,9 @@ func TestImageRegistryMissingLabelConsistentFailure(t *testing.T) { testCache := t.TempDir() imgReg := &source.ContainersImageRegistry{ BaseCachePath: testCache, + SourceContextFunc: func(logger logr.Logger) (*types.SystemContext, error) { + return &types.SystemContext{}, nil + }, } // Start a new server running an image registry