"k8s.io/apimachinery/pkg/util/version" "k8s.io/klog/v2" <ins> "k8s.io/kubernetes/pkg/kubelet/userns/inuserns" </ins> utilkernel "k8s.io/kubernetes/pkg/util/kernel" "k8s.io/mount-utils" )


return false } <ins> if inuserns.RunningInUserNS() { // Turning off swap in unprivileged tmpfs mounts unsupported // https://github.com/torvalds/linux/blob/v6.8/mm/shmem.c#L4004-L4011 // https://github.com/kubernetes/kubernetes/issues/125137 klog.InfoS("Running under a user namespace - tmpfs noswap is not supported") return false } </ins> kernelVersion, err := utilkernel.GetVersion() if err != nil { klog.ErrorS(err, "cannot determine kernel version, unable to determine is tmpfs noswap is supported")


} return parameters, nil default: <ins> sort.Slice(objs, func(i, j int) bool { obj1, obj2 := objs[i].(*resourcev1alpha2.ResourceClassParameters), objs[j].(*resourcev1alpha2.ResourceClassParameters) if obj1 == nil || obj2 == nil { return false } return obj1.Name < obj2.Name }) </ins> return nil, statusError(logger, fmt.Errorf("multiple generated class parameters for %s.%s %s found: %s", class.ParametersRef.Kind, class.ParametersRef.APIGroup, klog.KRef(class.Namespace, class.ParametersRef.Name), klog.KObjSlice(objs))) } }


} return parameters, nil default: <ins> sort.Slice(objs, func(i, j int) bool { obj1, obj2 := objs[i].(*resourcev1alpha2.ResourceClaimParameters), objs[j].(*resourcev1alpha2.ResourceClaimParameters) if obj1 == nil || obj2 == nil { return false } return obj1.Name < obj2.Name }) </ins> return nil, statusError(logger, fmt.Errorf("multiple generated claim parameters for %s.%s %s found: %s", claim.Spec.ParametersRef.Kind, claim.Spec.ParametersRef.APIGroup, klog.KRef(claim.Namespace, claim.Spec.ParametersRef.Name), klog.KObjSlice(objs))) } }


if sysruntime.GOOS == "linux" { // AppArmor is a Linux kernel security module and it does not support other operating systems. klet.appArmorValidator = apparmor.NewValidator() <del> klet.softAdmitHandlers.AddPodAdmitHandler(lifecycle.NewAppArmorAdmitHandler(klet.appArmorValidator)) </del> <ins> klet.admitHandlers.AddPodAdmitHandler(lifecycle.NewAppArmorAdmitHandler(klet.appArmorValidator)) </ins> } leaseDuration := time.Duration(kubeCfg.NodeLeaseDurationSeconds) * time.Second


// the list of handlers to call during pod admission. admitHandlers lifecycle.PodAdmitHandlers <del> // softAdmithandlers are applied to the pod after it is admitted by the Kubelet, but before it is // run. A pod rejected by a softAdmitHandler will be left in a Pending state indefinitely. If a // rejected pod should not be recreated, or the scheduler is not aware of the rejection rule, the // admission rule should be applied by a softAdmitHandler. softAdmitHandlers lifecycle.PodAdmitHandlers </del> // the list of handlers to call during pod sync loop. lifecycle.PodSyncLoopHandlers


return isTerminal, nil } <del> // If the pod should not be running, we request the pod's containers be stopped. This is not the same // as termination (we want to stop the pod, but potentially restart it later if soft admission allows // it later). Set the status and phase appropriately runnable := kl.canRunPod(pod) if !runnable.Admit { // Pod is not runnable; and update the Pod and Container statuses to why. if apiPodStatus.Phase != v1.PodFailed && apiPodStatus.Phase != v1.PodSucceeded { apiPodStatus.Phase = v1.PodPending } apiPodStatus.Reason = runnable.Reason apiPodStatus.Message = runnable.Message // Waiting containers are not creating. const waitingReason = "Blocked" for _, cs := range apiPodStatus.InitContainerStatuses { if cs.State.Waiting != nil { cs.State.Waiting.Reason = waitingReason } } for _, cs := range apiPodStatus.ContainerStatuses { if cs.State.Waiting != nil { cs.State.Waiting.Reason = waitingReason } } } </del> // Record the time it takes for the pod to become running // since kubelet first saw the pod if firstSeenTime is set. existingStatus, ok := kl.statusManager.GetPodStatus(pod.UID)


kl.statusManager.SetPodStatus(pod, apiPodStatus) <del> // Pods that are not runnable must be stopped - return a typed error to the pod worker if !runnable.Admit { klog.V(2).InfoS("Pod is not runnable and must have running containers stopped", "pod", klog.KObj(pod), "podUID", pod.UID, "message", runnable.Message) var syncErr error p := kubecontainer.ConvertPodStatusToRunningPod(kl.getRuntime().Type(), podStatus) if err := kl.killPod(ctx, pod, p, nil); err != nil { if !wait.Interrupted(err) { kl.recorder.Eventf(pod, v1.EventTypeWarning, events.FailedToKillPod, "error killing pod: %v", err) syncErr = fmt.Errorf("error killing pod: %w", err) utilruntime.HandleError(syncErr) } } else { // There was no error killing the pod, but the pod cannot be run. // Return an error to signal that the sync loop should back off. syncErr = fmt.Errorf("pod cannot be run: %v", runnable.Message) } return false, syncErr } </del> // If the network plugin is not ready, only start the pod if it uses the host network if err := kl.runtimeState.networkErrors(); err != nil && !kubecontainer.IsHostNetworkPod(pod) { kl.recorder.Eventf(pod, v1.EventTypeWarning, events.NetworkNotReady, "%s: %v", NetworkNotReadyErrorMsg, err)


return true, "", "" } <del> func (kl *Kubelet) canRunPod(pod *v1.Pod) lifecycle.PodAdmitResult { attrs := &lifecycle.PodAdmitAttributes{Pod: pod} // Get "OtherPods". Rejected pods are failed, so only include admitted pods that are alive. attrs.OtherPods = kl.GetActivePods() for _, handler := range kl.softAdmitHandlers { if result := handler.Admit(attrs); !result.Admit { return result } } return lifecycle.PodAdmitResult{Admit: true} } </del> // syncLoop is the main loop for processing changes. It watches for changes from // three channels (file, apiserver, and http) and creates a union of them. For // any new change seen, will run a sync against desired state and running state. If


if err != nil { t.Fatalf("Couldn't create cacher: %v", err) } <ins> if utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { // The tests assume that Get/GetList/Watch calls shouldn't fail. // However, 429 error can now be returned if watchcache is under initialization. // To avoid rewriting all tests, we wait for watchcache to initialize. if err := cacher.Wait(context.Background()); err != nil { t.Fatal(err) } } </ins> d := destroyFunc s = cacher destroyFunc = func() {


return nil } <ins> // Wait blocks until the cacher is Ready or Stopped, it returns an error if Stopped. func (c *Cacher) Wait(ctx context.Context) error { return c.ready.wait(ctx) } </ins> // errWatcher implements watch.Interface to return a single error type errWatcher struct { result chan watch.Event


if utilfeature.DefaultFeatureGate.Enabled(features.ResilientWatchCacheInitialization) { // The tests assume that Get/GetList/Watch calls shouldn't fail. // However, 429 error can now be returned if watchcache is under initialization. <del> // To avoid rewriting all tests, we wait for watcache to initialize. if err := cacher.ready.wait(ctx); err != nil { </del> <ins> // To avoid rewriting all tests, we wait for watchcache to initialize. if err := cacher.Wait(ctx); err != nil { </ins> t.Fatal(err) } }


// The tests assume that Get/GetList/Watch calls shouldn't fail. // However, 429 error can now be returned if watchcache is under initialization. // To avoid rewriting all tests, we wait for watcache to initialize. <del> if err := cacher.ready.wait(context.Background()); err != nil { </del> <ins> if err := cacher.Wait(context.Background()); err != nil { </ins> return nil, storage.APIObjectVersioner{}, err } }


<del> /* Copyright 2022 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package plugin import ( "context" "errors" "fmt" "net" "sync" "time" "google.golang.org/grpc" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" utilversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/klog/v2" drapb "k8s.io/kubelet/pkg/apis/dra/v1alpha4" ) // NewDRAPluginClient returns a wrapper around those gRPC methods of a DRA // driver kubelet plugin which need to be called by kubelet. The wrapper // handles gRPC connection management and logging. Connections are reused // across different NewDRAPluginClient calls. func NewDRAPluginClient(pluginName string) (*Plugin, error) { if pluginName == "" { return nil, fmt.Errorf("plugin name is empty") } existingPlugin := draPlugins.get(pluginName) if existingPlugin == nil { return nil, fmt.Errorf("plugin name %s not found in the list of registered DRA plugins", pluginName) } return existingPlugin, nil } type Plugin struct { backgroundCtx context.Context cancel        func(cause error) mutex                   sync.Mutex conn                    *grpc.ClientConn endpoint                string highestSupportedVersion *utilversion.Version clientCallTimeout       time.Duration } func (p *Plugin) getOrCreateGRPCConn() (*grpc.ClientConn, error) { p.mutex.Lock() defer p.mutex.Unlock() if p.conn != nil { return p.conn, nil } ctx := p.backgroundCtx logger := klog.FromContext(ctx) network := "unix" logger.V(4).Info("Creating new gRPC connection", "protocol", network, "endpoint", p.endpoint) // grpc.Dial is deprecated. grpc.NewClient should be used instead. // For now this gets ignored because this function is meant to establish // the connection, with the one second timeout below. Perhaps that // approach should be reconsidered? //nolint:staticcheck conn, err := grpc.Dial( p.endpoint, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(func(ctx context.Context, target string) (net.Conn, error) { return (&net.Dialer{}).DialContext(ctx, network, target) }), ) if err != nil { return nil, err } ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() if ok := conn.WaitForStateChange(ctx, connectivity.Connecting); !ok { return nil, errors.New("timed out waiting for gRPC connection to be ready") } p.conn = conn return p.conn, nil } func (p *Plugin) NodePrepareResources( ctx context.Context, req *drapb.NodePrepareResourcesRequest, opts ...grpc.CallOption, ) (*drapb.NodePrepareResourcesResponse, error) { logger := klog.FromContext(ctx) logger.V(4).Info("Calling NodePrepareResources rpc", "request", req) conn, err := p.getOrCreateGRPCConn() if err != nil { return nil, err } ctx, cancel := context.WithTimeout(ctx, p.clientCallTimeout) defer cancel() nodeClient := drapb.NewNodeClient(conn) response, err := nodeClient.NodePrepareResources(ctx, req) logger.V(4).Info("Done calling NodePrepareResources rpc", "response", response, "err", err) return response, err } func (p *Plugin) NodeUnprepareResources( ctx context.Context, req *drapb.NodeUnprepareResourcesRequest, opts ...grpc.CallOption, ) (*drapb.NodeUnprepareResourcesResponse, error) { logger := klog.FromContext(ctx) logger.V(4).Info("Calling NodeUnprepareResource rpc", "request", req) conn, err := p.getOrCreateGRPCConn() if err != nil { return nil, err } ctx, cancel := context.WithTimeout(ctx, p.clientCallTimeout) defer cancel() nodeClient := drapb.NewNodeClient(conn) response, err := nodeClient.NodeUnprepareResources(ctx, req) logger.V(4).Info("Done calling NodeUnprepareResources rpc", "response", response, "err", err) return response, err } </del>


<del> /* Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package plugin import ( "context" "fmt" "net" "os" "path/filepath" "sync" "testing" "github.com/stretchr/testify/assert" "google.golang.org/grpc" drapb "k8s.io/kubelet/pkg/apis/dra/v1alpha4" "k8s.io/kubernetes/test/utils/ktesting" ) const ( v1alpha4Version = "v1alpha4" ) type fakeV1alpha4GRPCServer struct { drapb.UnimplementedNodeServer } var _ drapb.NodeServer = &fakeV1alpha4GRPCServer{} func (f *fakeV1alpha4GRPCServer) NodePrepareResources(ctx context.Context, in *drapb.NodePrepareResourcesRequest) (*drapb.NodePrepareResourcesResponse, error) { return &drapb.NodePrepareResourcesResponse{Claims: map[string]*drapb.NodePrepareResourceResponse{"claim-uid": { Devices: []*drapb.Device{ { RequestNames: []string{"test-request"}, CDIDeviceIDs: []string{"test-cdi-id"}, }, }, }}}, nil } func (f *fakeV1alpha4GRPCServer) NodeUnprepareResources(ctx context.Context, in *drapb.NodeUnprepareResourcesRequest) (*drapb.NodeUnprepareResourcesResponse, error) { return &drapb.NodeUnprepareResourcesResponse{}, nil } type tearDown func() func setupFakeGRPCServer(version string) (string, tearDown, error) { p, err := os.MkdirTemp("", "dra_plugin") if err != nil { return "", nil, err } closeCh := make(chan struct{}) addr := filepath.Join(p, "server.sock") teardown := func() { close(closeCh) os.RemoveAll(addr) } listener, err := net.Listen("unix", addr) if err != nil { teardown() return "", nil, err } s := grpc.NewServer() switch version { case v1alpha4Version: fakeGRPCServer := &fakeV1alpha4GRPCServer{} drapb.RegisterNodeServer(s, fakeGRPCServer) default: return "", nil, fmt.Errorf("unsupported version: %s", version) } go func() { go s.Serve(listener) <-closeCh s.GracefulStop() }() return addr, teardown, nil } func TestGRPCConnIsReused(t *testing.T) { tCtx := ktesting.Init(t) addr, teardown, err := setupFakeGRPCServer(v1alpha4Version) if err != nil { t.Fatal(err) } defer teardown() reusedConns := make(map[*grpc.ClientConn]int) wg := sync.WaitGroup{} m := sync.Mutex{} p := &Plugin{ backgroundCtx:     tCtx, endpoint:          addr, clientCallTimeout: defaultClientCallTimeout, } conn, err := p.getOrCreateGRPCConn() defer func() { err := conn.Close() if err != nil { t.Error(err) } }() if err != nil { t.Fatal(err) } // ensure the plugin we are using is registered draPlugins.add("dummy-plugin", p) defer draPlugins.delete("dummy-plugin") // we call `NodePrepareResource` 2 times and check whether a new connection is created or the same is reused for i := 0; i < 2; i++ { wg.Add(1) go func() { defer wg.Done() client, err := NewDRAPluginClient("dummy-plugin") if err != nil { t.Error(err) return } req := &drapb.NodePrepareResourcesRequest{ Claims: []*drapb.Claim{ { Namespace: "dummy-namespace", UID:       "dummy-uid", Name:      "dummy-claim", }, }, } _, err = client.NodePrepareResources(tCtx, req) assert.NoError(t, err) client.mutex.Lock() conn := client.conn client.mutex.Unlock() m.Lock() defer m.Unlock() reusedConns[conn]++ }() } wg.Wait() // We should have only one entry otherwise it means another gRPC connection has been created if len(reusedConns) != 1 { t.Errorf("expected length to be 1 but got %d", len(reusedConns)) } if counter, ok := reusedConns[conn]; ok && counter != 2 { t.Errorf("expected counter to be 2 but got %d", counter) } } func TestNewDRAPluginClient(t *testing.T) { for _, test := range []struct { description string setup       func(string) tearDown pluginName  string shouldError bool }{ { description: "plugin name is empty", setup: func(_ string) tearDown { return func() {} }, pluginName:  "", shouldError: true, }, { description: "plugin name not found in the list", setup: func(_ string) tearDown { return func() {} }, pluginName:  "plugin-name-not-found-in-the-list", shouldError: true, }, { description: "plugin exists", setup: func(name string) tearDown { draPlugins.add(name, &Plugin{}) return func() { draPlugins.delete(name) } }, pluginName: "dummy-plugin", }, } { t.Run(test.description, func(t *testing.T) { teardown := test.setup(test.pluginName) defer teardown() client, err := NewDRAPluginClient(test.pluginName) if test.shouldError { assert.Nil(t, client) assert.Error(t, err) } else { assert.NotNil(t, client) assert.Nil(t, err) } }) } } func TestNodeUnprepareResources(t *testing.T) { for _, test := range []struct { description   string serverSetup   func(string) (string, tearDown, error) serverVersion string request       *drapb.NodeUnprepareResourcesRequest }{ { description:   "server supports v1alpha4", serverSetup:   setupFakeGRPCServer, serverVersion: v1alpha4Version, request:       &drapb.NodeUnprepareResourcesRequest{}, }, } { t.Run(test.description, func(t *testing.T) { tCtx := ktesting.Init(t) addr, teardown, err := setupFakeGRPCServer(test.serverVersion) if err != nil { t.Fatal(err) } defer teardown() p := &Plugin{ backgroundCtx:     tCtx, endpoint:          addr, clientCallTimeout: defaultClientCallTimeout, } conn, err := p.getOrCreateGRPCConn() defer func() { err := conn.Close() if err != nil { t.Error(err) } }() if err != nil { t.Fatal(err) } draPlugins.add("dummy-plugin", p) defer draPlugins.delete("dummy-plugin") client, err := NewDRAPluginClient("dummy-plugin") if err != nil { t.Fatal(err) } _, err = client.NodeUnprepareResources(tCtx, test.request) if err != nil { t.Fatal(err) } }) } } </del>


"context" "errors" "fmt" <ins> "net" "sync" </ins> "time" <del> v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1alpha3" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" </del> <ins> "google.golang.org/grpc" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials/insecure" </ins> utilversion "k8s.io/apimachinery/pkg/util/version" <del> "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" </del> "k8s.io/klog/v2" <del> "k8s.io/kubernetes/pkg/kubelet/pluginmanager/cache" </del> <ins> drapb "k8s.io/kubelet/pkg/apis/dra/v1alpha4" </ins> ) <del> // defaultClientCallTimeout is the default amount of time that a DRA driver has // to respond to any of the gRPC calls. kubelet uses this value by passing nil // to RegisterPlugin. Some tests use a different, usually shorter timeout to // speed up testing. // // This is half of the kubelet retry period (according to // https://github.com/kubernetes/kubernetes/commit/0449cef8fd5217d394c5cd331d852bd50983e6b3). const defaultClientCallTimeout = 45 * time.Second // RegistrationHandler is the handler which is fed to the pluginwatcher API. type RegistrationHandler struct { // backgroundCtx is used for all future activities of the handler. // This is necessary because it implements APIs which don't // provide a context. backgroundCtx context.Context kubeClient    kubernetes.Interface getNode       func() (*v1.Node, error) } </del> <ins> // NewDRAPluginClient returns a wrapper around those gRPC methods of a DRA // driver kubelet plugin which need to be called by kubelet. The wrapper // handles gRPC connection management and logging. Connections are reused // across different NewDRAPluginClient calls. func NewDRAPluginClient(pluginName string) (*Plugin, error) { if pluginName == "" { return nil, fmt.Errorf("plugin name is empty") } </ins> <del> var _ cache.PluginHandler = &RegistrationHandler{} // NewPluginHandler returns new registration handler. // // Must only be called once per process because it manages global state. // If a kubeClient is provided, then it synchronizes ResourceSlices // with the resource information provided by plugins. func NewRegistrationHandler(kubeClient kubernetes.Interface, getNode func() (*v1.Node, error)) *RegistrationHandler { handler := &RegistrationHandler{ // The context and thus logger should come from the caller. backgroundCtx: klog.NewContext(context.TODO(), klog.LoggerWithName(klog.TODO(), "DRA registration handler")), kubeClient:    kubeClient, getNode:       getNode, </del> <ins> existingPlugin := draPlugins.get(pluginName) if existingPlugin == nil { return nil, fmt.Errorf("plugin name %s not found in the list of registered DRA plugins", pluginName) </ins> } <del> // When kubelet starts up, no DRA driver has registered yet. None of // the drivers are usable until they come back, which might not happen // at all. Therefore it is better to not advertise any local resources // because pods could get stuck on the node waiting for the driver // to start up. // // This has to run in the background. go handler.wipeResourceSlices("") </del> <ins> return existingPlugin, nil } type Plugin struct { backgroundCtx context.Context cancel        func(cause error) </ins> <del> return handler </del> <ins> mutex                   sync.Mutex conn                    *grpc.ClientConn endpoint                string highestSupportedVersion *utilversion.Version clientCallTimeout       time.Duration </ins> } <del> // wipeResourceSlices deletes ResourceSlices of the node, optionally just for a specific driver. func (h *RegistrationHandler) wipeResourceSlices(driver string) { if h.kubeClient == nil { return } ctx := h.backgroundCtx logger := klog.FromContext(ctx) </del> <ins> func (p *Plugin) getOrCreateGRPCConn() (*grpc.ClientConn, error) { p.mutex.Lock() defer p.mutex.Unlock() </ins> <del> backoff := wait.Backoff{ Duration: time.Second, Factor:   2, Jitter:   0.2, Cap:      5 * time.Minute, Steps:    100, </del> <ins> if p.conn != nil { return p.conn, nil </ins> } <del> // Error logging is done inside the loop. Context cancellation doesn't get logged. _ = wait.ExponentialBackoffWithContext(ctx, backoff, func(ctx context.Context) (bool, error) { node, err := h.getNode() if apierrors.IsNotFound(err) { return false, nil } if err != nil { logger.Error(err, "Unexpected error checking for node") return false, nil } fieldSelector := fields.Set{resourceapi.ResourceSliceSelectorNodeName: node.Name} if driver != "" { fieldSelector[resourceapi.ResourceSliceSelectorDriver] = driver } err = h.kubeClient.ResourceV1alpha3().ResourceSlices().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: fieldSelector.String()}) switch { case err == nil: logger.V(3).Info("Deleted ResourceSlices", "fieldSelector", fieldSelector) return true, nil case apierrors.IsUnauthorized(err): // This can happen while kubelet is still figuring out // its credentials. logger.V(5).Info("Deleting ResourceSlice failed, retrying", "fieldSelector", fieldSelector, "err", err) return false, nil default: // Log and retry for other errors. logger.V(3).Info("Deleting ResourceSlice failed, retrying", "fieldSelector", fieldSelector, "err", err) return false, nil } }) } // RegisterPlugin is called when a plugin can be registered. func (h *RegistrationHandler) RegisterPlugin(pluginName string, endpoint string, versions []string, pluginClientTimeout *time.Duration) error { // Prepare a context with its own logger for the plugin. // // The lifecycle of the plugin's background activities is tied to our // root context, so canceling that will also cancel the plugin. // // The logger injects the plugin name as additional value // into all log output related to the plugin. ctx := h.backgroundCtx </del> <ins> ctx := p.backgroundCtx </ins> logger := klog.FromContext(ctx) <del> logger = klog.LoggerWithValues(logger, "pluginName", pluginName) ctx = klog.NewContext(ctx, logger) logger.V(3).Info("Register new DRA plugin", "endpoint", endpoint) highestSupportedVersion, err := h.validateVersions(pluginName, versions) </del> <ins> network := "unix" logger.V(4).Info("Creating new gRPC connection", "protocol", network, "endpoint", p.endpoint) // grpc.Dial is deprecated. grpc.NewClient should be used instead. // For now this gets ignored because this function is meant to establish // the connection, with the one second timeout below. Perhaps that // approach should be reconsidered? //nolint:staticcheck conn, err := grpc.Dial( p.endpoint, grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(func(ctx context.Context, target string) (net.Conn, error) { return (&net.Dialer{}).DialContext(ctx, network, target) }), ) </ins> if err != nil { <del> return fmt.Errorf("version check of plugin %s failed: %w", pluginName, err) } var timeout time.Duration if pluginClientTimeout == nil { timeout = defaultClientCallTimeout } else { timeout = *pluginClientTimeout </del> <ins> return nil, err </ins> } <del> ctx, cancel := context.WithCancelCause(ctx) </del> <ins> ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() </ins> <del> pluginInstance := &Plugin{ backgroundCtx:           ctx, cancel:                  cancel, conn:                    nil, endpoint:                endpoint, highestSupportedVersion: highestSupportedVersion, clientCallTimeout:       timeout, </del> <ins> if ok := conn.WaitForStateChange(ctx, connectivity.Connecting); !ok { return nil, errors.New("timed out waiting for gRPC connection to be ready") </ins> } <del> // Storing endpoint of newly registered DRA Plugin into the map, where plugin name will be the key // all other DRA components will be able to get the actual socket of DRA plugins by its name. if draPlugins.add(pluginName, pluginInstance) { logger.V(1).Info("Already registered, previous plugin was replaced") } return nil </del> <ins> p.conn = conn return p.conn, nil </ins> } <del> func (h *RegistrationHandler) validateVersions( pluginName string, versions []string, ) (*utilversion.Version, error) { if len(versions) == 0 { return nil, errors.New("empty list for supported versions") } </del> <ins> func (p *Plugin) NodePrepareResources( ctx context.Context, req *drapb.NodePrepareResourcesRequest, opts ...grpc.CallOption, ) (*drapb.NodePrepareResourcesResponse, error) { logger := klog.FromContext(ctx) logger.V(4).Info("Calling NodePrepareResources rpc", "request", req) </ins> <del> // Validate version newPluginHighestVersion, err := utilversion.HighestSupportedVersion(versions) </del> <ins> conn, err := p.getOrCreateGRPCConn() </ins> if err != nil { <del> // HighestSupportedVersion includes the list of versions in its error // if relevant, no need to repeat it here. return nil, fmt.Errorf("none of the versions are supported: %w", err) } existingPlugin := draPlugins.get(pluginName) if existingPlugin == nil { return newPluginHighestVersion, nil </del> <ins> return nil, err </ins> } <del> if existingPlugin.highestSupportedVersion.LessThan(newPluginHighestVersion) { return newPluginHighestVersion, nil } return nil, fmt.Errorf("another plugin instance is already registered with a higher supported version: %q < %q", newPluginHighestVersion, existingPlugin.highestSupportedVersion) } // DeRegisterPlugin is called when a plugin has removed its socket, // signaling it is no longer available. func (h *RegistrationHandler) DeRegisterPlugin(pluginName string) { if p := draPlugins.delete(pluginName); p != nil { logger := klog.FromContext(p.backgroundCtx) logger.V(3).Info("Deregister DRA plugin", "endpoint", p.endpoint) // Clean up the ResourceSlices for the deleted Plugin since it // may have died without doing so itself and might never come // back. go h.wipeResourceSlices(pluginName) return } </del> <ins> ctx, cancel := context.WithTimeout(ctx, p.clientCallTimeout) defer cancel() </ins> <del> logger := klog.FromContext(h.backgroundCtx) logger.V(3).Info("Deregister DRA plugin not necessary, was already removed") </del> <ins> nodeClient := drapb.NewNodeClient(conn) response, err := nodeClient.NodePrepareResources(ctx, req) logger.V(4).Info("Done calling NodePrepareResources rpc", "response", response, "err", err) return response, err </ins> } <del> // ValidatePlugin is called by kubelet's plugin watcher upon detection // of a new registration socket opened by DRA plugin. func (h *RegistrationHandler) ValidatePlugin(pluginName string, endpoint string, versions []string) error { _, err := h.validateVersions(pluginName, versions) </del> <ins> func (p *Plugin) NodeUnprepareResources( ctx context.Context, req *drapb.NodeUnprepareResourcesRequest, opts ...grpc.CallOption, ) (*drapb.NodeUnprepareResourcesResponse, error) { logger := klog.FromContext(ctx) logger.V(4).Info("Calling NodeUnprepareResource rpc", "request", req) conn, err := p.getOrCreateGRPCConn() </ins> if err != nil { <del> return fmt.Errorf("invalid versions of plugin %s: %w", pluginName, err) </del> <ins> return nil, err </ins> } <del> return err </del> <ins> ctx, cancel := context.WithTimeout(ctx, p.clientCallTimeout) defer cancel() nodeClient := drapb.NewNodeClient(conn) response, err := nodeClient.NodeUnprepareResources(ctx, req) logger.V(4).Info("Done calling NodeUnprepareResources rpc", "response", response, "err", err) return response, err </ins> }


package plugin import ( <ins> "context" "fmt" "net" "os" "path/filepath" "sync" </ins> "testing" "github.com/stretchr/testify/assert" <del> v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" </del> <ins> "google.golang.org/grpc" drapb "k8s.io/kubelet/pkg/apis/dra/v1alpha4" "k8s.io/kubernetes/test/utils/ktesting" </ins> ) <del> func getFakeNode() (*v1.Node, error) { return &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "worker"}}, nil </del> <ins> const ( v1alpha4Version = "v1alpha4" ) type fakeV1alpha4GRPCServer struct { drapb.UnimplementedNodeServer } var _ drapb.NodeServer = &fakeV1alpha4GRPCServer{} func (f *fakeV1alpha4GRPCServer) NodePrepareResources(ctx context.Context, in *drapb.NodePrepareResourcesRequest) (*drapb.NodePrepareResourcesResponse, error) { return &drapb.NodePrepareResourcesResponse{Claims: map[string]*drapb.NodePrepareResourceResponse{"claim-uid": { Devices: []*drapb.Device{ { RequestNames: []string{"test-request"}, CDIDeviceIDs: []string{"test-cdi-id"}, }, }, }}}, nil } func (f *fakeV1alpha4GRPCServer) NodeUnprepareResources(ctx context.Context, in *drapb.NodeUnprepareResourcesRequest) (*drapb.NodeUnprepareResourcesResponse, error) { return &drapb.NodeUnprepareResourcesResponse{}, nil } type tearDown func() func setupFakeGRPCServer(version string) (string, tearDown, error) { p, err := os.MkdirTemp("", "dra_plugin") if err != nil { return "", nil, err } closeCh := make(chan struct{}) addr := filepath.Join(p, "server.sock") teardown := func() { close(closeCh) if err := os.RemoveAll(addr); err != nil { panic(err) } } listener, err := net.Listen("unix", addr) if err != nil { teardown() return "", nil, err } s := grpc.NewServer() switch version { case v1alpha4Version: fakeGRPCServer := &fakeV1alpha4GRPCServer{} drapb.RegisterNodeServer(s, fakeGRPCServer) default: return "", nil, fmt.Errorf("unsupported version: %s", version) } go func() { go func() { if err := s.Serve(listener); err != nil { panic(err) } }() <-closeCh s.GracefulStop() }() return addr, teardown, nil </ins> } <del> func TestRegistrationHandler_ValidatePlugin(t *testing.T) { newRegistrationHandler := func() *RegistrationHandler { return NewRegistrationHandler(nil, getFakeNode) </del> <ins> func TestGRPCConnIsReused(t *testing.T) { tCtx := ktesting.Init(t) addr, teardown, err := setupFakeGRPCServer(v1alpha4Version) if err != nil { t.Fatal(err) } defer teardown() reusedConns := make(map[*grpc.ClientConn]int) wg := sync.WaitGroup{} m := sync.Mutex{} p := &Plugin{ backgroundCtx:     tCtx, endpoint:          addr, clientCallTimeout: defaultClientCallTimeout, </ins> } <ins> conn, err := p.getOrCreateGRPCConn() defer func() { err := conn.Close() if err != nil { t.Error(err) } }() if err != nil { t.Fatal(err) } // ensure the plugin we are using is registered draPlugins.add("dummy-plugin", p) defer draPlugins.delete("dummy-plugin") // we call `NodePrepareResource` 2 times and check whether a new connection is created or the same is reused for i := 0; i < 2; i++ { wg.Add(1) go func() { defer wg.Done() client, err := NewDRAPluginClient("dummy-plugin") if err != nil { t.Error(err) return } req := &drapb.NodePrepareResourcesRequest{ Claims: []*drapb.Claim{ { Namespace: "dummy-namespace", UID:       "dummy-uid", Name:      "dummy-claim", }, }, } _, err = client.NodePrepareResources(tCtx, req) assert.NoError(t, err) client.mutex.Lock() conn := client.conn client.mutex.Unlock() m.Lock() defer m.Unlock() reusedConns[conn]++ }() } wg.Wait() // We should have only one entry otherwise it means another gRPC connection has been created if len(reusedConns) != 1 { t.Errorf("expected length to be 1 but got %d", len(reusedConns)) } if counter, ok := reusedConns[conn]; ok && counter != 2 { t.Errorf("expected counter to be 2 but got %d", counter) } } func TestNewDRAPluginClient(t *testing.T) { </ins> for _, test := range []struct { description string <del> handler     func() *RegistrationHandler </del> <ins> setup       func(string) tearDown </ins> pluginName  string <del> endpoint    string versions    []string </del> shouldError bool }{ { <del> description: "no versions provided", handler:     newRegistrationHandler, </del> <ins> description: "plugin name is empty", setup: func(_ string) tearDown { return func() {} }, pluginName:  "", </ins> shouldError: true, }, { <del> description: "unsupported version", handler:     newRegistrationHandler, versions:    []string{"v2.0.0"}, </del> <ins> description: "plugin name not found in the list", setup: func(_ string) tearDown { return func() {} }, pluginName:  "plugin-name-not-found-in-the-list", </ins> shouldError: true, }, { <del> description: "plugin already registered with a higher supported version", handler: func() *RegistrationHandler { handler := newRegistrationHandler() if err := handler.RegisterPlugin("this-plugin-already-exists-and-has-a-long-name-so-it-doesnt-collide", "", []string{"v1.1.0"}, nil); err != nil { t.Fatal(err) </del> <ins> description: "plugin exists", setup: func(name string) tearDown { draPlugins.add(name, &Plugin{}) return func() { draPlugins.delete(name) </ins> } <del> return handler </del> }, <del> pluginName:  "this-plugin-already-exists-and-has-a-long-name-so-it-doesnt-collide", versions:    []string{"v1.0.0"}, shouldError: true, }, { description: "should validate the plugin", handler:     newRegistrationHandler, pluginName:  "this-is-a-dummy-plugin-with-a-long-name-so-it-doesnt-collide", versions:    []string{"v1.3.0"}, </del> <ins> pluginName: "dummy-plugin", </ins> }, } { t.Run(test.description, func(t *testing.T) { <del> handler := test.handler() err := handler.ValidatePlugin(test.pluginName, test.endpoint, test.versions) </del> <ins> teardown := test.setup(test.pluginName) defer teardown() client, err := NewDRAPluginClient(test.pluginName) </ins> if test.shouldError { <ins> assert.Nil(t, client) </ins> assert.Error(t, err) } else { <ins> assert.NotNil(t, client) </ins> assert.Nil(t, err) } }) } <ins> } </ins> <del> t.Cleanup(func() { handler := newRegistrationHandler() handler.DeRegisterPlugin("this-plugin-already-exists-and-has-a-long-name-so-it-doesnt-collide") handler.DeRegisterPlugin("this-is-a-dummy-plugin-with-a-long-name-so-it-doesnt-collide") }) </del> <ins> func TestNodeUnprepareResources(t *testing.T) { for _, test := range []struct { description   string serverSetup   func(string) (string, tearDown, error) serverVersion string request       *drapb.NodeUnprepareResourcesRequest }{ { description:   "server supports v1alpha4", serverSetup:   setupFakeGRPCServer, serverVersion: v1alpha4Version, request:       &drapb.NodeUnprepareResourcesRequest{}, }, } { t.Run(test.description, func(t *testing.T) { tCtx := ktesting.Init(t) addr, teardown, err := setupFakeGRPCServer(test.serverVersion) if err != nil { t.Fatal(err) } defer teardown() p := &Plugin{ backgroundCtx:     tCtx, endpoint:          addr, clientCallTimeout: defaultClientCallTimeout, } conn, err := p.getOrCreateGRPCConn() defer func() { err := conn.Close() if err != nil { t.Error(err) } }() if err != nil { t.Fatal(err) } draPlugins.add("dummy-plugin", p) defer draPlugins.delete("dummy-plugin") client, err := NewDRAPluginClient("dummy-plugin") if err != nil { t.Fatal(err) } _, err = client.NodeUnprepareResources(tCtx, test.request) if err != nil { t.Fatal(err) } }) } </ins> }


<ins> /* Copyright 2022 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package plugin import ( "context" "errors" "fmt" "time" v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1alpha3" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" utilversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/kubelet/pluginmanager/cache" ) // defaultClientCallTimeout is the default amount of time that a DRA driver has // to respond to any of the gRPC calls. kubelet uses this value by passing nil // to RegisterPlugin. Some tests use a different, usually shorter timeout to // speed up testing. // // This is half of the kubelet retry period (according to // https://github.com/kubernetes/kubernetes/commit/0449cef8fd5217d394c5cd331d852bd50983e6b3). const defaultClientCallTimeout = 45 * time.Second // RegistrationHandler is the handler which is fed to the pluginwatcher API. type RegistrationHandler struct { // backgroundCtx is used for all future activities of the handler. // This is necessary because it implements APIs which don't // provide a context. backgroundCtx context.Context kubeClient    kubernetes.Interface getNode       func() (*v1.Node, error) } var _ cache.PluginHandler = &RegistrationHandler{} // NewPluginHandler returns new registration handler. // // Must only be called once per process because it manages global state. // If a kubeClient is provided, then it synchronizes ResourceSlices // with the resource information provided by plugins. func NewRegistrationHandler(kubeClient kubernetes.Interface, getNode func() (*v1.Node, error)) *RegistrationHandler { handler := &RegistrationHandler{ // The context and thus logger should come from the caller. backgroundCtx: klog.NewContext(context.TODO(), klog.LoggerWithName(klog.TODO(), "DRA registration handler")), kubeClient:    kubeClient, getNode:       getNode, } // When kubelet starts up, no DRA driver has registered yet. None of // the drivers are usable until they come back, which might not happen // at all. Therefore it is better to not advertise any local resources // because pods could get stuck on the node waiting for the driver // to start up. // // This has to run in the background. go handler.wipeResourceSlices("") return handler } // wipeResourceSlices deletes ResourceSlices of the node, optionally just for a specific driver. func (h *RegistrationHandler) wipeResourceSlices(driver string) { if h.kubeClient == nil { return } ctx := h.backgroundCtx logger := klog.FromContext(ctx) backoff := wait.Backoff{ Duration: time.Second, Factor:   2, Jitter:   0.2, Cap:      5 * time.Minute, Steps:    100, } // Error logging is done inside the loop. Context cancellation doesn't get logged. _ = wait.ExponentialBackoffWithContext(ctx, backoff, func(ctx context.Context) (bool, error) { node, err := h.getNode() if apierrors.IsNotFound(err) { return false, nil } if err != nil { logger.Error(err, "Unexpected error checking for node") return false, nil } fieldSelector := fields.Set{resourceapi.ResourceSliceSelectorNodeName: node.Name} if driver != "" { fieldSelector[resourceapi.ResourceSliceSelectorDriver] = driver } err = h.kubeClient.ResourceV1alpha3().ResourceSlices().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: fieldSelector.String()}) switch { case err == nil: logger.V(3).Info("Deleted ResourceSlices", "fieldSelector", fieldSelector) return true, nil case apierrors.IsUnauthorized(err): // This can happen while kubelet is still figuring out // its credentials. logger.V(5).Info("Deleting ResourceSlice failed, retrying", "fieldSelector", fieldSelector, "err", err) return false, nil default: // Log and retry for other errors. logger.V(3).Info("Deleting ResourceSlice failed, retrying", "fieldSelector", fieldSelector, "err", err) return false, nil } }) } // RegisterPlugin is called when a plugin can be registered. func (h *RegistrationHandler) RegisterPlugin(pluginName string, endpoint string, versions []string, pluginClientTimeout *time.Duration) error { // Prepare a context with its own logger for the plugin. // // The lifecycle of the plugin's background activities is tied to our // root context, so canceling that will also cancel the plugin. // // The logger injects the plugin name as additional value // into all log output related to the plugin. ctx := h.backgroundCtx logger := klog.FromContext(ctx) logger = klog.LoggerWithValues(logger, "pluginName", pluginName) ctx = klog.NewContext(ctx, logger) logger.V(3).Info("Register new DRA plugin", "endpoint", endpoint) highestSupportedVersion, err := h.validateVersions(pluginName, versions) if err != nil { return fmt.Errorf("version check of plugin %s failed: %w", pluginName, err) } var timeout time.Duration if pluginClientTimeout == nil { timeout = defaultClientCallTimeout } else { timeout = *pluginClientTimeout } ctx, cancel := context.WithCancelCause(ctx) pluginInstance := &Plugin{ backgroundCtx:           ctx, cancel:                  cancel, conn:                    nil, endpoint:                endpoint, highestSupportedVersion: highestSupportedVersion, clientCallTimeout:       timeout, } // Storing endpoint of newly registered DRA Plugin into the map, where plugin name will be the key // all other DRA components will be able to get the actual socket of DRA plugins by its name. if draPlugins.add(pluginName, pluginInstance) { logger.V(1).Info("Already registered, previous plugin was replaced") } return nil } func (h *RegistrationHandler) validateVersions( pluginName string, versions []string, ) (*utilversion.Version, error) { if len(versions) == 0 { return nil, errors.New("empty list for supported versions") } // Validate version newPluginHighestVersion, err := utilversion.HighestSupportedVersion(versions) if err != nil { // HighestSupportedVersion includes the list of versions in its error // if relevant, no need to repeat it here. return nil, fmt.Errorf("none of the versions are supported: %w", err) } existingPlugin := draPlugins.get(pluginName) if existingPlugin == nil { return newPluginHighestVersion, nil } if existingPlugin.highestSupportedVersion.LessThan(newPluginHighestVersion) { return newPluginHighestVersion, nil } return nil, fmt.Errorf("another plugin instance is already registered with a higher supported version: %q < %q", newPluginHighestVersion, existingPlugin.highestSupportedVersion) } // DeRegisterPlugin is called when a plugin has removed its socket, // signaling it is no longer available. func (h *RegistrationHandler) DeRegisterPlugin(pluginName string) { if p := draPlugins.delete(pluginName); p != nil { logger := klog.FromContext(p.backgroundCtx) logger.V(3).Info("Deregister DRA plugin", "endpoint", p.endpoint) // Clean up the ResourceSlices for the deleted Plugin since it // may have died without doing so itself and might never come // back. go h.wipeResourceSlices(pluginName) return } logger := klog.FromContext(h.backgroundCtx) logger.V(3).Info("Deregister DRA plugin not necessary, was already removed") } // ValidatePlugin is called by kubelet's plugin watcher upon detection // of a new registration socket opened by DRA plugin. func (h *RegistrationHandler) ValidatePlugin(pluginName string, endpoint string, versions []string) error { _, err := h.validateVersions(pluginName, versions) if err != nil { return fmt.Errorf("invalid versions of plugin %s: %w", pluginName, err) } return err } </ins>


<ins> /* Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package plugin import ( "testing" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func getFakeNode() (*v1.Node, error) { return &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "worker"}}, nil } func TestRegistrationHandler_ValidatePlugin(t *testing.T) { newRegistrationHandler := func() *RegistrationHandler { return NewRegistrationHandler(nil, getFakeNode) } for _, test := range []struct { description string handler     func() *RegistrationHandler pluginName  string endpoint    string versions    []string shouldError bool }{ { description: "no versions provided", handler:     newRegistrationHandler, shouldError: true, }, { description: "unsupported version", handler:     newRegistrationHandler, versions:    []string{"v2.0.0"}, shouldError: true, }, { description: "plugin already registered with a higher supported version", handler: func() *RegistrationHandler { handler := newRegistrationHandler() if err := handler.RegisterPlugin("this-plugin-already-exists-and-has-a-long-name-so-it-doesnt-collide", "", []string{"v1.1.0"}, nil); err != nil { t.Fatal(err) } return handler }, pluginName:  "this-plugin-already-exists-and-has-a-long-name-so-it-doesnt-collide", versions:    []string{"v1.0.0"}, shouldError: true, }, { description: "should validate the plugin", handler:     newRegistrationHandler, pluginName:  "this-is-a-dummy-plugin-with-a-long-name-so-it-doesnt-collide", versions:    []string{"v1.3.0"}, }, } { t.Run(test.description, func(t *testing.T) { handler := test.handler() err := handler.ValidatePlugin(test.pluginName, test.endpoint, test.versions) if test.shouldError { assert.Error(t, err) } else { assert.NoError(t, err) } }) } t.Cleanup(func() { handler := newRegistrationHandler() handler.DeRegisterPlugin("this-plugin-already-exists-and-has-a-long-name-so-it-doesnt-collide") handler.DeRegisterPlugin("this-is-a-dummy-plugin-with-a-long-name-so-it-doesnt-collide") }) } </ins>


``` cd kubernetes <del> sudo hack/local-up-cluster.sh </del> <ins> hack/local-up-cluster.sh </ins> ``` This will build and start a lightweight local cluster, consisting of a master


GO_OUT="${KUBE_ROOT}/_output/local/bin/${host_os}/${host_arch}" APISERVER_LOG=/tmp/kube-apiserver.log <del> "${GO_OUT}/kube-apiserver"  </del> <ins> sudo "${GO_OUT}/kube-apiserver"  </ins> -v=${LOG_LEVEL}  --address="${API_HOST}"  --port="${API_PORT}"


kube::util::wait_for_url "http://${API_HOST}:${API_PORT}/api/v1beta1/pods" "apiserver: " CTLRMGR_LOG=/tmp/kube-controller-manager.log <del> "${GO_OUT}/kube-controller-manager"  </del> <ins> sudo "${GO_OUT}/kube-controller-manager"  </ins> -v=${LOG_LEVEL}  --machines="127.0.0.1"  --master="${API_HOST}:${API_PORT}" >"${CTLRMGR_LOG}" 2>&1 & CTLRMGR_PID=$! KUBELET_LOG=/tmp/kubelet.log <del> "${GO_OUT}/kubelet"  </del> <ins> sudo "${GO_OUT}/kubelet"  </ins> -v=${LOG_LEVEL}  --etcd_servers="http://127.0.0.1:4001"  --hostname_override="127.0.0.1"


KUBELET_PID=$! PROXY_LOG=/tmp/kube-proxy.log <del> "${GO_OUT}/kube-proxy"  </del> <ins> sudo "${GO_OUT}/kube-proxy"  </ins> -v=${LOG_LEVEL}  --master="http://${API_HOST}:${API_PORT}" >"${PROXY_LOG}" 2>&1 & PROXY_PID=$! SCHEDULER_LOG=/tmp/kube-scheduler.log <del> "${GO_OUT}/kube-scheduler"  </del> <ins> sudo "${GO_OUT}/kube-scheduler"  </ins> -v=${LOG_LEVEL}  --master="http://${API_HOST}:${API_PORT}" >"${SCHEDULER_LOG}" 2>&1 & SCHEDULER_PID=$!


cleanup() { echo "Cleaning up..." <del> kill "${APISERVER_PID}" kill "${CTLRMGR_PID}" kill "${KUBELET_PID}" kill "${PROXY_PID}" kill "${SCHEDULER_PID}" </del> <ins> sudo kill "${APISERVER_PID}" sudo kill "${CTLRMGR_PID}" sudo kill "${KUBELET_PID}" sudo kill "${PROXY_PID}" sudo kill "${SCHEDULER_PID}" </ins> kill "${ETCD_PID}" rm -rf "${ETCD_DIR}"


} ginkgo.BeforeEach(func() { <ins> // Precautionary check that kubelet is healthy before running the test. waitForKubeletToStart(context.TODO(), f) </ins> ginkgo.By("setting up the pod to be used in the test") e2epod.NewPodClient(f).Create(context.TODO(), testCase.podSpec) })


Name: podName, }, Spec: v1.PodSpec{ <del> RestartPolicy: v1.RestartPolicyNever, </del> <ins> PriorityClassName: "system-node-critical", RestartPolicy:     v1.RestartPolicyNever, </ins> Containers: []v1.Container{ createContainer(ctnName), },


"sh", "-c", // use the dd tool to attempt to allocate huge block of memory which exceeds the node allocatable <del> "sleep 5 && dd if=/dev/zero of=/dev/null iflag=fullblock count=10 bs=10G", </del> <ins> "sleep 5 && dd if=/dev/zero of=/dev/null iflag=fullblock count=10 bs=1024G", </ins> }, SecurityContext: &v1.SecurityContext{ SeccompProfile: &v1.SeccompProfile{


} if len(sorted) != len(graph) { <del> fmt.Printf("%s", (sorted)) </del> return nil // Cycle detected }


} logger.V(4).Info("Update endpoints", "service", klog.KObj(service), "readyEndpoints", totalReadyEps, "notreadyEndpoints", totalNotReadyEps) <ins> var updatedEndpoints *v1.Endpoints </ins> if createEndpoints { // No previous endpoints, create them _, err = e.client.CoreV1().Endpoints(service.Namespace).Create(ctx, newEndpoints, metav1.CreateOptions{}) } else { // Pre-existing <del> _, err = e.client.CoreV1().Endpoints(service.Namespace).Update(ctx, newEndpoints, metav1.UpdateOptions{}) </del> <ins> updatedEndpoints, err = e.client.CoreV1().Endpoints(service.Namespace).Update(ctx, newEndpoints, metav1.UpdateOptions{}) </ins> } if err != nil { if createEndpoints && errors.IsForbidden(err) {


// If the current endpoints is updated we track the old resource version, so // if we obtain this resource version again from the lister we know is outdated // and we need to retry later to wait for the informer cache to be up-to-date. <del> if !createEndpoints { </del> <ins> // there are some operations (webhooks, truncated endpoints, ...) that can potentially cause endpoints updates became noop // and return the same resourceVersion. // Ref: https://issues.k8s.io/127370 , https://issues.k8s.io/126578 if updatedEndpoints != nil && updatedEndpoints.ResourceVersion != currentEndpoints.ResourceVersion { </ins> e.staleEndpointsTracker.Stale(currentEndpoints) } return nil


apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" <ins> "k8s.io/apimachinery/pkg/util/sets" </ins> "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes"


"k8s.io/kubernetes/pkg/controller/endpoint" "k8s.io/kubernetes/test/integration/framework" "k8s.io/kubernetes/test/utils/ktesting" <ins> netutils "k8s.io/utils/net" </ins> ) func TestEndpointUpdates(t *testing.T) {


svc.Spec.ExternalName = "google.com" return svc } <ins> func TestEndpointTruncate(t *testing.T) { // Disable ServiceAccount admission plugin as we don't have serviceaccount controller running. server := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), framework.SharedEtcd()) defer server.TearDownFn() client, err := clientset.NewForConfig(server.ClientConfig) if err != nil { t.Fatalf("Error creating clientset: %v", err) } informers := informers.NewSharedInformerFactory(client, 0) tCtx := ktesting.Init(t) epController := endpoint.NewEndpointController( tCtx, informers.Core().V1().Pods(), informers.Core().V1().Services(), informers.Core().V1().Endpoints(), client, 0) // Start informer and controllers informers.Start(tCtx.Done()) go epController.Run(tCtx, 1) // Create namespace ns := framework.CreateNamespaceOrDie(client, "test-endpoints-truncate", t) defer framework.DeleteNamespaceOrDie(client, ns, t) // Create a pod with labels basePod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:   "test-pod", Labels: labelMap(), }, Spec: v1.PodSpec{ NodeName: "fake-node", Containers: []v1.Container{ { Name:  "fakename", Image: "fakeimage", Ports: []v1.ContainerPort{ { Name:          "port-443", ContainerPort: 443, }, }, }, }, }, Status: v1.PodStatus{ Phase: v1.PodRunning, Conditions: []v1.PodCondition{ { Type:   v1.PodReady, Status: v1.ConditionTrue, }, }, PodIP: "10.0.0.1", PodIPs: []v1.PodIP{ { IP: "10.0.0.1", }, }, }, } // create 1001 Pods to reach endpoint max capacity that is set to 1000 allPodNames := sets.New[string]() baseIP := netutils.BigForIP(netutils.ParseIPSloppy("10.0.0.1")) for i := 0; i < 1001; i++ { pod := basePod.DeepCopy() pod.Name = fmt.Sprintf("%s-%d", basePod.Name, i) allPodNames.Insert(pod.Name) podIP := netutils.AddIPOffset(baseIP, i).String() pod.Status.PodIP = podIP pod.Status.PodIPs[0] = v1.PodIP{IP: podIP} createdPod, err := client.CoreV1().Pods(ns.Name).Create(tCtx, pod, metav1.CreateOptions{}) if err != nil { t.Fatalf("Failed to create pod %s: %v", pod.Name, err) } createdPod.Status = pod.Status _, err = client.CoreV1().Pods(ns.Name).UpdateStatus(tCtx, createdPod, metav1.UpdateOptions{}) if err != nil { t.Fatalf("Failed to update status of pod %s: %v", pod.Name, err) } } // Create a service associated to the pod svc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name:      "test-service", Namespace: ns.Name, Labels: map[string]string{ "foo": "bar", }, }, Spec: v1.ServiceSpec{ Selector: map[string]string{ "foo": "bar", }, Ports: []v1.ServicePort{ {Name: "port-443", Port: 443, Protocol: "TCP", TargetPort: intstr.FromInt32(443)}, }, }, } _, err = client.CoreV1().Services(ns.Name).Create(tCtx, svc, metav1.CreateOptions{}) if err != nil { t.Fatalf("Failed to create service %s: %v", svc.Name, err) } var truncatedPodName string // poll until associated Endpoints to the previously created Service exists if err := wait.PollUntilContextTimeout(tCtx, 1*time.Second, 10*time.Second, true, func(context.Context) (bool, error) { podNames := sets.New[string]() endpoints, err := client.CoreV1().Endpoints(ns.Name).Get(tCtx, svc.Name, metav1.GetOptions{}) if err != nil { return false, nil } for _, subset := range endpoints.Subsets { for _, address := range subset.Addresses { podNames.Insert(address.TargetRef.Name) } } if podNames.Len() != 1000 { return false, nil } truncated, ok := endpoints.Annotations[v1.EndpointsOverCapacity] if !ok || truncated != "truncated" { return false, nil } // There is only 1 truncated Pod. truncatedPodName, _ = allPodNames.Difference(podNames).PopAny() return true, nil }); err != nil { t.Fatalf("endpoints not found: %v", err) } // Update the truncated Pod several times to make endpoints controller resync the service. truncatedPod, err := client.CoreV1().Pods(ns.Name).Get(tCtx, truncatedPodName, metav1.GetOptions{}) if err != nil { t.Fatalf("Failed to get pod %s: %v", truncatedPodName, err) } for i := 0; i < 10; i++ { truncatedPod.Status.Conditions[0].Status = v1.ConditionFalse truncatedPod, err = client.CoreV1().Pods(ns.Name).UpdateStatus(tCtx, truncatedPod, metav1.UpdateOptions{}) if err != nil { t.Fatalf("Failed to update status of pod %s: %v", truncatedPod.Name, err) } truncatedPod.Status.Conditions[0].Status = v1.ConditionTrue truncatedPod, err = client.CoreV1().Pods(ns.Name).UpdateStatus(tCtx, truncatedPod, metav1.UpdateOptions{}) if err != nil { t.Fatalf("Failed to update status of pod %s: %v", truncatedPod.Name, err) } } // delete 501 Pods for i := 500; i < 1001; i++ { podName := fmt.Sprintf("%s-%d", basePod.Name, i) err = client.CoreV1().Pods(ns.Name).Delete(tCtx, podName, metav1.DeleteOptions{}) if err != nil { t.Fatalf("error deleting test pod: %v", err) } } // poll until endpoints for deleted Pod are no longer in Endpoints. if err := wait.PollUntilContextTimeout(tCtx, 1*time.Second, 10*time.Second, true, func(context.Context) (bool, error) { endpoints, err := client.CoreV1().Endpoints(ns.Name).Get(tCtx, svc.Name, metav1.GetOptions{}) if err != nil { return false, nil } numEndpoints := 0 for _, subset := range endpoints.Subsets { numEndpoints += len(subset.Addresses) } if numEndpoints != 500 { return false, nil } truncated, ok := endpoints.Annotations[v1.EndpointsOverCapacity] if ok || truncated == "truncated" { return false, nil } return true, nil }); err != nil { t.Fatalf("error checking for no endpoints with terminating pods: %v", err) } } </ins>


// our target metric should be updated by now if err := wait.PollUntilContextTimeout(ctx, 10*time.Second, 2*time.Minute, true, func(_ context.Context) (bool, error) { metrics, err := metricsGrabber.GrabFromKubeProxy(ctx, nodeName) <del> framework.ExpectNoError(err) </del> <ins> if err != nil { return false, fmt.Errorf("failed to fetch metrics: %w", err) } </ins> targetMetricAfter, err := metrics.GetCounterMetricValue(metricName) <del> framework.ExpectNoError(err) </del> <ins> if err != nil { return false, fmt.Errorf("failed to fetch metric: %w", err) } </ins> return targetMetricAfter > targetMetricBefore, nil }); err != nil { <del> framework.Failf("expected %s metric to be updated after accessing endpoints via localhost nodeports", metricName) </del> <ins> if wait.Interrupted(err) { framework.Failf("expected %s metric to be updated after accessing endpoints via localhost nodeports", metricName) } framework.ExpectNoError(err) </ins> } }) })


APIServerTracing featuregate.Feature = "APIServerTracing" // owner: @linxiulei <del> // beta: v1.30 </del> <ins> // alpha: v1.30 </ins> // // Enables serving watch requests in separate goroutines. APIServingWithRoutine featuregate.Feature = "APIServingWithRoutine"


} if len(strings.Split(stringValue, "/")) == 1 { if !standardFinalizers.Has(stringValue) { <del> allWarnings = append(allWarnings, fmt.Sprintf("%s: %q: prefer a domain-qualified finalizer name to avoid accidental conflicts with other finalizer writers", fldPath.String(), stringValue)) </del> <ins> if strings.Contains(stringValue, ".") { allWarnings = append(allWarnings, fmt.Sprintf("%s: %q: prefer a domain-qualified finalizer name including a path (/) to avoid accidental conflicts with other finalizer writers", fldPath.String(), stringValue)) } else { allWarnings = append(allWarnings, fmt.Sprintf("%s: %q: prefer a domain-qualified finalizer name to avoid accidental conflicts with other finalizer writers", fldPath.String(), stringValue)) } </ins> } } return allWarnings


finalizer: []string{"kubernetes.io/valid-finalizer"}, }, { <ins> name:      "create-crd-with-fqdn-like-finalizer-without-path", finalizer: []string{"finalizer.without.valid-path.io"}, expectCreateWarnings: []string{ `metadata.finalizers: "finalizer.without.valid-path.io": prefer a domain-qualified finalizer name including a path (/) to avoid accidental conflicts with other finalizer writers`, }, }, { </ins> name:             "update-crd-with-invalid-finalizer", finalizer:        []string{"invalid-finalizer"}, updatedFinalizer: []string{"another-invalid-finalizer"},


<ins> /* Copyright 2024 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package storage import ( "context" "fmt" "time" storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" utilrand "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/util/retry" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/utils" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" ) var _ = utils.SIGDescribe("CSINodes", func() { f := framework.NewDefaultFramework("csinodes") ginkgo.Describe("CSI Conformance", func() { ginkgo.It("should run through the lifecycle of a csinode", func(ctx context.Context) { csiNodeClient := f.ClientSet.StorageV1().CSINodes() initialCSINode := storagev1.CSINode{ ObjectMeta: metav1.ObjectMeta{ Name: "e2e-csinode-" + utilrand.String(5), }, } ginkgo.By(fmt.Sprintf("Creating initial csiNode %q", initialCSINode.Name)) csiNode, err := csiNodeClient.Create(ctx, &initialCSINode, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create csiNode %q", initialCSINode.Name) ginkgo.By(fmt.Sprintf("Getting initial csiNode %q", initialCSINode.Name)) retrievedCSINode, err := csiNodeClient.Get(ctx, initialCSINode.Name, metav1.GetOptions{}) framework.ExpectNoError(err, "Failed to retrieve csiNode %q", initialCSINode.Name) gomega.Expect(retrievedCSINode.Name).To(gomega.Equal(csiNode.Name), "Checking that the retrieved name has been found") ginkgo.By(fmt.Sprintf("Patching initial csiNode: %q", initialCSINode.Name)) payload := "{"metadata":{"labels":{"" + csiNode.Name + "":"patched"}}}" patchedCSINode, err := csiNodeClient.Patch(ctx, csiNode.Name, types.StrategicMergePatchType, []byte(payload), metav1.PatchOptions{}) framework.ExpectNoError(err, "Failed to patch csiNode %q", csiNode.Name) gomega.Expect(patchedCSINode.Labels).To(gomega.HaveKeyWithValue(csiNode.Name, "patched"), "Checking that patched label has been applied") patchedSelector := labels.Set{csiNode.Name: "patched"}.AsSelector().String() ginkgo.By(fmt.Sprintf("Listing csiNodes with LabelSelector %q", patchedSelector)) csiNodeList, err := csiNodeClient.List(ctx, metav1.ListOptions{LabelSelector: patchedSelector}) framework.ExpectNoError(err, "failed to list csiNodes") gomega.Expect(csiNodeList.Items).To(gomega.HaveLen(1)) ginkgo.By(fmt.Sprintf("Delete initial csiNode: %q", initialCSINode.Name)) err = csiNodeClient.Delete(ctx, csiNode.Name, metav1.DeleteOptions{}) framework.ExpectNoError(err, "failed to delete csiNode %q", initialCSINode.Name) ginkgo.By(fmt.Sprintf("Confirm deletion of csiNode %q", initialCSINode.Name)) type state struct { CSINodes []storagev1.CSINode } err = framework.Gomega().Eventually(ctx, framework.HandleRetry(func(ctx context.Context) (*state, error) { csiNodeList, err := csiNodeClient.List(ctx, metav1.ListOptions{LabelSelector: patchedSelector}) if err != nil { return nil, fmt.Errorf("failed to list CSINode: %w", err) } return &state{ CSINodes: csiNodeList.Items, }, nil })).WithTimeout(30 * time.Second).Should(framework.MakeMatcher(func(s *state) (func() string, error) { if len(s.CSINodes) == 0 { return nil, nil } return func() string { return fmt.Sprintf("Expected CSINode to be deleted, found %q", s.CSINodes[0].Name) }, nil })) framework.ExpectNoError(err, "Timeout while waiting to confirm CSINode %q deletion", initialCSINode.Name) replacementCSINode := storagev1.CSINode{ ObjectMeta: metav1.ObjectMeta{ Name: "e2e-csinode-" + utilrand.String(5), }, } ginkgo.By(fmt.Sprintf("Creating replacement csiNode %q", replacementCSINode.Name)) secondCSINode, err := csiNodeClient.Create(ctx, &replacementCSINode, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create csiNode %q", replacementCSINode.Name) ginkgo.By(fmt.Sprintf("Getting replacement csiNode %q", replacementCSINode.Name)) retrievedCSINode, err = csiNodeClient.Get(ctx, secondCSINode.Name, metav1.GetOptions{}) framework.ExpectNoError(err, "Failed to retrieve CSINode %q", replacementCSINode.Name) gomega.Expect(retrievedCSINode.Name).To(gomega.Equal(secondCSINode.Name), "Checking that the retrieved name has been found") ginkgo.By(fmt.Sprintf("Updating replacement csiNode %q", retrievedCSINode.Name)) var updatedCSINode *storagev1.CSINode err = retry.RetryOnConflict(retry.DefaultRetry, func() error { tmpCSINode, err := csiNodeClient.Get(ctx, retrievedCSINode.Name, metav1.GetOptions{}) framework.ExpectNoError(err, "Unable to get %q", replacementCSINode.Name) tmpCSINode.Labels = map[string]string{replacementCSINode.Name: "updated"} updatedCSINode, err = csiNodeClient.Update(ctx, tmpCSINode, metav1.UpdateOptions{}) return err }) framework.ExpectNoError(err, "failed to update %q", replacementCSINode.Name) gomega.Expect(updatedCSINode.Labels).To(gomega.HaveKeyWithValue(secondCSINode.Name, "updated"), "Checking that updated label has been applied") updatedSelector := labels.Set{retrievedCSINode.Name: "updated"}.AsSelector().String() ginkgo.By(fmt.Sprintf("DeleteCollection of CSINodes with %q label", updatedSelector)) err = csiNodeClient.DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: updatedSelector}) framework.ExpectNoError(err, "failed to delete csiNode Colllection") ginkgo.By(fmt.Sprintf("Confirm deletion of replacement csiNode with LabelSelector %q", updatedSelector)) err = framework.Gomega().Eventually(ctx, framework.HandleRetry(func(ctx context.Context) (*state, error) { csiNodeList, err := csiNodeClient.List(ctx, metav1.ListOptions{LabelSelector: updatedSelector}) if err != nil { return nil, fmt.Errorf("failed to list CSINode: %w", err) } return &state{ CSINodes: csiNodeList.Items, }, nil })).WithTimeout(30 * time.Second).Should(framework.MakeMatcher(func(s *state) (func() string, error) { if len(s.CSINodes) == 0 { return nil, nil } return func() string { return fmt.Sprintf("Expected CSINode to be deleted, found %q", s.CSINodes[0].Name) }, nil })) framework.ExpectNoError(err, "Timeout while waiting to confirm CSINode %q deletion", replacementCSINode.Name) }) }) }) </ins>