Question:
The PR above moved everything related to into because that was where most of the code already was, but didn't renamed the files to avoid a larger diff and merge conflicts. Once is merged, we can do the renaming. Code clarity. /sig node
/triage accepted /priority important-soon
/assign
a PR can be created for this now.
will do

package plugin import ( <ins> "context" "fmt" "net" "os" "path/filepath" "sync" </ins> "testing" "github.com/stretchr/testify/assert" <del> v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" </del> <ins> "google.golang.org/grpc" drapb "k8s.io/kubelet/pkg/apis/dra/v1alpha4" "k8s.io/kubernetes/test/utils/ktesting" </ins> ) <del> func getFakeNode() (*v1.Node, error) { return &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "worker"}}, nil </del> <ins> const ( v1alpha4Version = "v1alpha4" ) type fakeV1alpha4GRPCServer struct { drapb.UnimplementedNodeServer } var _ drapb.NodeServer = &fakeV1alpha4GRPCServer{} func (f *fakeV1alpha4GRPCServer) NodePrepareResources(ctx context.Context, in *drapb.NodePrepareResourcesRequest) (*drapb.NodePrepareResourcesResponse, error) { return &drapb.NodePrepareResourcesResponse{Claims: map[string]*drapb.NodePrepareResourceResponse{"claim-uid": { Devices: []*drapb.Device{ { RequestNames: []string{"test-request"}, CDIDeviceIDs: []string{"test-cdi-id"}, }, }, }}}, nil } func (f *fakeV1alpha4GRPCServer) NodeUnprepareResources(ctx context.Context, in *drapb.NodeUnprepareResourcesRequest) (*drapb.NodeUnprepareResourcesResponse, error) { return &drapb.NodeUnprepareResourcesResponse{}, nil } type tearDown func() func setupFakeGRPCServer(version string) (string, tearDown, error) { p, err := os.MkdirTemp("", "dra_plugin") if err != nil { return "", nil, err } closeCh := make(chan struct{}) addr := filepath.Join(p, "server.sock") teardown := func() { close(closeCh) if err := os.RemoveAll(addr); err != nil { panic(err) } } listener, err := net.Listen("unix", addr) if err != nil { teardown() return "", nil, err } s := grpc.NewServer() switch version { case v1alpha4Version: fakeGRPCServer := &fakeV1alpha4GRPCServer{} drapb.RegisterNodeServer(s, fakeGRPCServer) default: return "", nil, fmt.Errorf("unsupported version: %s", version) } go func() { go func() { if err := s.Serve(listener); err != nil { panic(err) } }() <-closeCh s.GracefulStop() }() return addr, teardown, nil </ins> } <del> func TestRegistrationHandler_ValidatePlugin(t *testing.T) { newRegistrationHandler := func() *RegistrationHandler { return NewRegistrationHandler(nil, getFakeNode) </del> <ins> func TestGRPCConnIsReused(t *testing.T) { tCtx := ktesting.Init(t) addr, teardown, err := setupFakeGRPCServer(v1alpha4Version) if err != nil { t.Fatal(err) } defer teardown() reusedConns := make(map[*grpc.ClientConn]int) wg := sync.WaitGroup{} m := sync.Mutex{} p := &Plugin{ backgroundCtx:     tCtx, endpoint:          addr, clientCallTimeout: defaultClientCallTimeout, </ins> } <ins> conn, err := p.getOrCreateGRPCConn() defer func() { err := conn.Close() if err != nil { t.Error(err) } }() if err != nil { t.Fatal(err) } // ensure the plugin we are using is registered draPlugins.add("dummy-plugin", p) defer draPlugins.delete("dummy-plugin") // we call `NodePrepareResource` 2 times and check whether a new connection is created or the same is reused for i := 0; i < 2; i++ { wg.Add(1) go func() { defer wg.Done() client, err := NewDRAPluginClient("dummy-plugin") if err != nil { t.Error(err) return } req := &drapb.NodePrepareResourcesRequest{ Claims: []*drapb.Claim{ { Namespace: "dummy-namespace", UID:       "dummy-uid", Name:      "dummy-claim", }, }, } _, err = client.NodePrepareResources(tCtx, req) assert.NoError(t, err) client.mutex.Lock() conn := client.conn client.mutex.Unlock() m.Lock() defer m.Unlock() reusedConns[conn]++ }() } wg.Wait() // We should have only one entry otherwise it means another gRPC connection has been created if len(reusedConns) != 1 { t.Errorf("expected length to be 1 but got %d", len(reusedConns)) } if counter, ok := reusedConns[conn]; ok && counter != 2 { t.Errorf("expected counter to be 2 but got %d", counter) } } func TestNewDRAPluginClient(t *testing.T) { </ins> for _, test := range []struct { description string <del> handler     func() *RegistrationHandler </del> <ins> setup       func(string) tearDown </ins> pluginName  string <del> endpoint    string versions    []string </del> shouldError bool }{ { <del> description: "no versions provided", handler:     newRegistrationHandler, </del> <ins> description: "plugin name is empty", setup: func(_ string) tearDown { return func() {} }, pluginName:  "", </ins> shouldError: true, }, { <del> description: "unsupported version", handler:     newRegistrationHandler, versions:    []string{"v2.0.0"}, </del> <ins> description: "plugin name not found in the list", setup: func(_ string) tearDown { return func() {} }, pluginName:  "plugin-name-not-found-in-the-list", </ins> shouldError: true, }, { <del> description: "plugin already registered with a higher supported version", handler: func() *RegistrationHandler { handler := newRegistrationHandler() if err := handler.RegisterPlugin("this-plugin-already-exists-and-has-a-long-name-so-it-doesnt-collide", "", []string{"v1.1.0"}, nil); err != nil { t.Fatal(err) </del> <ins> description: "plugin exists", setup: func(name string) tearDown { draPlugins.add(name, &Plugin{}) return func() { draPlugins.delete(name) </ins> } <del> return handler </del> }, <del> pluginName:  "this-plugin-already-exists-and-has-a-long-name-so-it-doesnt-collide", versions:    []string{"v1.0.0"}, shouldError: true, }, { description: "should validate the plugin", handler:     newRegistrationHandler, pluginName:  "this-is-a-dummy-plugin-with-a-long-name-so-it-doesnt-collide", versions:    []string{"v1.3.0"}, </del> <ins> pluginName: "dummy-plugin", </ins> }, } { t.Run(test.description, func(t *testing.T) { <del> handler := test.handler() err := handler.ValidatePlugin(test.pluginName, test.endpoint, test.versions) </del> <ins> teardown := test.setup(test.pluginName) defer teardown() client, err := NewDRAPluginClient(test.pluginName) </ins> if test.shouldError { <ins> assert.Nil(t, client) </ins> assert.Error(t, err) } else { <ins> assert.NotNil(t, client) </ins> assert.Nil(t, err) } }) } <ins> } </ins> <del> t.Cleanup(func() { handler := newRegistrationHandler() handler.DeRegisterPlugin("this-plugin-already-exists-and-has-a-long-name-so-it-doesnt-collide") handler.DeRegisterPlugin("this-is-a-dummy-plugin-with-a-long-name-so-it-doesnt-collide") }) </del> <ins> func TestNodeUnprepareResources(t *testing.T) { for _, test := range []struct { description   string serverSetup   func(string) (string, tearDown, error) serverVersion string request       *drapb.NodeUnprepareResourcesRequest }{ { description:   "server supports v1alpha4", serverSetup:   setupFakeGRPCServer, serverVersion: v1alpha4Version, request:       &drapb.NodeUnprepareResourcesRequest{}, }, } { t.Run(test.description, func(t *testing.T) { tCtx := ktesting.Init(t) addr, teardown, err := setupFakeGRPCServer(test.serverVersion) if err != nil { t.Fatal(err) } defer teardown() p := &Plugin{ backgroundCtx:     tCtx, endpoint:          addr, clientCallTimeout: defaultClientCallTimeout, } conn, err := p.getOrCreateGRPCConn() defer func() { err := conn.Close() if err != nil { t.Error(err) } }() if err != nil { t.Fatal(err) } draPlugins.add("dummy-plugin", p) defer draPlugins.delete("dummy-plugin") client, err := NewDRAPluginClient("dummy-plugin") if err != nil { t.Fatal(err) } _, err = client.NodeUnprepareResources(tCtx, test.request) if err != nil { t.Fatal(err) } }) } </ins> }


Question:
The PR above moved everything related to into because that was where most of the code already was, but didn't renamed the files to avoid a larger diff and merge conflicts. Once is merged, we can do the renaming. Code clarity. /sig node
/triage accepted /priority important-soon
/assign
a PR can be created for this now.
will do

<ins> /* Copyright 2022 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package plugin import ( "context" "errors" "fmt" "time" v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1alpha3" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" utilversion "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/klog/v2" "k8s.io/kubernetes/pkg/kubelet/pluginmanager/cache" ) // defaultClientCallTimeout is the default amount of time that a DRA driver has // to respond to any of the gRPC calls. kubelet uses this value by passing nil // to RegisterPlugin. Some tests use a different, usually shorter timeout to // speed up testing. // // This is half of the kubelet retry period (according to // https://github.com/kubernetes/kubernetes/commit/0449cef8fd5217d394c5cd331d852bd50983e6b3). const defaultClientCallTimeout = 45 * time.Second // RegistrationHandler is the handler which is fed to the pluginwatcher API. type RegistrationHandler struct { // backgroundCtx is used for all future activities of the handler. // This is necessary because it implements APIs which don't // provide a context. backgroundCtx context.Context kubeClient    kubernetes.Interface getNode       func() (*v1.Node, error) } var _ cache.PluginHandler = &RegistrationHandler{} // NewPluginHandler returns new registration handler. // // Must only be called once per process because it manages global state. // If a kubeClient is provided, then it synchronizes ResourceSlices // with the resource information provided by plugins. func NewRegistrationHandler(kubeClient kubernetes.Interface, getNode func() (*v1.Node, error)) *RegistrationHandler { handler := &RegistrationHandler{ // The context and thus logger should come from the caller. backgroundCtx: klog.NewContext(context.TODO(), klog.LoggerWithName(klog.TODO(), "DRA registration handler")), kubeClient:    kubeClient, getNode:       getNode, } // When kubelet starts up, no DRA driver has registered yet. None of // the drivers are usable until they come back, which might not happen // at all. Therefore it is better to not advertise any local resources // because pods could get stuck on the node waiting for the driver // to start up. // // This has to run in the background. go handler.wipeResourceSlices("") return handler } // wipeResourceSlices deletes ResourceSlices of the node, optionally just for a specific driver. func (h *RegistrationHandler) wipeResourceSlices(driver string) { if h.kubeClient == nil { return } ctx := h.backgroundCtx logger := klog.FromContext(ctx) backoff := wait.Backoff{ Duration: time.Second, Factor:   2, Jitter:   0.2, Cap:      5 * time.Minute, Steps:    100, } // Error logging is done inside the loop. Context cancellation doesn't get logged. _ = wait.ExponentialBackoffWithContext(ctx, backoff, func(ctx context.Context) (bool, error) { node, err := h.getNode() if apierrors.IsNotFound(err) { return false, nil } if err != nil { logger.Error(err, "Unexpected error checking for node") return false, nil } fieldSelector := fields.Set{resourceapi.ResourceSliceSelectorNodeName: node.Name} if driver != "" { fieldSelector[resourceapi.ResourceSliceSelectorDriver] = driver } err = h.kubeClient.ResourceV1alpha3().ResourceSlices().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: fieldSelector.String()}) switch { case err == nil: logger.V(3).Info("Deleted ResourceSlices", "fieldSelector", fieldSelector) return true, nil case apierrors.IsUnauthorized(err): // This can happen while kubelet is still figuring out // its credentials. logger.V(5).Info("Deleting ResourceSlice failed, retrying", "fieldSelector", fieldSelector, "err", err) return false, nil default: // Log and retry for other errors. logger.V(3).Info("Deleting ResourceSlice failed, retrying", "fieldSelector", fieldSelector, "err", err) return false, nil } }) } // RegisterPlugin is called when a plugin can be registered. func (h *RegistrationHandler) RegisterPlugin(pluginName string, endpoint string, versions []string, pluginClientTimeout *time.Duration) error { // Prepare a context with its own logger for the plugin. // // The lifecycle of the plugin's background activities is tied to our // root context, so canceling that will also cancel the plugin. // // The logger injects the plugin name as additional value // into all log output related to the plugin. ctx := h.backgroundCtx logger := klog.FromContext(ctx) logger = klog.LoggerWithValues(logger, "pluginName", pluginName) ctx = klog.NewContext(ctx, logger) logger.V(3).Info("Register new DRA plugin", "endpoint", endpoint) highestSupportedVersion, err := h.validateVersions(pluginName, versions) if err != nil { return fmt.Errorf("version check of plugin %s failed: %w", pluginName, err) } var timeout time.Duration if pluginClientTimeout == nil { timeout = defaultClientCallTimeout } else { timeout = *pluginClientTimeout } ctx, cancel := context.WithCancelCause(ctx) pluginInstance := &Plugin{ backgroundCtx:           ctx, cancel:                  cancel, conn:                    nil, endpoint:                endpoint, highestSupportedVersion: highestSupportedVersion, clientCallTimeout:       timeout, } // Storing endpoint of newly registered DRA Plugin into the map, where plugin name will be the key // all other DRA components will be able to get the actual socket of DRA plugins by its name. if draPlugins.add(pluginName, pluginInstance) { logger.V(1).Info("Already registered, previous plugin was replaced") } return nil } func (h *RegistrationHandler) validateVersions( pluginName string, versions []string, ) (*utilversion.Version, error) { if len(versions) == 0 { return nil, errors.New("empty list for supported versions") } // Validate version newPluginHighestVersion, err := utilversion.HighestSupportedVersion(versions) if err != nil { // HighestSupportedVersion includes the list of versions in its error // if relevant, no need to repeat it here. return nil, fmt.Errorf("none of the versions are supported: %w", err) } existingPlugin := draPlugins.get(pluginName) if existingPlugin == nil { return newPluginHighestVersion, nil } if existingPlugin.highestSupportedVersion.LessThan(newPluginHighestVersion) { return newPluginHighestVersion, nil } return nil, fmt.Errorf("another plugin instance is already registered with a higher supported version: %q < %q", newPluginHighestVersion, existingPlugin.highestSupportedVersion) } // DeRegisterPlugin is called when a plugin has removed its socket, // signaling it is no longer available. func (h *RegistrationHandler) DeRegisterPlugin(pluginName string) { if p := draPlugins.delete(pluginName); p != nil { logger := klog.FromContext(p.backgroundCtx) logger.V(3).Info("Deregister DRA plugin", "endpoint", p.endpoint) // Clean up the ResourceSlices for the deleted Plugin since it // may have died without doing so itself and might never come // back. go h.wipeResourceSlices(pluginName) return } logger := klog.FromContext(h.backgroundCtx) logger.V(3).Info("Deregister DRA plugin not necessary, was already removed") } // ValidatePlugin is called by kubelet's plugin watcher upon detection // of a new registration socket opened by DRA plugin. func (h *RegistrationHandler) ValidatePlugin(pluginName string, endpoint string, versions []string) error { _, err := h.validateVersions(pluginName, versions) if err != nil { return fmt.Errorf("invalid versions of plugin %s: %w", pluginName, err) } return err } </ins>


Question:
The PR above moved everything related to into because that was where most of the code already was, but didn't renamed the files to avoid a larger diff and merge conflicts. Once is merged, we can do the renaming. Code clarity. /sig node
/triage accepted /priority important-soon
/assign
a PR can be created for this now.
will do

<ins> /* Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package plugin import ( "testing" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func getFakeNode() (*v1.Node, error) { return &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "worker"}}, nil } func TestRegistrationHandler_ValidatePlugin(t *testing.T) { newRegistrationHandler := func() *RegistrationHandler { return NewRegistrationHandler(nil, getFakeNode) } for _, test := range []struct { description string handler     func() *RegistrationHandler pluginName  string endpoint    string versions    []string shouldError bool }{ { description: "no versions provided", handler:     newRegistrationHandler, shouldError: true, }, { description: "unsupported version", handler:     newRegistrationHandler, versions:    []string{"v2.0.0"}, shouldError: true, }, { description: "plugin already registered with a higher supported version", handler: func() *RegistrationHandler { handler := newRegistrationHandler() if err := handler.RegisterPlugin("this-plugin-already-exists-and-has-a-long-name-so-it-doesnt-collide", "", []string{"v1.1.0"}, nil); err != nil { t.Fatal(err) } return handler }, pluginName:  "this-plugin-already-exists-and-has-a-long-name-so-it-doesnt-collide", versions:    []string{"v1.0.0"}, shouldError: true, }, { description: "should validate the plugin", handler:     newRegistrationHandler, pluginName:  "this-is-a-dummy-plugin-with-a-long-name-so-it-doesnt-collide", versions:    []string{"v1.3.0"}, }, } { t.Run(test.description, func(t *testing.T) { handler := test.handler() err := handler.ValidatePlugin(test.pluginName, test.endpoint, test.versions) if test.shouldError { assert.Error(t, err) } else { assert.NoError(t, err) } }) } t.Cleanup(func() { handler := newRegistrationHandler() handler.DeRegisterPlugin("this-plugin-already-exists-and-has-a-long-name-so-it-doesnt-collide") handler.DeRegisterPlugin("this-is-a-dummy-plugin-with-a-long-name-so-it-doesnt-collide") }) } </ins>


Question:
/kind failing-test <!-- If this is a flake, please add: /kind flake --<!-- Please assign a SIG using: /sig SIG-NAME --
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
looks like started a few days ago ! /sig cluster-lifecycle
cc
fix was sent earlier
/sig cluster-lifecycle
thanks :) i saw it yesterday and forgot to raise the issue. i should have checked the PR queue :)

``` cd kubernetes <del> sudo hack/local-up-cluster.sh </del> <ins> hack/local-up-cluster.sh </ins> ``` This will build and start a lightweight local cluster, consisting of a master


Question:
/kind failing-test <!-- If this is a flake, please add: /kind flake --<!-- Please assign a SIG using: /sig SIG-NAME --
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
looks like started a few days ago ! /sig cluster-lifecycle
cc
fix was sent earlier
/sig cluster-lifecycle
thanks :) i saw it yesterday and forgot to raise the issue. i should have checked the PR queue :)

GO_OUT="${KUBE_ROOT}/_output/local/bin/${host_os}/${host_arch}" APISERVER_LOG=/tmp/kube-apiserver.log <del> "${GO_OUT}/kube-apiserver"  </del> <ins> sudo "${GO_OUT}/kube-apiserver"  </ins> -v=${LOG_LEVEL}  --address="${API_HOST}"  --port="${API_PORT}"


Question:
/kind failing-test <!-- If this is a flake, please add: /kind flake --<!-- Please assign a SIG using: /sig SIG-NAME --
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
looks like started a few days ago ! /sig cluster-lifecycle
cc
fix was sent earlier
/sig cluster-lifecycle
thanks :) i saw it yesterday and forgot to raise the issue. i should have checked the PR queue :)

kube::util::wait_for_url "http://${API_HOST}:${API_PORT}/api/v1beta1/pods" "apiserver: " CTLRMGR_LOG=/tmp/kube-controller-manager.log <del> "${GO_OUT}/kube-controller-manager"  </del> <ins> sudo "${GO_OUT}/kube-controller-manager"  </ins> -v=${LOG_LEVEL}  --machines="127.0.0.1"  --master="${API_HOST}:${API_PORT}" >"${CTLRMGR_LOG}" 2>&1 & CTLRMGR_PID=$! KUBELET_LOG=/tmp/kubelet.log <del> "${GO_OUT}/kubelet"  </del> <ins> sudo "${GO_OUT}/kubelet"  </ins> -v=${LOG_LEVEL}  --etcd_servers="http://127.0.0.1:4001"  --hostname_override="127.0.0.1"


Question:
/kind failing-test <!-- If this is a flake, please add: /kind flake --<!-- Please assign a SIG using: /sig SIG-NAME --
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
looks like started a few days ago ! /sig cluster-lifecycle
cc
fix was sent earlier
/sig cluster-lifecycle
thanks :) i saw it yesterday and forgot to raise the issue. i should have checked the PR queue :)

KUBELET_PID=$! PROXY_LOG=/tmp/kube-proxy.log <del> "${GO_OUT}/kube-proxy"  </del> <ins> sudo "${GO_OUT}/kube-proxy"  </ins> -v=${LOG_LEVEL}  --master="http://${API_HOST}:${API_PORT}" >"${PROXY_LOG}" 2>&1 & PROXY_PID=$! SCHEDULER_LOG=/tmp/kube-scheduler.log <del> "${GO_OUT}/kube-scheduler"  </del> <ins> sudo "${GO_OUT}/kube-scheduler"  </ins> -v=${LOG_LEVEL}  --master="http://${API_HOST}:${API_PORT}" >"${SCHEDULER_LOG}" 2>&1 & SCHEDULER_PID=$!


Question:
/kind failing-test <!-- If this is a flake, please add: /kind flake --<!-- Please assign a SIG using: /sig SIG-NAME --
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
looks like started a few days ago ! /sig cluster-lifecycle
cc
fix was sent earlier
/sig cluster-lifecycle
thanks :) i saw it yesterday and forgot to raise the issue. i should have checked the PR queue :)

cleanup() { echo "Cleaning up..." <del> kill "${APISERVER_PID}" kill "${CTLRMGR_PID}" kill "${KUBELET_PID}" kill "${PROXY_PID}" kill "${SCHEDULER_PID}" </del> <ins> sudo kill "${APISERVER_PID}" sudo kill "${CTLRMGR_PID}" sudo kill "${KUBELET_PID}" sudo kill "${PROXY_PID}" sudo kill "${SCHEDULER_PID}" </ins> kill "${ETCD_PID}" rm -rf "${ETCD_DIR}"


Question:
ci-cgroupv1-containerd-node-arm64-e2e-serial-ec2-eks ci-cgroupv1-containerd-node-e2e-serial-ec2-eks It has been failing consistently for a long time. No response /sig node
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/cc
There's another bug that was filed a long time ago. But there's no update on it and the same test in gce jobs seem to be green now.
cc

} ginkgo.BeforeEach(func() { <ins> // Precautionary check that kubelet is healthy before running the test. waitForKubeletToStart(context.TODO(), f) </ins> ginkgo.By("setting up the pod to be used in the test") e2epod.NewPodClient(f).Create(context.TODO(), testCase.podSpec) })


Question:
ci-cgroupv1-containerd-node-arm64-e2e-serial-ec2-eks ci-cgroupv1-containerd-node-e2e-serial-ec2-eks It has been failing consistently for a long time. No response /sig node
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/cc
There's another bug that was filed a long time ago. But there's no update on it and the same test in gce jobs seem to be green now.
cc

Name: podName, }, Spec: v1.PodSpec{ <del> RestartPolicy: v1.RestartPolicyNever, </del> <ins> PriorityClassName: "system-node-critical", RestartPolicy:     v1.RestartPolicyNever, </ins> Containers: []v1.Container{ createContainer(ctnName), },


Question:
ci-cgroupv1-containerd-node-arm64-e2e-serial-ec2-eks ci-cgroupv1-containerd-node-e2e-serial-ec2-eks It has been failing consistently for a long time. No response /sig node
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/cc
There's another bug that was filed a long time ago. But there's no update on it and the same test in gce jobs seem to be green now.
cc

"sh", "-c", // use the dd tool to attempt to allocate huge block of memory which exceeds the node allocatable <del> "sleep 5 && dd if=/dev/zero of=/dev/null iflag=fullblock count=10 bs=10G", </del> <ins> "sleep 5 && dd if=/dev/zero of=/dev/null iflag=fullblock count=10 bs=1024G", </ins> }, SecurityContext: &v1.SecurityContext{ SeccompProfile: &v1.SeccompProfile{


Question:
ci-kubernetes-unit TestPickBestStrategy shows the jobs are success. but in test-grid and ci report, it shows a failure. After This failed the {Failed;Failed;Failed;Failed;Failed;Failed;Failed; === RUN TestPickBestStrategy === RUN TestPickBestStrategy/singlecandidate,singlepreferredstrategy === RUN TestPickBestStrategy/multiplecandidates,differentpreferredstrategiesshouldfail === RUN TestPickBestStrategy/multiplecandidates,multipleresolvedpreferredstrategy No response_ /sig testing
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/cc
Thanks for opening this Paco, just noticed.
cc
! It is weird as the test passed indeed and I cannot reproduce it.
Fixed in

} if len(sorted) != len(graph) { <del> fmt.Printf("%s", (sorted)) </del> return nil // Cycle detected }


Question:
This issue was supposed to have fixed the issue where endpoint stay out of sync I also wrote a small script which would get me the out of sync endpoints compared to the endpointslices I expect the endpoints to eventually sync and reflect the most upto date information. I have just deployed the newer patch to our cluster and that has resulted in endpoints never ending up being updated if the status goes out of sync. No response Client Version: v1.29.7 Kustomize Version: v5.0.4-0.- Server Version: v1.29.7 <details</detailsalmalinux-9 <details</detailscri-o No response
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig network
from the logs, endpoint controller was retrying test1/test-qa in the expected way. It has retried 5 times and the next retry should be after 320ms, which should long enough for the informer cache to get the updated endpoints. If there is no more logs related to the failure, I think it may have synced it successfully. The python script used to detect the failure seems capturing diff between endpoints and endpointslice at a moment, but that could happen if there is any Pod change in the cluster, and technically the script is comparing endpoints of one moment with endpointslice of another moment. So did you see the endpoints not being synced after endpoint controller stopped retrying it?
Yeah i still see it and see similar messages continuously. Let me post some additional log messages after the one i posted above.
Observed the same issue in 1.29 and 1.30. I keep seeing the same error then new pods were behind a service. And the endpoint never catch up after all pods are settled.
could you provide complete log of kube-controller-manager and the yamls of the related endpoints and endpointslice?
/assign Are you able to triage this since you are already looking at this?
Yes i can get that over by EOD today
Sure, I'm waiting for more logs and the content of the endpoints to understand how it happened. Currently there are only retry logs which is expected in some cases.
We found a customer that triggered an edge case in your change of , which will cause Endpoints never updated. They have a mutatingWebhook on Endpoints object that automatically injects a label, let's say . syncServices, the endpoints controller , and they won't match since Endpoints have the extra "dummy-k: value". endpoints controller will by removing the label. mutatingWebhook will automatically adds back the label as part of the update request. From the APIServer's POV, the Endpoints object don't have any changes, thus the update request will succeed but returns the same resourceVersion as before. change in will mark that resourceVersion as stale. future pods events won't be handled correctly due to the step didn't result in a new resourceVersion. I think we should fix this by check the resulting resourceVersion from the update request, and only mark the old resourceVersion as stale when a new resourceVersion is generated. Not sure whether this is the same case others in this thread encountered :D
thanks for explaining the use case and the analysis. I did analyze the code after the issue was opened and suspected the only possibility is the returned resource version somehow equals to the one in the request but didn't know how it could happen after it has checked the desired endpoints and the current endpoints are different: Now I understand this could happen when the endpoints appear different on client side but are mutated to be same on server side. Do you plan to create a PR since you have proposed a fix? could you confirm if this is also your case, i.e. there is a mutatingWebhook for endpoints objects that could make the endpoints different from what endpoints controller expects?
Yeah, i can do a fix for this
- can we assign to you?
Sure, I'm already assigned. Given that we've had three user reports on this issue, I think it's important to address it as soon as possible. Could you please let me know when you plan to create the PR? If you're currently occupied, I'm happy to handle it and submit the PR.
Sorry, were occupied by some internal incidents last week. Just created a WIP PR for it. Will test it in kops for this specific webhook scenario in the following two days and and remove the WIP flag
/assign

} logger.V(4).Info("Update endpoints", "service", klog.KObj(service), "readyEndpoints", totalReadyEps, "notreadyEndpoints", totalNotReadyEps) <ins> var updatedEndpoints *v1.Endpoints </ins> if createEndpoints { // No previous endpoints, create them _, err = e.client.CoreV1().Endpoints(service.Namespace).Create(ctx, newEndpoints, metav1.CreateOptions{}) } else { // Pre-existing <del> _, err = e.client.CoreV1().Endpoints(service.Namespace).Update(ctx, newEndpoints, metav1.UpdateOptions{}) </del> <ins> updatedEndpoints, err = e.client.CoreV1().Endpoints(service.Namespace).Update(ctx, newEndpoints, metav1.UpdateOptions{}) </ins> } if err != nil { if createEndpoints && errors.IsForbidden(err) {


Question:
This issue was supposed to have fixed the issue where endpoint stay out of sync I also wrote a small script which would get me the out of sync endpoints compared to the endpointslices I expect the endpoints to eventually sync and reflect the most upto date information. I have just deployed the newer patch to our cluster and that has resulted in endpoints never ending up being updated if the status goes out of sync. No response Client Version: v1.29.7 Kustomize Version: v5.0.4-0.- Server Version: v1.29.7 <details</detailsalmalinux-9 <details</detailscri-o No response
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig network
from the logs, endpoint controller was retrying test1/test-qa in the expected way. It has retried 5 times and the next retry should be after 320ms, which should long enough for the informer cache to get the updated endpoints. If there is no more logs related to the failure, I think it may have synced it successfully. The python script used to detect the failure seems capturing diff between endpoints and endpointslice at a moment, but that could happen if there is any Pod change in the cluster, and technically the script is comparing endpoints of one moment with endpointslice of another moment. So did you see the endpoints not being synced after endpoint controller stopped retrying it?
Yeah i still see it and see similar messages continuously. Let me post some additional log messages after the one i posted above.
Observed the same issue in 1.29 and 1.30. I keep seeing the same error then new pods were behind a service. And the endpoint never catch up after all pods are settled.
could you provide complete log of kube-controller-manager and the yamls of the related endpoints and endpointslice?
/assign Are you able to triage this since you are already looking at this?
Yes i can get that over by EOD today
Sure, I'm waiting for more logs and the content of the endpoints to understand how it happened. Currently there are only retry logs which is expected in some cases.
We found a customer that triggered an edge case in your change of , which will cause Endpoints never updated. They have a mutatingWebhook on Endpoints object that automatically injects a label, let's say . syncServices, the endpoints controller , and they won't match since Endpoints have the extra "dummy-k: value". endpoints controller will by removing the label. mutatingWebhook will automatically adds back the label as part of the update request. From the APIServer's POV, the Endpoints object don't have any changes, thus the update request will succeed but returns the same resourceVersion as before. change in will mark that resourceVersion as stale. future pods events won't be handled correctly due to the step didn't result in a new resourceVersion. I think we should fix this by check the resulting resourceVersion from the update request, and only mark the old resourceVersion as stale when a new resourceVersion is generated. Not sure whether this is the same case others in this thread encountered :D
thanks for explaining the use case and the analysis. I did analyze the code after the issue was opened and suspected the only possibility is the returned resource version somehow equals to the one in the request but didn't know how it could happen after it has checked the desired endpoints and the current endpoints are different: Now I understand this could happen when the endpoints appear different on client side but are mutated to be same on server side. Do you plan to create a PR since you have proposed a fix? could you confirm if this is also your case, i.e. there is a mutatingWebhook for endpoints objects that could make the endpoints different from what endpoints controller expects?
Yeah, i can do a fix for this
- can we assign to you?
Sure, I'm already assigned. Given that we've had three user reports on this issue, I think it's important to address it as soon as possible. Could you please let me know when you plan to create the PR? If you're currently occupied, I'm happy to handle it and submit the PR.
Sorry, were occupied by some internal incidents last week. Just created a WIP PR for it. Will test it in kops for this specific webhook scenario in the following two days and and remove the WIP flag
/assign

// If the current endpoints is updated we track the old resource version, so // if we obtain this resource version again from the lister we know is outdated // and we need to retry later to wait for the informer cache to be up-to-date. <del> if !createEndpoints { </del> <ins> // there are some operations (webhooks, truncated endpoints, ...) that can potentially cause endpoints updates became noop // and return the same resourceVersion. // Ref: https://issues.k8s.io/127370 , https://issues.k8s.io/126578 if updatedEndpoints != nil && updatedEndpoints.ResourceVersion != currentEndpoints.ResourceVersion { </ins> e.staleEndpointsTracker.Stale(currentEndpoints) } return nil


Question:
This issue was supposed to have fixed the issue where endpoint stay out of sync I also wrote a small script which would get me the out of sync endpoints compared to the endpointslices I expect the endpoints to eventually sync and reflect the most upto date information. I have just deployed the newer patch to our cluster and that has resulted in endpoints never ending up being updated if the status goes out of sync. No response Client Version: v1.29.7 Kustomize Version: v5.0.4-0.- Server Version: v1.29.7 <details</detailsalmalinux-9 <details</detailscri-o No response
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig network
from the logs, endpoint controller was retrying test1/test-qa in the expected way. It has retried 5 times and the next retry should be after 320ms, which should long enough for the informer cache to get the updated endpoints. If there is no more logs related to the failure, I think it may have synced it successfully. The python script used to detect the failure seems capturing diff between endpoints and endpointslice at a moment, but that could happen if there is any Pod change in the cluster, and technically the script is comparing endpoints of one moment with endpointslice of another moment. So did you see the endpoints not being synced after endpoint controller stopped retrying it?
Yeah i still see it and see similar messages continuously. Let me post some additional log messages after the one i posted above.
Observed the same issue in 1.29 and 1.30. I keep seeing the same error then new pods were behind a service. And the endpoint never catch up after all pods are settled.
could you provide complete log of kube-controller-manager and the yamls of the related endpoints and endpointslice?
/assign Are you able to triage this since you are already looking at this?
Yes i can get that over by EOD today
Sure, I'm waiting for more logs and the content of the endpoints to understand how it happened. Currently there are only retry logs which is expected in some cases.
We found a customer that triggered an edge case in your change of , which will cause Endpoints never updated. They have a mutatingWebhook on Endpoints object that automatically injects a label, let's say . syncServices, the endpoints controller , and they won't match since Endpoints have the extra "dummy-k: value". endpoints controller will by removing the label. mutatingWebhook will automatically adds back the label as part of the update request. From the APIServer's POV, the Endpoints object don't have any changes, thus the update request will succeed but returns the same resourceVersion as before. change in will mark that resourceVersion as stale. future pods events won't be handled correctly due to the step didn't result in a new resourceVersion. I think we should fix this by check the resulting resourceVersion from the update request, and only mark the old resourceVersion as stale when a new resourceVersion is generated. Not sure whether this is the same case others in this thread encountered :D
thanks for explaining the use case and the analysis. I did analyze the code after the issue was opened and suspected the only possibility is the returned resource version somehow equals to the one in the request but didn't know how it could happen after it has checked the desired endpoints and the current endpoints are different: Now I understand this could happen when the endpoints appear different on client side but are mutated to be same on server side. Do you plan to create a PR since you have proposed a fix? could you confirm if this is also your case, i.e. there is a mutatingWebhook for endpoints objects that could make the endpoints different from what endpoints controller expects?
Yeah, i can do a fix for this
- can we assign to you?
Sure, I'm already assigned. Given that we've had three user reports on this issue, I think it's important to address it as soon as possible. Could you please let me know when you plan to create the PR? If you're currently occupied, I'm happy to handle it and submit the PR.
Sorry, were occupied by some internal incidents last week. Just created a WIP PR for it. Will test it in kops for this specific webhook scenario in the following two days and and remove the WIP flag
/assign

apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" <ins> "k8s.io/apimachinery/pkg/util/sets" </ins> "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes"


Question:
This issue was supposed to have fixed the issue where endpoint stay out of sync I also wrote a small script which would get me the out of sync endpoints compared to the endpointslices I expect the endpoints to eventually sync and reflect the most upto date information. I have just deployed the newer patch to our cluster and that has resulted in endpoints never ending up being updated if the status goes out of sync. No response Client Version: v1.29.7 Kustomize Version: v5.0.4-0.- Server Version: v1.29.7 <details</detailsalmalinux-9 <details</detailscri-o No response
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig network
from the logs, endpoint controller was retrying test1/test-qa in the expected way. It has retried 5 times and the next retry should be after 320ms, which should long enough for the informer cache to get the updated endpoints. If there is no more logs related to the failure, I think it may have synced it successfully. The python script used to detect the failure seems capturing diff between endpoints and endpointslice at a moment, but that could happen if there is any Pod change in the cluster, and technically the script is comparing endpoints of one moment with endpointslice of another moment. So did you see the endpoints not being synced after endpoint controller stopped retrying it?
Yeah i still see it and see similar messages continuously. Let me post some additional log messages after the one i posted above.
Observed the same issue in 1.29 and 1.30. I keep seeing the same error then new pods were behind a service. And the endpoint never catch up after all pods are settled.
could you provide complete log of kube-controller-manager and the yamls of the related endpoints and endpointslice?
/assign Are you able to triage this since you are already looking at this?
Yes i can get that over by EOD today
Sure, I'm waiting for more logs and the content of the endpoints to understand how it happened. Currently there are only retry logs which is expected in some cases.
We found a customer that triggered an edge case in your change of , which will cause Endpoints never updated. They have a mutatingWebhook on Endpoints object that automatically injects a label, let's say . syncServices, the endpoints controller , and they won't match since Endpoints have the extra "dummy-k: value". endpoints controller will by removing the label. mutatingWebhook will automatically adds back the label as part of the update request. From the APIServer's POV, the Endpoints object don't have any changes, thus the update request will succeed but returns the same resourceVersion as before. change in will mark that resourceVersion as stale. future pods events won't be handled correctly due to the step didn't result in a new resourceVersion. I think we should fix this by check the resulting resourceVersion from the update request, and only mark the old resourceVersion as stale when a new resourceVersion is generated. Not sure whether this is the same case others in this thread encountered :D
thanks for explaining the use case and the analysis. I did analyze the code after the issue was opened and suspected the only possibility is the returned resource version somehow equals to the one in the request but didn't know how it could happen after it has checked the desired endpoints and the current endpoints are different: Now I understand this could happen when the endpoints appear different on client side but are mutated to be same on server side. Do you plan to create a PR since you have proposed a fix? could you confirm if this is also your case, i.e. there is a mutatingWebhook for endpoints objects that could make the endpoints different from what endpoints controller expects?
Yeah, i can do a fix for this
- can we assign to you?
Sure, I'm already assigned. Given that we've had three user reports on this issue, I think it's important to address it as soon as possible. Could you please let me know when you plan to create the PR? If you're currently occupied, I'm happy to handle it and submit the PR.
Sorry, were occupied by some internal incidents last week. Just created a WIP PR for it. Will test it in kops for this specific webhook scenario in the following two days and and remove the WIP flag
/assign

"k8s.io/kubernetes/pkg/controller/endpoint" "k8s.io/kubernetes/test/integration/framework" "k8s.io/kubernetes/test/utils/ktesting" <ins> netutils "k8s.io/utils/net" </ins> ) func TestEndpointUpdates(t *testing.T) {


Question:
This issue was supposed to have fixed the issue where endpoint stay out of sync I also wrote a small script which would get me the out of sync endpoints compared to the endpointslices I expect the endpoints to eventually sync and reflect the most upto date information. I have just deployed the newer patch to our cluster and that has resulted in endpoints never ending up being updated if the status goes out of sync. No response Client Version: v1.29.7 Kustomize Version: v5.0.4-0.- Server Version: v1.29.7 <details</detailsalmalinux-9 <details</detailscri-o No response
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig network
from the logs, endpoint controller was retrying test1/test-qa in the expected way. It has retried 5 times and the next retry should be after 320ms, which should long enough for the informer cache to get the updated endpoints. If there is no more logs related to the failure, I think it may have synced it successfully. The python script used to detect the failure seems capturing diff between endpoints and endpointslice at a moment, but that could happen if there is any Pod change in the cluster, and technically the script is comparing endpoints of one moment with endpointslice of another moment. So did you see the endpoints not being synced after endpoint controller stopped retrying it?
Yeah i still see it and see similar messages continuously. Let me post some additional log messages after the one i posted above.
Observed the same issue in 1.29 and 1.30. I keep seeing the same error then new pods were behind a service. And the endpoint never catch up after all pods are settled.
could you provide complete log of kube-controller-manager and the yamls of the related endpoints and endpointslice?
/assign Are you able to triage this since you are already looking at this?
Yes i can get that over by EOD today
Sure, I'm waiting for more logs and the content of the endpoints to understand how it happened. Currently there are only retry logs which is expected in some cases.
We found a customer that triggered an edge case in your change of , which will cause Endpoints never updated. They have a mutatingWebhook on Endpoints object that automatically injects a label, let's say . syncServices, the endpoints controller , and they won't match since Endpoints have the extra "dummy-k: value". endpoints controller will by removing the label. mutatingWebhook will automatically adds back the label as part of the update request. From the APIServer's POV, the Endpoints object don't have any changes, thus the update request will succeed but returns the same resourceVersion as before. change in will mark that resourceVersion as stale. future pods events won't be handled correctly due to the step didn't result in a new resourceVersion. I think we should fix this by check the resulting resourceVersion from the update request, and only mark the old resourceVersion as stale when a new resourceVersion is generated. Not sure whether this is the same case others in this thread encountered :D
thanks for explaining the use case and the analysis. I did analyze the code after the issue was opened and suspected the only possibility is the returned resource version somehow equals to the one in the request but didn't know how it could happen after it has checked the desired endpoints and the current endpoints are different: Now I understand this could happen when the endpoints appear different on client side but are mutated to be same on server side. Do you plan to create a PR since you have proposed a fix? could you confirm if this is also your case, i.e. there is a mutatingWebhook for endpoints objects that could make the endpoints different from what endpoints controller expects?
Yeah, i can do a fix for this
- can we assign to you?
Sure, I'm already assigned. Given that we've had three user reports on this issue, I think it's important to address it as soon as possible. Could you please let me know when you plan to create the PR? If you're currently occupied, I'm happy to handle it and submit the PR.
Sorry, were occupied by some internal incidents last week. Just created a WIP PR for it. Will test it in kops for this specific webhook scenario in the following two days and and remove the WIP flag
/assign

svc.Spec.ExternalName = "google.com" return svc } <ins> func TestEndpointTruncate(t *testing.T) { // Disable ServiceAccount admission plugin as we don't have serviceaccount controller running. server := kubeapiservertesting.StartTestServerOrDie(t, nil, framework.DefaultTestServerFlags(), framework.SharedEtcd()) defer server.TearDownFn() client, err := clientset.NewForConfig(server.ClientConfig) if err != nil { t.Fatalf("Error creating clientset: %v", err) } informers := informers.NewSharedInformerFactory(client, 0) tCtx := ktesting.Init(t) epController := endpoint.NewEndpointController( tCtx, informers.Core().V1().Pods(), informers.Core().V1().Services(), informers.Core().V1().Endpoints(), client, 0) // Start informer and controllers informers.Start(tCtx.Done()) go epController.Run(tCtx, 1) // Create namespace ns := framework.CreateNamespaceOrDie(client, "test-endpoints-truncate", t) defer framework.DeleteNamespaceOrDie(client, ns, t) // Create a pod with labels basePod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:   "test-pod", Labels: labelMap(), }, Spec: v1.PodSpec{ NodeName: "fake-node", Containers: []v1.Container{ { Name:  "fakename", Image: "fakeimage", Ports: []v1.ContainerPort{ { Name:          "port-443", ContainerPort: 443, }, }, }, }, }, Status: v1.PodStatus{ Phase: v1.PodRunning, Conditions: []v1.PodCondition{ { Type:   v1.PodReady, Status: v1.ConditionTrue, }, }, PodIP: "10.0.0.1", PodIPs: []v1.PodIP{ { IP: "10.0.0.1", }, }, }, } // create 1001 Pods to reach endpoint max capacity that is set to 1000 allPodNames := sets.New[string]() baseIP := netutils.BigForIP(netutils.ParseIPSloppy("10.0.0.1")) for i := 0; i < 1001; i++ { pod := basePod.DeepCopy() pod.Name = fmt.Sprintf("%s-%d", basePod.Name, i) allPodNames.Insert(pod.Name) podIP := netutils.AddIPOffset(baseIP, i).String() pod.Status.PodIP = podIP pod.Status.PodIPs[0] = v1.PodIP{IP: podIP} createdPod, err := client.CoreV1().Pods(ns.Name).Create(tCtx, pod, metav1.CreateOptions{}) if err != nil { t.Fatalf("Failed to create pod %s: %v", pod.Name, err) } createdPod.Status = pod.Status _, err = client.CoreV1().Pods(ns.Name).UpdateStatus(tCtx, createdPod, metav1.UpdateOptions{}) if err != nil { t.Fatalf("Failed to update status of pod %s: %v", pod.Name, err) } } // Create a service associated to the pod svc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name:      "test-service", Namespace: ns.Name, Labels: map[string]string{ "foo": "bar", }, }, Spec: v1.ServiceSpec{ Selector: map[string]string{ "foo": "bar", }, Ports: []v1.ServicePort{ {Name: "port-443", Port: 443, Protocol: "TCP", TargetPort: intstr.FromInt32(443)}, }, }, } _, err = client.CoreV1().Services(ns.Name).Create(tCtx, svc, metav1.CreateOptions{}) if err != nil { t.Fatalf("Failed to create service %s: %v", svc.Name, err) } var truncatedPodName string // poll until associated Endpoints to the previously created Service exists if err := wait.PollUntilContextTimeout(tCtx, 1*time.Second, 10*time.Second, true, func(context.Context) (bool, error) { podNames := sets.New[string]() endpoints, err := client.CoreV1().Endpoints(ns.Name).Get(tCtx, svc.Name, metav1.GetOptions{}) if err != nil { return false, nil } for _, subset := range endpoints.Subsets { for _, address := range subset.Addresses { podNames.Insert(address.TargetRef.Name) } } if podNames.Len() != 1000 { return false, nil } truncated, ok := endpoints.Annotations[v1.EndpointsOverCapacity] if !ok || truncated != "truncated" { return false, nil } // There is only 1 truncated Pod. truncatedPodName, _ = allPodNames.Difference(podNames).PopAny() return true, nil }); err != nil { t.Fatalf("endpoints not found: %v", err) } // Update the truncated Pod several times to make endpoints controller resync the service. truncatedPod, err := client.CoreV1().Pods(ns.Name).Get(tCtx, truncatedPodName, metav1.GetOptions{}) if err != nil { t.Fatalf("Failed to get pod %s: %v", truncatedPodName, err) } for i := 0; i < 10; i++ { truncatedPod.Status.Conditions[0].Status = v1.ConditionFalse truncatedPod, err = client.CoreV1().Pods(ns.Name).UpdateStatus(tCtx, truncatedPod, metav1.UpdateOptions{}) if err != nil { t.Fatalf("Failed to update status of pod %s: %v", truncatedPod.Name, err) } truncatedPod.Status.Conditions[0].Status = v1.ConditionTrue truncatedPod, err = client.CoreV1().Pods(ns.Name).UpdateStatus(tCtx, truncatedPod, metav1.UpdateOptions{}) if err != nil { t.Fatalf("Failed to update status of pod %s: %v", truncatedPod.Name, err) } } // delete 501 Pods for i := 500; i < 1001; i++ { podName := fmt.Sprintf("%s-%d", basePod.Name, i) err = client.CoreV1().Pods(ns.Name).Delete(tCtx, podName, metav1.DeleteOptions{}) if err != nil { t.Fatalf("error deleting test pod: %v", err) } } // poll until endpoints for deleted Pod are no longer in Endpoints. if err := wait.PollUntilContextTimeout(tCtx, 1*time.Second, 10*time.Second, true, func(context.Context) (bool, error) { endpoints, err := client.CoreV1().Endpoints(ns.Name).Get(tCtx, svc.Name, metav1.GetOptions{}) if err != nil { return false, nil } numEndpoints := 0 for _, subset := range endpoints.Subsets { numEndpoints += len(subset.Addresses) } if numEndpoints != 500 { return false, nil } truncated, ok := endpoints.Annotations[v1.EndpointsOverCapacity] if ok || truncated == "truncated" { return false, nil } return true, nil }); err != nil { t.Fatalf("error checking for no endpoints with terminating pods: %v", err) } } </ins>


Question:
seen in this inside a like loop we execute a function that return a bool and an error. If we want to fail fast we return the error directly and assert on the output of the wait function. If we want to retry we log the error and return false nil /kind flake /kind bug /sig testing /sig network
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/cc since I think you this test but most probable the code is inherited and is a copy paste thing, but besides that, the change is to remove those there
/assign

// our target metric should be updated by now if err := wait.PollUntilContextTimeout(ctx, 10*time.Second, 2*time.Minute, true, func(_ context.Context) (bool, error) { metrics, err := metricsGrabber.GrabFromKubeProxy(ctx, nodeName) <del> framework.ExpectNoError(err) </del> <ins> if err != nil { return false, fmt.Errorf("failed to fetch metrics: %w", err) } </ins> targetMetricAfter, err := metrics.GetCounterMetricValue(metricName) <del> framework.ExpectNoError(err) </del> <ins> if err != nil { return false, fmt.Errorf("failed to fetch metric: %w", err) } </ins> return targetMetricAfter > targetMetricBefore, nil }); err != nil { <del> framework.Failf("expected %s metric to be updated after accessing endpoints via localhost nodeports", metricName) </del> <ins> if wait.Interrupted(err) { framework.Failf("expected %s metric to be updated after accessing endpoints via localhost nodeports", metricName) } framework.ExpectNoError(err) </ins> } }) })


Question:
The code says this (): While the comment says this (): So..., what happened? The feature is still Alpha or it is a Beta? Be consistent about the features we add or change. Read the code. No response 30.0 <details</details<details</details<details</details<details</details<details</details
According to the changelog: This feature has been demoted from Beta to Alpha. That is fine. We need to revise the code comment then.
/sig api-machinery (not sure who owns it per se)
not worry, i think this can be resolved with
I found same like this inconsistency between the comment and the actual feature gate definition for APIServingWithRoutine in due to this
I want to raise PR on this if need another PR separately
Looks like this comment should be updated to alpha based on . Thanks for catching. if you'd like to raise a PR to fix the comment that'd be great!
/triage accepted
sure I will raise

APIServerTracing featuregate.Feature = "APIServerTracing" // owner: @linxiulei <del> // beta: v1.30 </del> <ins> // alpha: v1.30 </ins> // // Enables serving watch requests in separate goroutines. APIServingWithRoutine featuregate.Feature = "APIServingWithRoutine"


Question:
k8s component developers are using FQDNs as finalizers (e.g. or which triggers an error message that isn't particularly easy for an end user to understand: To a naive end user, a fully qualified domain name is a pretty safe thing that is unlikely to conflict with anything else (as long as it's only used by someone from within the domain that owns it, which is a reasonable assumption). If the finalizer has s inside it but no , the message should suggest that finalizers need a . Something like: Do whatever it is that triggered the warning in (this is probably simpler than installing cert-manager and asking for a certificate) introduced: <details</details<detailsGKE </details<details</details<details</details<details</details<details</details
/sig api-machinery
If thats ok I would like to work on that.
/assign
/triage accepted

} if len(strings.Split(stringValue, "/")) == 1 { if !standardFinalizers.Has(stringValue) { <del> allWarnings = append(allWarnings, fmt.Sprintf("%s: %q: prefer a domain-qualified finalizer name to avoid accidental conflicts with other finalizer writers", fldPath.String(), stringValue)) </del> <ins> if strings.Contains(stringValue, ".") { allWarnings = append(allWarnings, fmt.Sprintf("%s: %q: prefer a domain-qualified finalizer name including a path (/) to avoid accidental conflicts with other finalizer writers", fldPath.String(), stringValue)) } else { allWarnings = append(allWarnings, fmt.Sprintf("%s: %q: prefer a domain-qualified finalizer name to avoid accidental conflicts with other finalizer writers", fldPath.String(), stringValue)) } </ins> } } return allWarnings


Question:
k8s component developers are using FQDNs as finalizers (e.g. or which triggers an error message that isn't particularly easy for an end user to understand: To a naive end user, a fully qualified domain name is a pretty safe thing that is unlikely to conflict with anything else (as long as it's only used by someone from within the domain that owns it, which is a reasonable assumption). If the finalizer has s inside it but no , the message should suggest that finalizers need a . Something like: Do whatever it is that triggered the warning in (this is probably simpler than installing cert-manager and asking for a certificate) introduced: <details</details<detailsGKE </details<details</details<details</details<details</details<details</details
/sig api-machinery
If thats ok I would like to work on that.
/assign
/triage accepted

finalizer: []string{"kubernetes.io/valid-finalizer"}, }, { <ins> name:      "create-crd-with-fqdn-like-finalizer-without-path", finalizer: []string{"finalizer.without.valid-path.io"}, expectCreateWarnings: []string{ `metadata.finalizers: "finalizer.without.valid-path.io": prefer a domain-qualified finalizer name including a path (/) to avoid accidental conflicts with other finalizer writers`, }, }, { </ins> name:             "update-crd-with-invalid-finalizer", finalizer:        []string{"invalid-finalizer"}, updatedFinalizer: []string{"another-invalid-finalizer"},


Question:
[ ] two weeks soak end date : xxxx-xx-xx According to following APIsnoop query, there are six CSINode endpoints that are untested. Using a number of existing e2e test practices a new has been created to provide future Conformance coverage for the 7 endpoints. The e2e logs for this test are listed below. This query shows the following CSINode endpoints are hit within a short period of running this e2e test. If a test with these calls gets merged, test coverage will go up by 6 points This test is also created with the goal of conformance promotion. /sig testing /sig architecture /area conformance
/triage accepted

<ins> /* Copyright 2024 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package storage import ( "context" "fmt" "time" storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" utilrand "k8s.io/apimachinery/pkg/util/rand" "k8s.io/client-go/util/retry" "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/storage/utils" "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" ) var _ = utils.SIGDescribe("CSINodes", func() { f := framework.NewDefaultFramework("csinodes") ginkgo.Describe("CSI Conformance", func() { ginkgo.It("should run through the lifecycle of a csinode", func(ctx context.Context) { csiNodeClient := f.ClientSet.StorageV1().CSINodes() initialCSINode := storagev1.CSINode{ ObjectMeta: metav1.ObjectMeta{ Name: "e2e-csinode-" + utilrand.String(5), }, } ginkgo.By(fmt.Sprintf("Creating initial csiNode %q", initialCSINode.Name)) csiNode, err := csiNodeClient.Create(ctx, &initialCSINode, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create csiNode %q", initialCSINode.Name) ginkgo.By(fmt.Sprintf("Getting initial csiNode %q", initialCSINode.Name)) retrievedCSINode, err := csiNodeClient.Get(ctx, initialCSINode.Name, metav1.GetOptions{}) framework.ExpectNoError(err, "Failed to retrieve csiNode %q", initialCSINode.Name) gomega.Expect(retrievedCSINode.Name).To(gomega.Equal(csiNode.Name), "Checking that the retrieved name has been found") ginkgo.By(fmt.Sprintf("Patching initial csiNode: %q", initialCSINode.Name)) payload := "{"metadata":{"labels":{"" + csiNode.Name + "":"patched"}}}" patchedCSINode, err := csiNodeClient.Patch(ctx, csiNode.Name, types.StrategicMergePatchType, []byte(payload), metav1.PatchOptions{}) framework.ExpectNoError(err, "Failed to patch csiNode %q", csiNode.Name) gomega.Expect(patchedCSINode.Labels).To(gomega.HaveKeyWithValue(csiNode.Name, "patched"), "Checking that patched label has been applied") patchedSelector := labels.Set{csiNode.Name: "patched"}.AsSelector().String() ginkgo.By(fmt.Sprintf("Listing csiNodes with LabelSelector %q", patchedSelector)) csiNodeList, err := csiNodeClient.List(ctx, metav1.ListOptions{LabelSelector: patchedSelector}) framework.ExpectNoError(err, "failed to list csiNodes") gomega.Expect(csiNodeList.Items).To(gomega.HaveLen(1)) ginkgo.By(fmt.Sprintf("Delete initial csiNode: %q", initialCSINode.Name)) err = csiNodeClient.Delete(ctx, csiNode.Name, metav1.DeleteOptions{}) framework.ExpectNoError(err, "failed to delete csiNode %q", initialCSINode.Name) ginkgo.By(fmt.Sprintf("Confirm deletion of csiNode %q", initialCSINode.Name)) type state struct { CSINodes []storagev1.CSINode } err = framework.Gomega().Eventually(ctx, framework.HandleRetry(func(ctx context.Context) (*state, error) { csiNodeList, err := csiNodeClient.List(ctx, metav1.ListOptions{LabelSelector: patchedSelector}) if err != nil { return nil, fmt.Errorf("failed to list CSINode: %w", err) } return &state{ CSINodes: csiNodeList.Items, }, nil })).WithTimeout(30 * time.Second).Should(framework.MakeMatcher(func(s *state) (func() string, error) { if len(s.CSINodes) == 0 { return nil, nil } return func() string { return fmt.Sprintf("Expected CSINode to be deleted, found %q", s.CSINodes[0].Name) }, nil })) framework.ExpectNoError(err, "Timeout while waiting to confirm CSINode %q deletion", initialCSINode.Name) replacementCSINode := storagev1.CSINode{ ObjectMeta: metav1.ObjectMeta{ Name: "e2e-csinode-" + utilrand.String(5), }, } ginkgo.By(fmt.Sprintf("Creating replacement csiNode %q", replacementCSINode.Name)) secondCSINode, err := csiNodeClient.Create(ctx, &replacementCSINode, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create csiNode %q", replacementCSINode.Name) ginkgo.By(fmt.Sprintf("Getting replacement csiNode %q", replacementCSINode.Name)) retrievedCSINode, err = csiNodeClient.Get(ctx, secondCSINode.Name, metav1.GetOptions{}) framework.ExpectNoError(err, "Failed to retrieve CSINode %q", replacementCSINode.Name) gomega.Expect(retrievedCSINode.Name).To(gomega.Equal(secondCSINode.Name), "Checking that the retrieved name has been found") ginkgo.By(fmt.Sprintf("Updating replacement csiNode %q", retrievedCSINode.Name)) var updatedCSINode *storagev1.CSINode err = retry.RetryOnConflict(retry.DefaultRetry, func() error { tmpCSINode, err := csiNodeClient.Get(ctx, retrievedCSINode.Name, metav1.GetOptions{}) framework.ExpectNoError(err, "Unable to get %q", replacementCSINode.Name) tmpCSINode.Labels = map[string]string{replacementCSINode.Name: "updated"} updatedCSINode, err = csiNodeClient.Update(ctx, tmpCSINode, metav1.UpdateOptions{}) return err }) framework.ExpectNoError(err, "failed to update %q", replacementCSINode.Name) gomega.Expect(updatedCSINode.Labels).To(gomega.HaveKeyWithValue(secondCSINode.Name, "updated"), "Checking that updated label has been applied") updatedSelector := labels.Set{retrievedCSINode.Name: "updated"}.AsSelector().String() ginkgo.By(fmt.Sprintf("DeleteCollection of CSINodes with %q label", updatedSelector)) err = csiNodeClient.DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: updatedSelector}) framework.ExpectNoError(err, "failed to delete csiNode Colllection") ginkgo.By(fmt.Sprintf("Confirm deletion of replacement csiNode with LabelSelector %q", updatedSelector)) err = framework.Gomega().Eventually(ctx, framework.HandleRetry(func(ctx context.Context) (*state, error) { csiNodeList, err := csiNodeClient.List(ctx, metav1.ListOptions{LabelSelector: updatedSelector}) if err != nil { return nil, fmt.Errorf("failed to list CSINode: %w", err) } return &state{ CSINodes: csiNodeList.Items, }, nil })).WithTimeout(30 * time.Second).Should(framework.MakeMatcher(func(s *state) (func() string, error) { if len(s.CSINodes) == 0 { return nil, nil } return func() string { return fmt.Sprintf("Expected CSINode to be deleted, found %q", s.CSINodes[0].Name) }, nil })) framework.ExpectNoError(err, "Timeout while waiting to confirm CSINode %q deletion", replacementCSINode.Name) }) }) }) </ins>