} } } <ins> const listTypeResourceSchema = ` apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: foos.test spec: group: test names: kind: Foo listKind: FooList plural: foos singular: foo scope: Cluster versions: - name: v1 schema: openAPIV3Schema: type: object properties: numArray: type: array x-kubernetes-list-type: set items: type: object served: true storage: true - name: v2 schema: openAPIV3Schema: type: object properties: numArray2: type: array ` func TestStatusStrategyValidateUpdate(t *testing.T) { crdV1 := &apiextensionsv1.CustomResourceDefinition{} err := yaml.Unmarshal([]byte(listTypeResourceSchema), &crdV1) if err != nil { t.Fatalf("unexpected decoding error: %v", err) } t.Logf("crd v1 details: %v", crdV1) crd := &apiextensions.CustomResourceDefinition{} if err = apiextensionsv1.Convert_v1_CustomResourceDefinition_To_apiextensions_CustomResourceDefinition(crdV1, crd, nil); err != nil { t.Fatalf("unexpected convert error: %v", err) } t.Logf("crd details: %v", crd) strategy := statusStrategy{} kind := schema.GroupVersionKind{ Version: crd.Spec.Versions[0].Name, Kind:    crd.Spec.Names.Kind, Group:   crd.Spec.Group, } strategy.customResourceStrategy.validator.kind = kind ss, _ := structuralschema.NewStructural(crd.Spec.Versions[0].Schema.OpenAPIV3Schema) strategy.structuralSchemas = map[string]*structuralschema.Structural{ crd.Spec.Versions[0].Name: ss, } ctx := context.TODO() tcs := []struct { name    string old     *unstructured.Unstructured obj     *unstructured.Unstructured isValid bool }{ { name:    "bothValid", old:     &unstructured.Unstructured{Object: map[string]interface{}{"apiVersion": "test/v1", "kind": "Foo", "numArray": []interface{}{1, 2}}}, obj:     &unstructured.Unstructured{Object: map[string]interface{}{"apiVersion": "test/v1", "kind": "Foo", "numArray": []interface{}{1, 3}, "metadata": map[string]interface{}{"resourceVersion": "1"}}}, isValid: true, }, { name:    "change to invalid", old:     &unstructured.Unstructured{Object: map[string]interface{}{"apiVersion": "test/v1", "kind": "Foo", "spec": "old", "numArray": []interface{}{1, 2}}}, obj:     &unstructured.Unstructured{Object: map[string]interface{}{"apiVersion": "test/v1", "kind": "Foo", "spec": "new", "numArray": []interface{}{1, 1}, "metadata": map[string]interface{}{"resourceVersion": "1"}}}, isValid: false, }, { name:    "change to valid", old:     &unstructured.Unstructured{Object: map[string]interface{}{"apiVersion": "test/v1", "kind": "Foo", "spec": "new", "numArray": []interface{}{1, 1}}}, obj:     &unstructured.Unstructured{Object: map[string]interface{}{"apiVersion": "test/v1", "kind": "Foo", "spec": "old", "numArray": []interface{}{1, 2}, "metadata": map[string]interface{}{"resourceVersion": "1"}}}, isValid: true, }, { name:    "keeps invalid", old:     &unstructured.Unstructured{Object: map[string]interface{}{"apiVersion": "test/v1", "kind": "Foo", "spec": "new", "numArray": []interface{}{1, 1}}}, obj:     &unstructured.Unstructured{Object: map[string]interface{}{"apiVersion": "test/v1", "kind": "Foo", "spec": "old", "numArray": []interface{}{1, 1}, "metadata": map[string]interface{}{"resourceVersion": "1"}}}, isValid: true, }, } for _, tc := range tcs { errs := strategy.ValidateUpdate(ctx, tc.obj, tc.old) if tc.isValid && len(errs) > 0 { t.Errorf("%v: unexpected error: %v", tc.name, errs) } if !tc.isValid && len(errs) == 0 { t.Errorf("%v: unexpected non-error", tc.name) } } } </ins>


concurrencyLimit:            6, evalDuration:                time.Second * 20, expectedFair:                []bool{true}, <del> expectedFairnessMargin:      []float64{0.15}, </del> <ins> expectedFairnessMargin:      []float64{0.155}, </ins> expectAllRequests:           true, evalInqueueMetrics:          true, evalExecutingMetrics:        true,


"k8s.io/kubernetes/pkg/features" kevents "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/pkg/util/goroutinemap/exponentialbackoff" <ins> nodeutil "k8s.io/kubernetes/pkg/util/node" </ins> "k8s.io/kubernetes/pkg/util/taints" "k8s.io/kubernetes/pkg/volume/util" "k8s.io/kubernetes/pkg/volume/util/operationexecutor"


return false, nil } <ins> // nodeIsHealthy returns true if the node looks healthy. func (rc *reconciler) nodeIsHealthy(nodeName types.NodeName) (bool, error) { node, err := rc.nodeLister.Get(string(nodeName)) if err != nil { return false, err } return nodeutil.IsNodeReady(node), nil } </ins> func (rc *reconciler) reconcile() { // Detaches are triggered before attaches so that volumes referenced by // pods that are rescheduled to a different node are detached first.


// Check whether timeout has reached the maximum waiting time timeout := elapsedTime > rc.maxWaitForUnmountDuration <ins> isHealthy, err := rc.nodeIsHealthy(attachedVolume.NodeName) if err != nil { klog.Errorf("failed to get health of node %s: %s", attachedVolume.NodeName, err.Error()) } // Force detach volumes from unhealthy nodes after maxWaitForUnmountDuration. forceDetach := !isHealthy && timeout </ins> hasOutOfServiceTaint, err := rc.hasOutOfServiceTaint(attachedVolume.NodeName) if err != nil { klog.Errorf("failed to get taint specs for node %s: %s", attachedVolume.NodeName, err.Error()) } <del> // Check whether volume is still mounted. Skip detach if it is still mounted unless timeout </del> <ins> // Check whether volume is still mounted. Skip detach if it is still mounted unless force detach timeout </ins> // or the node has `node.kubernetes.io/out-of-service` taint. <del> if attachedVolume.MountedByNode && !timeout && !hasOutOfServiceTaint { </del> <ins> if attachedVolume.MountedByNode && !forceDetach && !hasOutOfServiceTaint { </ins> klog.V(5).InfoS("Cannot detach volume because it is still mounted", "volume", attachedVolume) continue }


volumeSpec := controllervolumetesting.GetTestVolumeSpec(string(volumeName), volumeName) nodeName := k8stypes.NodeName("node-name") dsw.AddNode(nodeName, false /*keepTerminatedPodVolumes*/) volumeExists := dsw.VolumeExists(volumeName, nodeName) if volumeExists { t.Fatalf(


waitForDetachCallCount(t, 0 /* expectedDetachCallCount */, fakePlugin) } <ins> // Populates desiredStateOfWorld cache with one node/volume/pod tuple. // The node starts as healthy. // // Calls Run() // Verifies there is one attach call and no detach calls. // Deletes the pod from desiredStateOfWorld cache without first marking the node/volume as unmounted. // Verifies that the volume is NOT detached after maxWaitForUnmountDuration. // Marks the node as unhealthy. // Verifies that the volume is detached after maxWaitForUnmountDuration. func Test_Run_OneVolumeDetachOnUnhealthyNode(t *testing.T) { // Arrange volumePluginMgr, fakePlugin := volumetesting.GetTestVolumePluginMgr(t) dsw := cache.NewDesiredStateOfWorld(volumePluginMgr) asw := cache.NewActualStateOfWorld(volumePluginMgr) fakeKubeClient := controllervolumetesting.CreateTestClient() fakeRecorder := &record.FakeRecorder{} fakeHandler := volumetesting.NewBlockVolumePathHandler() ad := operationexecutor.NewOperationExecutor(operationexecutor.NewOperationGenerator( fakeKubeClient, volumePluginMgr, fakeRecorder, fakeHandler)) informerFactory := informers.NewSharedInformerFactory(fakeKubeClient, controller.NoResyncPeriodFunc()) nsu := statusupdater.NewFakeNodeStatusUpdater(false /* returnError */) nodeLister := informerFactory.Core().V1().Nodes().Lister() reconciler := NewReconciler( reconcilerLoopPeriod, maxWaitForUnmountDuration, syncLoopPeriod, false, dsw, asw, ad, nsu, nodeLister, fakeRecorder) podName1 := "pod-uid1" volumeName1 := v1.UniqueVolumeName("volume-name1") volumeSpec1 := controllervolumetesting.GetTestVolumeSpec(string(volumeName1), volumeName1) nodeName1 := k8stypes.NodeName("worker-0") node1 := &v1.Node{ ObjectMeta: metav1.ObjectMeta{Name: string(nodeName1)}, Status: v1.NodeStatus{ Conditions: []v1.NodeCondition{ { Type:   v1.NodeReady, Status: v1.ConditionTrue, }, }, }, } informerFactory.Core().V1().Nodes().Informer().GetStore().Add(node1) dsw.AddNode(nodeName1, false /*keepTerminatedPodVolumes*/) volumeExists := dsw.VolumeExists(volumeName1, nodeName1) if volumeExists { t.Fatalf( "Volume %q/node %q should not exist, but it does.", volumeName1, nodeName1) } generatedVolumeName, podErr := dsw.AddPod(types.UniquePodName(podName1), controllervolumetesting.NewPod(podName1, podName1), volumeSpec1, nodeName1) if podErr != nil { t.Fatalf("AddPod failed. Expected: <no error> Actual: <%v>", podErr) } // Act ch := make(chan struct{}) go reconciler.Run(ch) defer close(ch) // Assert waitForNewAttacherCallCount(t, 1 /* expectedCallCount */, fakePlugin) verifyNewAttacherCallCount(t, false /* expectZeroNewAttacherCallCount */, fakePlugin) waitForAttachCallCount(t, 1 /* expectedAttachCallCount */, fakePlugin) verifyNewDetacherCallCount(t, true /* expectZeroNewDetacherCallCount */, fakePlugin) waitForDetachCallCount(t, 0 /* expectedDetachCallCount */, fakePlugin) // Act // Delete the pod and the volume will be detached even after the maxWaitForUnmountDuration expires as volume is // not unmounted and the node is healthy. dsw.DeletePod(types.UniquePodName(podName1), generatedVolumeName, nodeName1) time.Sleep(maxWaitForUnmountDuration * 5) // Assert waitForNewDetacherCallCount(t, 0 /* expectedCallCount */, fakePlugin) verifyNewAttacherCallCount(t, false /* expectZeroNewAttacherCallCount */, fakePlugin) waitForAttachCallCount(t, 1 /* expectedAttachCallCount */, fakePlugin) verifyNewDetacherCallCount(t, true /* expectZeroNewDetacherCallCount */, fakePlugin) waitForDetachCallCount(t, 0 /* expectedDetachCallCount */, fakePlugin) // Act // Mark the node unhealthy node2 := node1.DeepCopy() node2.Status.Conditions[0].Status = v1.ConditionFalse informerFactory.Core().V1().Nodes().Informer().GetStore().Update(node2) // Assert -- Detach was triggered after maxWaitForUnmountDuration waitForNewDetacherCallCount(t, 1 /* expectedCallCount */, fakePlugin) verifyNewAttacherCallCount(t, false /* expectZeroNewAttacherCallCount */, fakePlugin) waitForAttachCallCount(t, 1 /* expectedAttachCallCount */, fakePlugin) verifyNewDetacherCallCount(t, false /* expectZeroNewDetacherCallCount */, fakePlugin) waitForDetachCallCount(t, 1 /* expectedDetachCallCount */, fakePlugin) } </ins> func Test_ReportMultiAttachError(t *testing.T) { type nodeWithPods struct { name     k8stypes.NodeName


"k8s.io/kubernetes/pkg/apis/scheduling" kubelettypes "k8s.io/kubernetes/pkg/kubelet/types" "k8s.io/kubernetes/test/e2e/framework" <del> e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" </del> imageutils "k8s.io/kubernetes/test/utils/image" "github.com/onsi/ginkgo"


f := framework.NewDefaultFramework("critical-pod-test") ginkgo.Context("when we need to admit a critical pod", func() { ginkgo.It("[Flaky] should be able to create and delete a critical pod", func() { <del> configEnabled, err := isKubeletConfigEnabled(f) framework.ExpectNoError(err) if !configEnabled { e2eskipper.Skipf("unable to run test without dynamic kubelet config enabled.") } </del> // because adminssion Priority enable, If the priority class is not found, the Pod is rejected. node := getNodeName(f) // Define test pods


// Add each PodStats to the result. result := make([]statsapi.PodStats, 0, len(podToStats)) for _, podStats := range podToStats { <del> // Lookup the volume stats for each pod. podUID := types.UID(podStats.PodRef.UID) var ephemeralStats []statsapi.VolumeStats if vstats, found := p.resourceAnalyzer.GetPodVolumeStats(podUID); found { ephemeralStats = make([]statsapi.VolumeStats, len(vstats.EphemeralVolumes)) copy(ephemeralStats, vstats.EphemeralVolumes) podStats.VolumeStats = append(append([]statsapi.VolumeStats{}, vstats.EphemeralVolumes...), vstats.PersistentVolumes...) } logStats, err := p.hostStatsProvider.getPodLogStats(podStats.PodRef.Namespace, podStats.PodRef.Name, podUID, &rootFsInfo) if err != nil { klog.ErrorS(err, "Unable to fetch pod log stats", "pod", klog.KRef(podStats.PodRef.Namespace, podStats.PodRef.Name)) } etcHostsStats, err := p.hostStatsProvider.getPodEtcHostsStats(podUID, &rootFsInfo) if err != nil { klog.ErrorS(err, "Unable to fetch pod etc hosts stats", "pod", klog.KRef(podStats.PodRef.Namespace, podStats.PodRef.Name)) } </del> <ins> makePodStorageStats(podStats, &rootFsInfo, p.resourceAnalyzer, p.hostStatsProvider, false) </ins> <del> podStats.EphemeralStorage = calcEphemeralStorage(podStats.Containers, ephemeralStats, &rootFsInfo, logStats, etcHostsStats, false) </del> <ins> podUID := types.UID(podStats.PodRef.UID) </ins> // Lookup the pod-level cgroup's CPU and memory stats podInfo := getCadvisorPodInfoFromPodUID(podUID, allInfos) if podInfo != nil {


result := make([]statsapi.PodStats, 0, len(sandboxIDToPodStats)) for _, s := range sandboxIDToPodStats { <del> p.makePodStorageStats(s, rootFsInfo) </del> <ins> makePodStorageStats(s, rootFsInfo, p.resourceAnalyzer, p.hostStatsProvider, true) </ins> result = append(result, *s) } return result, nil


addCRIPodCPUStats(ps, criSandboxStat) addCRIPodMemoryStats(ps, criSandboxStat) addCRIPodProcessStats(ps, criSandboxStat) <del> p.makePodStorageStats(ps, rootFsInfo) </del> <ins> makePodStorageStats(ps, rootFsInfo, p.resourceAnalyzer, p.hostStatsProvider, true) </ins> summarySandboxStats = append(summarySandboxStats, *ps) } return summarySandboxStats, nil


} } <del> func (p *criStatsProvider) makePodStorageStats(s *statsapi.PodStats, rootFsInfo *cadvisorapiv2.FsInfo) { podNs := s.PodRef.Namespace podName := s.PodRef.Name podUID := types.UID(s.PodRef.UID) vstats, found := p.resourceAnalyzer.GetPodVolumeStats(podUID) if !found { return } logStats, err := p.hostStatsProvider.getPodLogStats(podNs, podName, podUID, rootFsInfo) if err != nil { klog.ErrorS(err, "Unable to fetch pod log stats", "pod", klog.KRef(podNs, podName)) // If people do in-place upgrade, there might be pods still using // the old log path. For those pods, no pod log stats is returned. // We should continue generating other stats in that case. // calcEphemeralStorage tolerants logStats == nil. } etcHostsStats, err := p.hostStatsProvider.getPodEtcHostsStats(podUID, rootFsInfo) if err != nil { klog.ErrorS(err, "Unable to fetch pod etc hosts stats", "pod", klog.KRef(podNs, podName)) } ephemeralStats := make([]statsapi.VolumeStats, len(vstats.EphemeralVolumes)) copy(ephemeralStats, vstats.EphemeralVolumes) s.VolumeStats = append(append([]statsapi.VolumeStats{}, vstats.EphemeralVolumes...), vstats.PersistentVolumes...) s.EphemeralStorage = calcEphemeralStorage(s.Containers, ephemeralStats, rootFsInfo, logStats, etcHostsStats, true) } </del> func (p *criStatsProvider) addPodNetworkStats( ps *statsapi.PodStats, podSandboxID string,


cadvisorapiv1 "github.com/google/cadvisor/info/v1" cadvisorapiv2 "github.com/google/cadvisor/info/v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" <ins> "k8s.io/apimachinery/pkg/types" </ins> "k8s.io/klog/v2" statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1" "k8s.io/kubernetes/pkg/kubelet/cadvisor" <ins> "k8s.io/kubernetes/pkg/kubelet/server/stats" </ins> ) // defaultNetworkInterfaceName is used for collectng network stats.


total := *first + *second return &total } <ins> func makePodStorageStats(s *statsapi.PodStats, rootFsInfo *cadvisorapiv2.FsInfo, resourceAnalyzer stats.ResourceAnalyzer, hostStatsProvider HostStatsProvider, isCRIStatsProvider bool) { podNs := s.PodRef.Namespace podName := s.PodRef.Name podUID := types.UID(s.PodRef.UID) var ephemeralStats []statsapi.VolumeStats if vstats, found := resourceAnalyzer.GetPodVolumeStats(podUID); found { ephemeralStats = make([]statsapi.VolumeStats, len(vstats.EphemeralVolumes)) copy(ephemeralStats, vstats.EphemeralVolumes) s.VolumeStats = append(append([]statsapi.VolumeStats{}, vstats.EphemeralVolumes...), vstats.PersistentVolumes...) } logStats, err := hostStatsProvider.getPodLogStats(podNs, podName, podUID, rootFsInfo) if err != nil { klog.V(6).ErrorS(err, "Unable to fetch pod log stats", "pod", klog.KRef(podNs, podName)) // If people do in-place upgrade, there might be pods still using // the old log path. For those pods, no pod log stats is returned. // We should continue generating other stats in that case. // calcEphemeralStorage tolerants logStats == nil. } etcHostsStats, err := hostStatsProvider.getPodEtcHostsStats(podUID, rootFsInfo) if err != nil { klog.V(6).ErrorS(err, "Unable to fetch pod etc hosts stats", "pod", klog.KRef(podNs, podName)) } s.EphemeralStorage = calcEphemeralStorage(s.Containers, ephemeralStats, rootFsInfo, logStats, etcHostsStats, isCRIStatsProvider) } </ins>


"syscall" "time" <del> "k8s.io/apimachinery/pkg/apis/meta/v1" </del> <ins> v1 "k8s.io/apimachinery/pkg/apis/meta/v1" </ins> statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1" )


func Stats() (*statsapi.RlimitStats, error) { rlimit := &statsapi.RlimitStats{} <del> if content, err := ioutil.ReadFile("/proc/sys/kernel/pid_max"); err == nil { if maxPid, err := strconv.ParseInt(string(content[:len(content)-1]), 10, 64); err == nil { rlimit.MaxPID = &maxPid </del> <ins> taskMax := int64(-1) // Calculate the mininum of kernel.pid_max and kernel.threads-max as they both specify the // system-wide limit on the number of tasks. for _, file := range []string{"/proc/sys/kernel/pid_max", "/proc/sys/kernel/threads-max"} { if content, err := ioutil.ReadFile(file); err == nil { if limit, err := strconv.ParseInt(string(content[:len(content)-1]), 10, 64); err == nil { if taskMax == -1 || taskMax > limit { taskMax = limit } } </ins> } } <ins> // Both reads did not fail. if taskMax >= 0 { rlimit.MaxPID = &taskMax } </ins> var info syscall.Sysinfo_t syscall.Sysinfo(&info)


type RlimitStats struct { Time metav1.Time `json:"time"` <del> // The max PID of OS. </del> <ins> // The max number of extant process (threads, precisely on Linux) of OS. See RLIMIT_NPROC in getrlimit(2). // The operating system ceiling on the number of process IDs that can be assigned. // On Linux, tasks (either processes or threads) consume 1 PID each. </ins> MaxPID *int64 `json:"maxpid,omitempty"` <del> // The number of running process in the OS. </del> <ins> // The number of running process (threads, precisely on Linux) in the OS. </ins> NumOfRunningProcesses *int64 `json:"curproc,omitempty"` }


containerList, _ := runtimeService.ListContainers(nil) for _, c := range containerList { if _, exists := podSandboxMap[c.PodSandboxId]; !exists { <del> return nil, fmt.Errorf("no PodsandBox found with Id '%s'", c.PodSandboxId) </del> <ins> return nil, fmt.Errorf("no PodsandBox found with Id '%s' for container with ID '%s' and Name '%s'", c.PodSandboxId, c.Id, c.Metadata.Name) </ins> } containerMap.Add(podSandboxMap[c.PodSandboxId], c.Metadata.Name, c.Id) }


if err != nil { klog.ErrorS(err, "Failed to get node ip address matching nodeport cidrs, services with nodeport may not work as intended", "CIDRs", proxier.nodePortAddresses) } <ins> // nodeAddresses may contain dual-stack zero-CIDRs if proxier.nodePortAddresses is empty. // Ensure nodeAddresses only contains the addresses for this proxier's IP family. isIPv6 := proxier.iptables.IsIPv6() for addr := range nodeAddresses { if utilproxy.IsZeroCIDR(addr) && isIPv6 == netutils.IsIPv6CIDRString(addr) { // if any of the addresses is zero cidr of this IP family, non-zero IPs can be excluded. nodeAddresses = sets.NewString(addr) break } } </ins> // Build rules for each service. for svcName, svc := range proxier.serviceMap {


// Finally, tail-call to the nodeports chain.  This needs to be after all // other service portal rules. <del> isIPv6 := proxier.iptables.IsIPv6() </del> for address := range nodeAddresses { // TODO(thockin, m1093782566): If/when we have dual-stack support we will want to distinguish v4 from v6 zero-CIDRs. if utilproxy.IsZeroCIDR(address) {


-A KUBE-SERVICES -m comment --comment "kubernetes service nodeports; NOTE: this must be the last rule in this chain" -m addrtype --dst-type LOCAL -j KUBE-NODEPORTS COMMIT ` <ins> assert.Equal(t, []*netutils.LocalPort{ { Description: "nodePort for ns1/svc1:p80", IP:          "", IPFamily:    netutils.IPv4, Port:        svcNodePort, Protocol:    netutils.TCP, }, }, fp.portMapper.(*fakePortOpener).openPorts) </ins> assertIPTablesRulesEqual(t, expected, fp.iptablesData.String()) }


run(ctx, startSATokenController, initializersFunc) }, OnStoppedLeading: func() { <del> klog.Fatalf("leaderelection lost") </del> <ins> klog.ErrorS(nil, "leaderelection lost") klog.FlushAndExit(klog.ExitFlushTimeout, 1) </ins> }, })


run(ctx, nil, createInitializersFunc(leaderMigrator.FilterFunc, leadermigration.ControllerMigrated)) }, OnStoppedLeading: func() { <del> klog.Fatalf("migration leaderelection lost") </del> <ins> klog.ErrorS(nil, "migration leaderelection lost") klog.FlushAndExit(klog.ExitFlushTimeout, 1) </ins> }, }) }


default: // We lost the lock. klog.ErrorS(nil, "Leaderelection lost") <del> os.Exit(1) </del> <ins> klog.FlushAndExit(klog.ExitFlushTimeout, 1) </ins> } }, }


run(ctx, initializers) }, OnStoppedLeading: func() { <del> klog.Fatalf("leaderelection lost") </del> <ins> klog.ErrorS(nil, "leaderelection lost") klog.FlushAndExit(klog.ExitFlushTimeout, 1) </ins> }, })


run(ctx, filterInitializers(controllerInitializers, leaderMigrator.FilterFunc, leadermigration.ControllerMigrated)) }, OnStoppedLeading: func() { <del> klog.Fatalf("migration leaderelection lost") </del> <ins> klog.ErrorS(nil, "migration leaderelection lost") klog.FlushAndExit(klog.ExitFlushTimeout, 1) </ins> }, }) }


"crypto/x509" "encoding/json" "fmt" <del> "io" </del> "io/ioutil" "net" "net/http" "net/http/httptest" "net/http/httptrace" <del> "os" </del> "reflect" "strings" "sync"


res.Body.Close() } <del> func captureStdErr() (func() string, func(), error) { </del> <ins> func TestErrConnKilled(t *testing.T) { </ins> var buf bytes.Buffer <del> reader, writer, err := os.Pipe() if err != nil { return nil, nil, err } stderr := os.Stderr readerClosedCh := make(chan struct{}) stopReadingStdErr := func() string { writer.Close() <-readerClosedCh return buf.String() } klog.LogToStderr(true) cleanUp := func() { os.Stderr = stderr klog.LogToStderr(false) stopReadingStdErr() } os.Stderr = writer go func() { io.Copy(&buf, reader) readerClosedCh <- struct{}{} close(readerClosedCh) }() klog.LogToStderr(true) </del> <ins> klog.SetOutput(&buf) klog.LogToStderr(false) defer klog.LogToStderr(true) </ins> <del> return stopReadingStdErr, cleanUp, nil } func TestErrConnKilled(t *testing.T) { readStdErr, cleanUp, err := captureStdErr() if err != nil { t.Fatalf("unable to setup the test, err %v", err) } defer cleanUp() </del> handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // this error must be ignored by the WithPanicRecovery handler // it is thrown by WithTimeoutForNonLongRunningRequests handler when a response has been already sent to the client and the handler timed out


ts := httptest.NewServer(WithPanicRecovery(handler, resolver)) defer ts.Close() <del> _, err = http.Get(ts.URL) </del> <ins> _, err := http.Get(ts.URL) </ins> if err == nil { t.Fatal("expected to receive an error") } <del> capturedOutput := readStdErr() </del> <ins> klog.Flush() klog.SetOutput(&bytes.Buffer{}) // prevent further writes into buf capturedOutput := buf.String() </ins> // We don't expect stack trace from the panic to be included in the log. if isStackTraceLoggedByRuntime(capturedOutput) {


// TestErrConnKilledHTTP2 check if HTTP/2 connection is not closed when an HTTP handler panics // The net/http library recovers the panic and sends an HTTP/2 RST_STREAM. func TestErrConnKilledHTTP2(t *testing.T) { <del> readStdErr, cleanUp, err := captureStdErr() if err != nil { t.Fatalf("unable to setup the test, err %v", err) } defer cleanUp() </del> <ins> var buf bytes.Buffer klog.SetOutput(&buf) klog.LogToStderr(false) defer klog.LogToStderr(true) </ins> handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // this error must be ignored by the WithPanicRecovery handler // it is thrown by WithTimeoutForNonLongRunningRequests handler when a response has been already sent to the client and the handler timed out


t.Fatal("expected to receive an error") } <del> capturedOutput := readStdErr() </del> <ins> klog.Flush() klog.SetOutput(&bytes.Buffer{}) // prevent further writes into buf capturedOutput := buf.String() </ins> // We don't expect stack trace from the panic to be included in the log. if isStackTraceLoggedByRuntime(capturedOutput) {


ttlSeconds = int(v1.DefaultClientIPServiceAffinitySeconds) //default to 3 hours if not specified.  Should 0 be unlimited instead???? } <del> if _, exists := lb.services[svcPort]; !exists { </del> <ins> if state, exists := lb.services[svcPort]; !exists || state == nil { </ins> lb.services[svcPort] = &balancerState{affinity: *newAffinityPolicy(affinityType, ttlSeconds)} klog.V(4).InfoS("LoadBalancerRR service does not exist, created", "servicePortName", svcPort) } else if affinityType != "" {


lb.lock.RLock() defer lb.lock.RUnlock() state, exists := lb.services[svcPort] <del> // TODO: while nothing ever assigns nil to the map, *some* of the code using the map // checks for it.  The code should all follow the same convention. return exists && state != nil && len(state.endpoints) > 0 </del> <ins> if !exists || state == nil { return false } return len(state.endpoints) > 0 </ins> } // NextEndpoint returns a service endpoint.


} state, exists := lb.services[svcPort] <del> if !exists { </del> <ins> if !exists || state == nil { </ins> return } for _, existingEndpoint := range state.endpoints {


func (lb *LoadBalancerRR) resetService(svcPort proxy.ServicePortName) { // If the service is still around, reset but don't delete. <del> if state, ok := lb.services[svcPort]; ok { </del> <ins> if state, ok := lb.services[svcPort]; ok && state != nil { </ins> if len(state.endpoints) > 0 { klog.V(2).InfoS("LoadBalancerRR: Removing endpoints service", "servicePortName", svcPort) state.endpoints = []string{}


defer lb.lock.Unlock() state, exists := lb.services[svcPort] <del> if !exists { </del> <ins> if !exists || state == nil { </ins> return } for ip, affinity := range state.affinity.affinityMap {


return err } <ins> // Drop EndpointSlices that have been marked for deletion to prevent the controller from getting stuck. endpointSlices = dropEndpointSlicesPendingDeletion(endpointSlices) </ins> if c.endpointSliceTracker.StaleSlices(service, endpointSlices) { return endpointsliceutil.NewStaleInformerCache("EndpointSlice informer cache is out of date") }


} endpointslicemetrics.EndpointSliceSyncs.WithLabelValues(metricLabel).Inc() } <ins> func dropEndpointSlicesPendingDeletion(endpointSlices []*discovery.EndpointSlice) []*discovery.EndpointSlice { n := 0 for _, endpointSlice := range endpointSlices { if endpointSlice.DeletionTimestamp == nil { endpointSlices[n] = endpointSlice n++ } } return endpointSlices[:n] } </ins>


assert.EqualValues(t, endpoint.TargetRef, &v1.ObjectReference{Kind: "Pod", Namespace: ns, Name: pod1.Name}) } <ins> func TestSyncServiceEndpointSlicePendingDeletion(t *testing.T) { client, esController := newController([]string{"node-1"}, time.Duration(0)) ns := metav1.NamespaceDefault serviceName := "testing-1" service := createService(t, esController, ns, serviceName) err := esController.syncService(fmt.Sprintf("%s/%s", ns, serviceName)) assert.Nil(t, err, "Expected no error syncing service") gvk := schema.GroupVersionKind{Version: "v1", Kind: "Service"} ownerRef := metav1.NewControllerRef(service, gvk) deletedTs := metav1.Now() endpointSlice := &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name:            "epSlice-1", Namespace:       ns, OwnerReferences: []metav1.OwnerReference{*ownerRef}, Labels: map[string]string{ discovery.LabelServiceName: serviceName, discovery.LabelManagedBy:   controllerName, }, DeletionTimestamp: &deletedTs, }, AddressType: discovery.AddressTypeIPv4, } err = esController.endpointSliceStore.Add(endpointSlice) if err != nil { t.Fatalf("Expected no error adding EndpointSlice: %v", err) } _, err = client.DiscoveryV1().EndpointSlices(ns).Create(context.TODO(), endpointSlice, metav1.CreateOptions{}) if err != nil { t.Fatalf("Expected no error creating EndpointSlice: %v", err) } numActionsBefore := len(client.Actions()) err = esController.syncService(fmt.Sprintf("%s/%s", ns, serviceName)) assert.Nil(t, err, "Expected no error syncing service") // The EndpointSlice marked for deletion should be ignored by the controller, and thus // should not result in more than one action from the client (an update to the non-terminating // EndpointSlice removing the trigger time annotation.) if len(client.Actions()) != numActionsBefore+1 { t.Errorf("Expected 1 more action, got %d", len(client.Actions())-numActionsBefore) } } </ins> // Ensure SyncService correctly selects and labels EndpointSlices. func TestSyncServiceEndpointSliceLabelSelection(t *testing.T) { client, esController := newController([]string{"node-1"}, time.Duration(0))


} } } <ins> func Test_dropEndpointSlicesPendingDeletion(t *testing.T) { now := metav1.Now() endpointSlices := []*discovery.EndpointSlice{ { ObjectMeta: metav1.ObjectMeta{ Name:              "epSlice1", DeletionTimestamp: &now, }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "epSlice2", }, AddressType: discovery.AddressTypeIPv4, Endpoints: []discovery.Endpoint{ { Addresses: []string{"172.18.0.2"}, }, }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "epSlice3", }, AddressType: discovery.AddressTypeIPv6, Endpoints: []discovery.Endpoint{ { Addresses: []string{"3001:0da8:75a3:0000:0000:8a2e:0370:7334"}, }, }, }, } epSlice2 := endpointSlices[1] epSlice3 := endpointSlices[2] result := dropEndpointSlicesPendingDeletion(endpointSlices) assert.Len(t, result, 2) for _, epSlice := range result { if epSlice.Name == "epSlice1" { t.Errorf("Expected EndpointSlice marked for deletion to be dropped.") } } // We don't use endpointSlices and instead check manually for equality, because // `dropEndpointSlicesPendingDeletion` mutates the slice it receives, so it's easy // to break this test later. This way, we can be absolutely sure that the result // has exactly what we expect it to. if !reflect.DeepEqual(epSlice2, result[0]) { t.Errorf("EndpointSlice was unexpectedly mutated. Expected: %+v, Mutated: %+v", epSlice2, result[0]) } if !reflect.DeepEqual(epSlice3, result[1]) { t.Errorf("EndpointSlice was unexpectedly mutated. Expected: %+v, Mutated: %+v", epSlice3, result[1]) } } </ins>


// 1. One or more of the provided EndpointSlices have older generations than the //    corresponding tracked ones. // 2. The tracker is expecting one or more of the provided EndpointSlices to be <del> //    deleted. </del> <ins> //    deleted. (EndpointSlices that have already been marked for deletion are ignored here.) </ins> // 3. The tracker is tracking EndpointSlices that have not been provided. func (est *EndpointSliceTracker) StaleSlices(service *v1.Service, endpointSlices []*discovery.EndpointSlice) bool { est.lock.Lock()


epSlice1NewerGen := epSlice1.DeepCopy() epSlice1NewerGen.Generation = 2 <ins> epTerminatingSlice := epSlice1.DeepCopy() now := metav1.Now() epTerminatingSlice.DeletionTimestamp = &now </ins> testCases := []struct { name         string tracker      *EndpointSliceTracker


"context" "fmt" "os" <del> "os/exec" "path" </del> "regexp" "strings" "time" <ins> "github.com/onsi/ginkgo" "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" </ins> v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" <ins> e2epod "k8s.io/kubernetes/test/e2e/framework/pod" </ins> e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" imageutils "k8s.io/kubernetes/test/utils/image" <del> "github.com/onsi/ginkgo" "github.com/onsi/gomega" </del> ) const (


gmsaCustomResourceName = "gmsa-e2e" // gmsaWebhookDeployScriptURL is the URL of the deploy script for the GMSA webook <del> // TODO(wk8): we should pin versions. </del> gmsaWebhookDeployScriptURL = "https://raw.githubusercontent.com/kubernetes-sigs/windows-gmsa/master/admission-webhook/deploy/deploy-gmsa-webhook.sh" // output from the nltest /query command should have this in it


ginkgo.By("retrieving the contents of the GMSACredentialSpec custom resource manifest from the node") crdManifestContents := retrieveCRDManifestFileContents(f, node) <del> ginkgo.By("downloading the GMSA webhook deploy script") deployScriptPath, err := downloadFile(gmsaWebhookDeployScriptURL) defer func() { os.Remove(deployScriptPath) }() if err != nil { framework.Failf(err.Error()) } </del> ginkgo.By("deploying the GMSA webhook") <del> webhookCleanUp, err := deployGmsaWebhook(f, deployScriptPath) </del> <ins> webhookCleanUp, err := deployGmsaWebhook(f) </ins> defer webhookCleanUp() if err != nil { framework.Failf(err.Error())


} return strings.Contains(output, "This is a test file.") }, 1*time.Minute, 1*time.Second).Should(gomega.BeTrue()) }) }) })


// deployGmsaWebhook deploys the GMSA webhook, and returns a cleanup function // to be called when done with testing, that removes the temp files it's created // on disks as well as the API resources it's created. <del> func deployGmsaWebhook(f *framework.Framework, deployScriptPath string) (func(), error) { cleanUpFunc := func() {} </del> <ins> func deployGmsaWebhook(f *framework.Framework) (func(), error) { deployerName := "webhook-deployer" deployerNamespace := f.Namespace.Name webHookName := "gmsa-webhook" webHookNamespace := deployerNamespace + "-webhook" </ins> <del> tempDir, err := os.MkdirTemp("", "") if err != nil { return cleanUpFunc, fmt.Errorf("unable to create temp dir: %w", err) } </del> <ins> // regardless of whether the deployment succeeded, let's do a best effort at cleanup cleanUpFunc := func() { framework.Logf("Best effort clean up of the webhook:n") stdout, err := framework.RunKubectl("", "delete", "CustomResourceDefinition", "gmsacredentialspecs.windows.k8s.io") framework.Logf("stdout:%snerror:%s", stdout, err) </ins> <del> manifestsFile := path.Join(tempDir, "manifests.yml") name := "gmsa-webhook" namespace := f.Namespace.Name + "-webhook" certsDir := path.Join(tempDir, "certs") </del> <ins> stdout, err = framework.RunKubectl("", "delete", "CertificateSigningRequest", fmt.Sprintf("%s.%s", webHookName, webHookNamespace)) framework.Logf("stdout:%snerror:%s", stdout, err) </ins> <del> // regardless of whether the deployment succeeded, let's do a best effort at cleanup cleanUpFunc = func() { framework.RunKubectl(f.Namespace.Name, "delete", "--filename", manifestsFile) framework.RunKubectl(f.Namespace.Name, "delete", "CustomResourceDefinition", "gmsacredentialspecs.windows.k8s.io") framework.RunKubectl(f.Namespace.Name, "delete", "CertificateSigningRequest", fmt.Sprintf("%s.%s", name, namespace)) os.RemoveAll(tempDir) </del> <ins> stdout, err = runKubectlExecInNamespace(deployerNamespace, deployerName, "--", "kubectl", "delete", "-f", "/manifests.yml") framework.Logf("stdout:%snerror:%s", stdout, err) } // ensure the deployer has ability to approve certificatesigningrequests to install the webhook s := createServiceAccount(f) bindClusterRBACRoleToServiceAccount(f, s, "cluster-admin") installSteps := []string{ "echo "@testing http://dl-cdn.alpinelinux.org/alpine/edge/testing/" >> /etc/apk/repositories", "&& apk add kubectl@testing gettext openssl", "&& apk add --update coreutils", fmt.Sprintf("&& curl %s > gmsa.sh", gmsaWebhookDeployScriptURL), "&& chmod +x gmsa.sh", fmt.Sprintf("&& ./gmsa.sh --file %s --name %s --namespace %s --certs-dir %s --tolerate-master", "/manifests.yml", webHookName, webHookNamespace, "certs"), "&& /agnhost pause", </ins> } <ins> installCommand := strings.Join(installSteps, " ") </ins> <del> cmd := exec.Command("bash", deployScriptPath, "--file", manifestsFile, "--name", name, "--namespace", namespace, "--certs-dir", certsDir, "--tolerate-master") </del> <ins> pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:      deployerName, Namespace: deployerNamespace, }, Spec: v1.PodSpec{ ServiceAccountName: s, NodeSelector: map[string]string{ "kubernetes.io/os": "linux", }, Containers: []v1.Container{ { Name:    deployerName, Image:   imageutils.GetE2EImage(imageutils.Agnhost), Command: []string{"bash", "-c"}, Args:    []string{installCommand}, }, }, Tolerations: []v1.Toleration{ { Operator: v1.TolerationOpExists, Effect:   v1.TaintEffectNoSchedule, }, }, }, } f.PodClient().CreateSync(pod) </ins> <del> output, err := cmd.CombinedOutput() </del> <ins> // Wait for the Webhook deployment to become ready. The deployer pod takes a few seconds to initialize and create resources err := waitForDeployment(func() (*appsv1.Deployment, error) { return f.ClientSet.AppsV1().Deployments(webHookNamespace).Get(context.TODO(), webHookName, metav1.GetOptions{}) }, 10*time.Second, f.Timeouts.PodStart) </ins> if err == nil { <del> framework.Logf("GMSA webhook successfully deployed, output:n%s", string(output)) </del> <ins> framework.Logf("GMSA webhook successfully deployed") </ins> } else { <del> err = fmt.Errorf("unable to deploy GMSA webhook, output:n%s: %w", string(output), err) </del> <ins> err = fmt.Errorf("GMSA webhook did not become ready: %w", err) </ins> } <ins> // Dump deployer logs logs, _ := e2epod.GetPodLogs(f.ClientSet, deployerNamespace, deployerName, deployerName) framework.Logf("GMSA deployment logs:n%s", logs) </ins> return cleanUpFunc, err }


// createServiceAccount creates a service account, and returns its name. func createServiceAccount(f *framework.Framework) string { <del> accountName := f.Namespace.Name + "-sa" </del> <ins> accountName := f.Namespace.Name + "-sa-" + string(uuid.NewUUID()) </ins> account := &v1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name:      accountName,


f.ClientSet.RbacV1().RoleBindings(f.Namespace.Name).Create(context.TODO(), binding, metav1.CreateOptions{}) } <ins> func bindClusterRBACRoleToServiceAccount(f *framework.Framework, serviceAccountName, rbacRoleName string) { binding := &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ Name:      f.Namespace.Name + "-rbac-binding", Namespace: f.Namespace.Name, }, Subjects: []rbacv1.Subject{ { Kind:      "ServiceAccount", Name:      serviceAccountName, Namespace: f.Namespace.Name, }, }, RoleRef: rbacv1.RoleRef{ APIGroup: "rbac.authorization.k8s.io", Kind:     "ClusterRole", Name:     rbacRoleName, }, } f.ClientSet.RbacV1().ClusterRoleBindings().Create(context.TODO(), binding, metav1.CreateOptions{}) } </ins> // createPodWithGmsa creates a pod using the test GMSA cred spec, and returns its name. func createPodWithGmsa(f *framework.Framework, serviceAccountName string) string { podName := "pod-with-gmsa"


package windows import ( <del> "fmt" "io" "net/http" "os" </del> <ins> "time" appsv1 "k8s.io/api/apps/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/kubernetes/pkg/controller/deployment/util" "k8s.io/kubernetes/test/e2e/framework" </ins> ) <del> // downloadFile saves a remote URL to a local temp file, and returns its path. // It's the caller's responsibility to clean up the temp file when done. func downloadFile(url string) (string, error) { response, err := http.Get(url) if err != nil { return "", fmt.Errorf("unable to download from %q: %w", url, err) } defer response.Body.Close() tempFile, err := os.CreateTemp("", "") if err != nil { return "", fmt.Errorf("unable to create temp file: %w", err) } defer tempFile.Close() _, err = io.Copy(tempFile, response.Body) return tempFile.Name(), err </del> <ins> // waits for a deployment to be created and the desired replicas // are updated and available, and no old pods are running. func waitForDeployment(getDeploymentFunc func() (*appsv1.Deployment, error), interval, timeout time.Duration) error { return wait.PollImmediate(interval, timeout, func() (bool, error) { deployment, err := getDeploymentFunc() if err != nil { if apierrors.IsNotFound(err) { framework.Logf("deployment not found, continue waiting: %s", err) return false, nil } framework.Logf("error while deploying, error %s", err) return false, err } framework.Logf("deployment status %s", &deployment.Status) return util.DeploymentComplete(deployment, &deployment.Status), nil }) </ins> }


if err != nil { return currentRevision, updateRevision, currentStatus, err } <del> // update the set's status err = ssc.updateStatefulSetStatus(ctx, set, currentStatus) if err != nil { return currentRevision, updateRevision, currentStatus, err } klog.V(4).InfoS("StatefulSet pod status", "statefulSet", klog.KObj(set), "replicas", currentStatus.Replicas, "readyReplicas", currentStatus.ReadyReplicas, "currentReplicas", currentStatus.CurrentReplicas, "updatedReplicas", currentStatus.UpdatedReplicas) </del> klog.V(4).InfoS("StatefulSet revisions", "statefulSet", klog.KObj(set), "currentRevision", currentStatus.CurrentRevision,


currentRevision *apps.ControllerRevision, updateRevision *apps.ControllerRevision, collisionCount int32, <del> pods []*v1.Pod) (*apps.StatefulSetStatus, error) { </del> <ins> pods []*v1.Pod) (statefulSetStatus *apps.StatefulSetStatus, updateErr error) { </ins> // get the current and update revisions of the set. currentSet, err := ApplyRevision(set, currentRevision) if err != nil {


// If the ordinal could not be parsed (ord < 0), ignore the Pod. } <ins> // make sure to update the latest status even if there is an error later defer func() { // update the set's status statusErr := ssc.updateStatefulSetStatus(ctx, set, &status) if statusErr == nil { klog.V(4).InfoS("Updated status", "statefulSet", klog.KObj(set), "replicas", status.Replicas, "readyReplicas", status.ReadyReplicas, "currentReplicas", status.CurrentReplicas, "updatedReplicas", status.UpdatedReplicas) } else if updateErr == nil { updateErr = statusErr } else { klog.V(4).InfoS("Could not update status", "statefulSet", klog.KObj(set), "err", statusErr) } }() </ins> // for any empty indices in the sequence [0,set.Spec.Replicas) create a new Pod at the correct revision for ord := 0; ord < replicaCount; ord++ { if replicas[ord] == nil {


"k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/dynamic" <ins> "k8s.io/client-go/informers" </ins> clientset "k8s.io/client-go/kubernetes" <ins> restclient "k8s.io/client-go/rest" </ins> featuregatetesting "k8s.io/component-base/featuregate/testing" apiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" podutil "k8s.io/kubernetes/pkg/api/v1/pod" <ins> "k8s.io/kubernetes/pkg/controller/statefulset" </ins> "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/test/integration/framework" )


t.Fatalf("failed to mark all StatefulSet pods to ready: %v", err) } } <ins> // add for issue: https://github.com/kubernetes/kubernetes/issues/108837 func TestStatefulSetStatusWithPodFail(t *testing.T) { limitedPodNumber := 2 controlPlaneConfig := framework.NewIntegrationTestControlPlaneConfig() controlPlaneConfig.GenericConfig.AdmissionControl = &fakePodFailAdmission{ limitedPodNumber: limitedPodNumber, } _, s, closeFn := framework.RunAnAPIServer(controlPlaneConfig) defer closeFn() config := restclient.Config{Host: s.URL} c, err := clientset.NewForConfig(&config) if err != nil { t.Fatalf("Could not create clientset: %v", err) } resyncPeriod := 12 * time.Hour informers := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "statefulset-informers")), resyncPeriod) ssc := statefulset.NewStatefulSetController( informers.Core().V1().Pods(), informers.Apps().V1().StatefulSets(), informers.Core().V1().PersistentVolumeClaims(), informers.Apps().V1().ControllerRevisions(), clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "statefulset-controller")), ) ns := framework.CreateTestingNamespace("test-pod-fail", s, t) defer framework.DeleteTestingNamespace(ns, s, t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() informers.Start(ctx.Done()) go ssc.Run(ctx, 5) sts := newSTS("sts", ns.Name, 4) _, err = c.AppsV1().StatefulSets(sts.Namespace).Create(context.TODO(), sts, metav1.CreateOptions{}) if err != nil { t.Fatalf("Could not create statefuleSet %s: %v", sts.Name, err) } wantReplicas := limitedPodNumber var gotReplicas int32 if err := wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) { newSTS, err := c.AppsV1().StatefulSets(sts.Namespace).Get(context.TODO(), sts.Name, metav1.GetOptions{}) if err != nil { return false, err } gotReplicas = newSTS.Status.Replicas return gotReplicas == int32(wantReplicas), nil }); err != nil { t.Fatalf("StatefulSet %s status has %d replicas, want replicas %d: %v", sts.Name, gotReplicas, wantReplicas, err) } } </ins>


metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" <ins> "k8s.io/apiserver/pkg/admission" </ins> "k8s.io/client-go/informers" clientset "k8s.io/client-go/kubernetes" typedappsv1 "k8s.io/client-go/kubernetes/typed/apps/v1" typedv1 "k8s.io/client-go/kubernetes/typed/core/v1" restclient "k8s.io/client-go/rest" "k8s.io/client-go/util/retry" <ins> api "k8s.io/kubernetes/pkg/apis/core" </ins> //svc "k8s.io/kubernetes/pkg/api/v1/service" "k8s.io/kubernetes/pkg/controller/statefulset"


} waitSTSStable(t, c, sts) } <ins> var _ admission.ValidationInterface = &fakePodFailAdmission{} type fakePodFailAdmission struct { limitedPodNumber int succeedPodsCount int } func (f *fakePodFailAdmission) Handles(operation admission.Operation) bool { return operation == admission.Create } func (f *fakePodFailAdmission) Validate(ctx context.Context, attr admission.Attributes, o admission.ObjectInterfaces) (err error) { if attr.GetKind().GroupKind() != api.Kind("Pod") { return nil } if f.succeedPodsCount >= f.limitedPodNumber { return fmt.Errorf("fakePodFailAdmission error") } f.succeedPodsCount++ return nil } </ins>


"context" "crypto/sha256" "encoding/binary" <ins> "encoding/json" </ins> "errors" "fmt" "math"


apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" <ins> apitypes "k8s.io/apimachinery/pkg/types" </ins> utilerrors "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets"


"k8s.io/utils/clock" flowcontrol "k8s.io/api/flowcontrol/v1beta2" <del> flowcontrolapplyconfiguration "k8s.io/client-go/applyconfigurations/flowcontrol/v1beta2" </del> flowcontrolclient "k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta2" flowcontrollister "k8s.io/client-go/listers/flowcontrol/v1beta2" )


// if we are going to issue an update, be sure we track every name we update so we know if we update it too often. currResult.updatedItems.Insert(fsu.flowSchema.Name) <ins> patchBytes, err := makeFlowSchemaConditionPatch(fsu.condition) if err != nil { // should never happen because these conditions are created here and well formed panic(fmt.Sprintf("Failed to json.Marshall(%#+v): %s", fsu.condition, err.Error())) } </ins> if klogV := klog.V(4); klogV.Enabled() { klogV.Infof("%s writing Condition %s to FlowSchema %s, which had ResourceVersion=%s, because its previous value was %s, diff: %s", cfgCtlr.name, fsu.condition, fsu.flowSchema.Name, fsu.flowSchema.ResourceVersion, fcfmt.Fmt(fsu.oldValue), cmp.Diff(fsu.oldValue, fsu.condition)) } fsIfc := cfgCtlr.flowcontrolClient.FlowSchemas() <del> applyOptions := metav1.ApplyOptions{FieldManager: cfgCtlr.asFieldManager, Force: true} // the condition field in fsStatusUpdate holds the new condition we want to update. // TODO: this will break when we have multiple conditions for a flowschema _, err := fsIfc.ApplyStatus(context.TODO(), toFlowSchemaApplyConfiguration(fsu), applyOptions) </del> <ins> patchOptions := metav1.PatchOptions{FieldManager: cfgCtlr.asFieldManager} _, err = fsIfc.Patch(context.TODO(), fsu.flowSchema.Name, apitypes.StrategicMergePatchType, patchBytes, patchOptions, "status") </ins> if err != nil { if apierrors.IsNotFound(err) { // This object has been deleted.  A notification is coming


return suggestedDelay, utilerrors.NewAggregate(errs) } <del> func toFlowSchemaApplyConfiguration(fsUpdate fsStatusUpdate) *flowcontrolapplyconfiguration.FlowSchemaApplyConfiguration { condition := flowcontrolapplyconfiguration.FlowSchemaCondition(). WithType(fsUpdate.condition.Type). WithStatus(fsUpdate.condition.Status). WithReason(fsUpdate.condition.Reason). WithLastTransitionTime(fsUpdate.condition.LastTransitionTime). WithMessage(fsUpdate.condition.Message) return flowcontrolapplyconfiguration.FlowSchema(fsUpdate.flowSchema.Name). WithStatus(flowcontrolapplyconfiguration.FlowSchemaStatus(). WithConditions(condition), ) </del> <ins> // makeFlowSchemaConditionPatch takes in a condition and returns the patch status as a json. func makeFlowSchemaConditionPatch(condition flowcontrol.FlowSchemaCondition) ([]byte, error) { o := struct { Status flowcontrol.FlowSchemaStatus `json:"status"` }{ Status: flowcontrol.FlowSchemaStatus{ Conditions: []flowcontrol.FlowSchemaCondition{ condition, }, }, } return json.Marshal(o) </ins> } // shouldDelayUpdate checks to see if a flowschema has been updated too often and returns true if a delay is needed.


<ins> /* Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package flowcontrol import ( "fmt" "reflect" "testing" "time" "github.com/google/go-cmp/cmp" flowcontrol "k8s.io/api/flowcontrol/v1beta2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func Test_configController_generatePatchBytes(t *testing.T) { now := time.Now().UTC() tests := []struct { name      string condition flowcontrol.FlowSchemaCondition want      []byte }{ { name: "check if only condition is parsed", condition: flowcontrol.FlowSchemaCondition{ Type:               flowcontrol.FlowSchemaConditionDangling, Status:             flowcontrol.ConditionTrue, Reason:             "test reason", Message:            "test none", LastTransitionTime: metav1.NewTime(now), }, want: []byte(fmt.Sprintf(`{"status":{"conditions":[{"type":"Dangling","status":"True","lastTransitionTime":"%s","reason":"test reason","message":"test none"}]}}`, now.Format(time.RFC3339))), }, { name: "check when message has double quotes", condition: flowcontrol.FlowSchemaCondition{ Type:               flowcontrol.FlowSchemaConditionDangling, Status:             flowcontrol.ConditionTrue, Reason:             "test reason", Message:            `test ""none`, LastTransitionTime: metav1.NewTime(now), }, want: []byte(fmt.Sprintf(`{"status":{"conditions":[{"type":"Dangling","status":"True","lastTransitionTime":"%s","reason":"test reason","message":"test ""none"}]}}`, now.Format(time.RFC3339))), }, { name: "check when message has a whitespace character that can be escaped", condition: flowcontrol.FlowSchemaCondition{ Type:   flowcontrol.FlowSchemaConditionDangling, Status: flowcontrol.ConditionTrue, Reason: "test reason", Message: `test 		none`, LastTransitionTime: metav1.NewTime(now), }, want: []byte(fmt.Sprintf(`{"status":{"conditions":[{"type":"Dangling","status":"True","lastTransitionTime":"%s","reason":"test reason","message":"test ttnone"}]}}`, now.Format(time.RFC3339))), }, { name: "check when a few fields (message & lastTransitionTime) are missing", condition: flowcontrol.FlowSchemaCondition{ Type:   flowcontrol.FlowSchemaConditionDangling, Status: flowcontrol.ConditionTrue, Reason: "test reason", }, want: []byte(`{"status":{"conditions":[{"type":"Dangling","status":"True","lastTransitionTime":null,"reason":"test reason"}]}}`), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, _ := makeFlowSchemaConditionPatch(tt.condition) if !reflect.DeepEqual(got, tt.want) { t.Errorf("makeFlowSchemaConditionPatch() got = %s, want %s; diff is %s", got, tt.want, cmp.Diff(tt.want, got)) } }) } } </ins>


// resetFieldsStatusDefault conflicts with statusDefault const resetFieldsStatusDefault = `{"status": {"conditions": [{"type": "MyStatus", "status":"False"}]}}` <del> var resetFieldsSkippedResources = map[string]struct{}{} </del> <ins> var resetFieldsSkippedResources = map[string]struct{}{ // TODO: flowschemas is flaking, // possible bug in the flowschemas controller. "flowschemas": {}, } </ins> // noConflicts is the set of reources for which // a conflict cannot occur.


} // Return the first match. <del> backingFilePath := strings.TrimSpace(string(data)) </del> <ins> backingFilePath := cleanBackingFilePath(string(data)) </ins> if backingFilePath == path || backingFilePath == realPath { return fmt.Sprintf("/dev/%s", filepath.Base(device)), nil }


return "", errors.New(ErrDeviceNotFound) } <ins> // cleanPath remove any trailing substrings that are not part of the backing file path. func cleanBackingFilePath(path string) string { // If the block device was deleted, the path will contain a "(deleted)" suffix path = strings.TrimSpace(path) path = strings.TrimSuffix(path, "(deleted)") return strings.TrimSpace(path) } </ins> // FindGlobalMapPathUUIDFromPod finds {pod uuid} bind mount under globalMapPath // corresponding to map path symlink, and then return global map path with pod uuid. // (See pkg/volume/volume.go for details on a global map path and a pod device map path.)


<ins> /* Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package volumepathhandler import ( "fmt" "testing" ) func pathWithSuffix(suffix string) string { return fmt.Sprintf("%s%s", "/var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/pvc-1d205234-06cd-4fe4-a7ea-0e8f3e2faf5f/dev/e196ebd3-2ab1-4185-bed4-b997ba38d1dc", suffix) } func TestCleanBackingFilePath(t *testing.T) { const defaultPath = "/var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/pvc-1d205234-06cd-4fe4-a7ea-0e8f3e2faf5f/dev/e196ebd3-2ab1-4185-bed4-b997ba38d1dc" testCases := []struct { name          string input         string expectedOuput string }{ { name:          "regular path", input:         defaultPath, expectedOuput: defaultPath, }, { name:          "path is suffixed with whitespaces", input:         fmt.Sprintf("%srtn ", defaultPath), expectedOuput: defaultPath, }, { name:          "path is suffixed with "(deleted)"", input:         pathWithSuffix("(deleted)"), expectedOuput: defaultPath, }, { name:          "path is suffixed with "(deleted)" and whitespaces", input:         pathWithSuffix(" (deleted)t"), expectedOuput: defaultPath, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { output := cleanBackingFilePath(tc.input) if output != tc.expectedOuput { t.Fatalf("expected %q, got %q", tc.expectedOuput, output) } }) } } </ins>


var r result client := newClient(true) for i := 0; i < 5; i++ { <del> r = doer.Do(client, func(httptrace.GotConnInfo) {}, fmt.Sprintf("/echo?message=attempt-%d", i), 100*time.Millisecond) </del> <ins> r = doer.Do(client, func(httptrace.GotConnInfo) {}, fmt.Sprintf("/echo?message=attempt-%d", i), 1*time.Second) </ins> if r.err != nil { break }


} if err := vec.Validate(); err != nil { <del> klog.Error(err) </del> <ins> klog.ErrorS(err, "the validation for HistogramVec is failed. The data for this metric won't be stored in a benchmark result file", "metric", metric, "labels", labels) return nil } if vec.GetAggregatedSampleCount() == 0 { klog.InfoS("It is expected that this metric wasn't recorded. The data for this metric won't be stored in a benchmark result file", "metric", metric, "labels", labels) </ins> return nil }


} func (rc *resourceMetricsCollector) collectContainerStartTime(ch chan<- metrics.Metric, pod summary.PodStats, s summary.ContainerStats) { <del> if s.StartTime.Unix() == 0 { </del> <ins> if s.StartTime.Unix() <= 0 { </ins> return }


`, }, { <ins> name: "arbitrary container metrics for negative StartTime", summary: &statsapi.Summary{ Pods: []statsapi.PodStats{ { PodRef: statsapi.PodReference{ Name:      "pod_a", Namespace: "namespace_a", }, Containers: []statsapi.ContainerStats{ { Name:      "container_a", StartTime: metav1.NewTime(time.Unix(0, -1624396278302091597)), CPU: &statsapi.CPUStats{ Time:                 testTime, UsageCoreNanoSeconds: uint64Ptr(10000000000), }, Memory: &statsapi.MemoryStats{ Time:            testTime, WorkingSetBytes: uint64Ptr(1000), }, }, }, }, }, }, summaryErr: nil, expectedMetrics: ` # HELP scrape_error [ALPHA] 1 if there was an error while getting container metrics, 0 otherwise # TYPE scrape_error gauge scrape_error 0 # HELP container_cpu_usage_seconds_total [ALPHA] Cumulative cpu time consumed by the container in core-seconds # TYPE container_cpu_usage_seconds_total counter container_cpu_usage_seconds_total{container="container_a",namespace="namespace_a",pod="pod_a"} 10 1624396278302 # HELP container_memory_working_set_bytes [ALPHA] Current working set of the container in bytes # TYPE container_memory_working_set_bytes gauge container_memory_working_set_bytes{container="container_a",namespace="namespace_a",pod="pod_a"} 1000 1624396278302 `, }, { </ins> name: "nil container metrics", summary: &statsapi.Summary{ Pods: []statsapi.PodStats{


} func NewResourceExpirationEvaluator(currentVersion apimachineryversion.Info) (ResourceExpirationEvaluator, error) { <del> ret := &resourceExpirationEvaluator{ // TODO https://github.com/kubernetes/kubernetes/issues/109799 set this back to false after beta is tagged. strictRemovedHandlingInAlpha: true, } </del> <ins> ret := &resourceExpirationEvaluator{} </ins> if len(currentVersion.Major) > 0 { currentMajor64, err := strconv.ParseInt(currentVersion.Major, 10, 32) if err != nil {


go k.ListenAndServe(kubeCfg, kubeDeps.TLSOptions, kubeDeps.Auth, kubeDeps.TracerProvider) } if kubeCfg.ReadOnlyPort > 0 { <del> go k.ListenAndServeReadOnly(netutils.ParseIPSloppy(kubeCfg.Address), uint(kubeCfg.ReadOnlyPort)) </del> <ins> go k.ListenAndServeReadOnly(netutils.ParseIPSloppy(kubeCfg.Address), uint(kubeCfg.ReadOnlyPort), kubeDeps.TracerProvider) </ins> } go k.ListenAndServePodResources() }


BirthCry() StartGarbageCollection() ListenAndServe(kubeCfg *kubeletconfiginternal.KubeletConfiguration, tlsOptions *server.TLSOptions, auth server.AuthInterface, tp trace.TracerProvider) <del> ListenAndServeReadOnly(address net.IP, port uint) </del> <ins> ListenAndServeReadOnly(address net.IP, port uint, tp trace.TracerProvider) </ins> ListenAndServePodResources() Run(<-chan kubetypes.PodUpdate) RunOnce(<-chan kubetypes.PodUpdate) ([]RunPodResult, error)


} // ListenAndServeReadOnly runs the kubelet HTTP server in read-only mode. <del> func (kl *Kubelet) ListenAndServeReadOnly(address net.IP, port uint) { server.ListenAndServeKubeletReadOnlyServer(kl, kl.resourceAnalyzer, address, port) </del> <ins> func (kl *Kubelet) ListenAndServeReadOnly(address net.IP, port uint, tp trace.TracerProvider) { server.ListenAndServeKubeletReadOnlyServer(kl, kl.resourceAnalyzer, address, port, tp) </ins> } // ListenAndServePodResources runs the kubelet podresources grpc service


address := netutils.ParseIPSloppy(kubeCfg.Address) port := uint(kubeCfg.Port) klog.InfoS("Starting to listen", "address", address, "port", port) <del> handler := NewServer(host, resourceAnalyzer, auth, tp, kubeCfg) </del> <ins> handler := NewServer(host, resourceAnalyzer, auth, kubeCfg) if utilfeature.DefaultFeatureGate.Enabled(features.KubeletTracing) { handler.InstallTracingFilter(tp) } </ins> s := &http.Server{ Addr:           net.JoinHostPort(address.String(), strconv.FormatUint(uint64(port), 10)), Handler:        &handler,


host HostInterface, resourceAnalyzer stats.ResourceAnalyzer, address net.IP, <del> port uint) { </del> <ins> port uint, tp oteltrace.TracerProvider) { </ins> klog.InfoS("Starting to listen read-only", "address", address, "port", port) <del> // TODO: https://github.com/kubernetes/kubernetes/issues/109829 tracer should use WithPublicEndpoint s := NewServer(host, resourceAnalyzer, nil, oteltrace.NewNoopTracerProvider(), nil) </del> <ins> s := NewServer(host, resourceAnalyzer, nil, nil) if utilfeature.DefaultFeatureGate.Enabled(features.KubeletTracing) { s.InstallTracingFilter(tp, otelrestful.WithPublicEndpoint()) } </ins> server := &http.Server{ Addr:           net.JoinHostPort(address.String(), strconv.FormatUint(uint64(port), 10)),


host HostInterface, resourceAnalyzer stats.ResourceAnalyzer, auth AuthInterface, <del> tp oteltrace.TracerProvider, </del> kubeCfg *kubeletconfiginternal.KubeletConfiguration) Server { server := Server{


if auth != nil { server.InstallAuthFilter() } <del> if utilfeature.DefaultFeatureGate.Enabled(features.KubeletTracing) { server.InstallTracingFilter(tp) } </del> server.InstallDefaultHandlers() if kubeCfg != nil && kubeCfg.EnableDebuggingHandlers { server.InstallDebuggingHandlers()


} // InstallTracingFilter installs OpenTelemetry tracing filter with the restful Container. <del> func (s *Server) InstallTracingFilter(tp oteltrace.TracerProvider) { s.restfulCont.Filter(otelrestful.OTelFilter("kubelet", otelrestful.WithTracerProvider(tp))) </del> <ins> func (s *Server) InstallTracingFilter(tp oteltrace.TracerProvider, opts ...otelrestful.Option) { s.restfulCont.Filter(otelrestful.OTelFilter("kubelet", append(opts, otelrestful.WithTracerProvider(tp))...)) </ins> } // addMetricsBucketMatcher adds a regexp matcher and the relevant bucket to use when


cadvisorapiv2 "github.com/google/cadvisor/info/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" <del> oteltrace "go.opentelemetry.io/otel/trace" </del> v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types"


fw.fakeKubelet, stats.NewResourceAnalyzer(fw.fakeKubelet, time.Minute, &record.FakeRecorder{}), fw.fakeAuth, <del> oteltrace.NewNoopTracerProvider(), </del> kubeCfg, ) fw.serverUnderTest = &server


"k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" <del> kubefeatures "k8s.io/kubernetes/pkg/features" </del> "k8s.io/kubernetes/test/e2e/framework" e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" <del> e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" </del> imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api"


f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged ginkgo.Context("Downward API tests for hugepages", func() { <del> ginkgo.BeforeEach(func() { e2eskipper.SkipUnlessFeatureGateEnabled(kubefeatures.DownwardAPIHugePages) }) </del> ginkgo.It("should provide container's limits.hugepages-<pagesize> and requests.hugepages-<pagesize> as env vars", func() { podName := "downward-api-" + string(uuid.NewUUID()) env := []v1.EnvVar{


Done(types.UID) Update(logger klog.Logger, oldPod, newPod *v1.Pod) Delete(pod *v1.Pod) <del> // TODO(sanposhiho): move all PreEnqueueCheck to Requeue and delete it from this parameter eventually. // Some PreEnqueueCheck include event filtering logic based on some in-tree plugins // and it affect badly to other plugins. // See https://github.com/kubernetes/kubernetes/issues/110175 </del> <ins> // Important Note: preCheck shouldn't include anything that depends on the in-tree plugins' logic. // (e.g., filter Pods based on added/updated Node's capacity, etc.) // We know currently some do, but we'll eventually remove them in favor of the scheduling queue hint. </ins> MoveAllToActiveOrBackoffQueue(logger klog.Logger, event framework.ClusterEvent, oldObj, newObj interface{}, preCheck PreEnqueueCheck) AssignedPodAdded(logger klog.Logger, pod *v1.Pod) AssignedPodUpdated(logger klog.Logger, oldPod, newPod *v1.Pod, event framework.ClusterEvent)


} func preCheckForNode(nodeInfo *framework.NodeInfo) queue.PreEnqueueCheck { <ins> if utilfeature.DefaultFeatureGate.Enabled(features.SchedulerQueueingHints) { // QHint is initially created from the motivation of replacing this preCheck. // It assumes that the scheduler only has in-tree plugins, which is problematic for our extensibility. // Here, we skip preCheck if QHint is enabled, and we eventually remove it after QHint is graduated. return nil } </ins> // Note: the following checks doesn't take preemption into considerations, in very rare // cases (e.g., node resizing), "pod" may still fail a check but preemption helps. We deliberately // chose to ignore those cases as unschedulable pods will be re-queued eventually.


nodeFn             func() *v1.Node existingPods, pods []*v1.Pod want               []bool <ins> qHintEnabled       bool </ins> }{ { name: "regular node, pods with a single constraint",


want: []bool{true, false, false, true, false, true, false, true, false}, }, { <ins> name: "no filtering when QHint is enabled", nodeFn: func() *v1.Node { return st.MakeNode().Name("fake-node").Label("hostname", "fake-node").Capacity(cpu8).Obj() }, existingPods: []*v1.Pod{ st.MakePod().Name("p").HostPort(80).Obj(), }, pods: []*v1.Pod{ st.MakePod().Name("p1").Req(cpu4).Obj(), st.MakePod().Name("p2").Req(cpu16).Obj(), st.MakePod().Name("p3").Req(cpu4).Req(cpu8).Obj(), st.MakePod().Name("p4").NodeAffinityIn("hostname", []string{"fake-node"}).Obj(), st.MakePod().Name("p5").NodeAffinityNotIn("hostname", []string{"fake-node"}).Obj(), st.MakePod().Name("p6").Obj(), st.MakePod().Name("p7").Node("invalid-node").Obj(), st.MakePod().Name("p8").HostPort(8080).Obj(), st.MakePod().Name("p9").HostPort(80).Obj(), }, qHintEnabled: true, want:         []bool{true, true, true, true, true, true, true, true, true}, }, { </ins> name: "tainted node, pods with a single constraint", nodeFn: func() *v1.Node { node := st.MakeNode().Name("fake-node").Obj()


for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { <ins> featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.SchedulerQueueingHints, tt.qHintEnabled) </ins> nodeInfo := framework.NewNodeInfo(tt.existingPods...) nodeInfo.SetNode(tt.nodeFn()) preCheckFn := preCheckForNode(nodeInfo) <del> var got []bool </del> <ins> got := make([]bool, 0, len(tt.pods)) </ins> for _, pod := range tt.pods { <del> got = append(got, preCheckFn(pod)) </del> <ins> got = append(got, preCheckFn == nil || preCheckFn(pod)) </ins> } if diff := cmp.Diff(tt.want, got); diff != "" {


} // newPodInformer creates a shared index informer that returns only non-terminal pods. <ins> // The PodInformer allows indexers to be added, but note that only non-conflict indexers are allowed. </ins> func newPodInformer(cs clientset.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { selector := fmt.Sprintf("status.phase!=%v,status.phase!=%v", v1.PodSucceeded, v1.PodFailed) tweakListOptions := func(options *metav1.ListOptions) { options.FieldSelector = selector } <del> return coreinformers.NewFilteredPodInformer(cs, metav1.NamespaceAll, resyncPeriod, nil, tweakListOptions) </del> <ins> return coreinformers.NewFilteredPodInformer(cs, metav1.NamespaceAll, resyncPeriod, cache.Indexers{}, tweakListOptions) </ins> }


v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" <ins> "k8s.io/apimachinery/pkg/runtime" </ins> "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes"


return s, fwk, nil } <ins> func TestInitPluginsWithIndexers(t *testing.T) { tests := []struct { name string // the plugin registration ordering must not matter, being map traversal random entrypoints map[string]frameworkruntime.PluginFactory wantErr     string }{ { name: "register indexer, no conflicts", entrypoints: map[string]frameworkruntime.PluginFactory{ "AddIndexer": func(obj runtime.Object, handle framework.Handle) (framework.Plugin, error) { podInformer := handle.SharedInformerFactory().Core().V1().Pods() err := podInformer.Informer().GetIndexer().AddIndexers(cache.Indexers{ "nodeName": indexByPodSpecNodeName, }) return &TestPlugin{name: "AddIndexer"}, err }, }, }, { name: "register the same indexer name multiple times, conflict", // order of registration doesn't matter entrypoints: map[string]frameworkruntime.PluginFactory{ "AddIndexer1": func(obj runtime.Object, handle framework.Handle) (framework.Plugin, error) { podInformer := handle.SharedInformerFactory().Core().V1().Pods() err := podInformer.Informer().GetIndexer().AddIndexers(cache.Indexers{ "nodeName": indexByPodSpecNodeName, }) return &TestPlugin{name: "AddIndexer1"}, err }, "AddIndexer2": func(obj runtime.Object, handle framework.Handle) (framework.Plugin, error) { podInformer := handle.SharedInformerFactory().Core().V1().Pods() err := podInformer.Informer().GetIndexer().AddIndexers(cache.Indexers{ "nodeName": indexByPodAnnotationNodeName, }) return &TestPlugin{name: "AddIndexer1"}, err }, }, wantErr: "indexer conflict", }, { name: "register the same indexer body with different names, no conflicts", // order of registration doesn't matter entrypoints: map[string]frameworkruntime.PluginFactory{ "AddIndexer1": func(obj runtime.Object, handle framework.Handle) (framework.Plugin, error) { podInformer := handle.SharedInformerFactory().Core().V1().Pods() err := podInformer.Informer().GetIndexer().AddIndexers(cache.Indexers{ "nodeName1": indexByPodSpecNodeName, }) return &TestPlugin{name: "AddIndexer1"}, err }, "AddIndexer2": func(obj runtime.Object, handle framework.Handle) (framework.Plugin, error) { podInformer := handle.SharedInformerFactory().Core().V1().Pods() err := podInformer.Informer().GetIndexer().AddIndexers(cache.Indexers{ "nodeName2": indexByPodAnnotationNodeName, }) return &TestPlugin{name: "AddIndexer2"}, err }, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { fakeInformerFactory := NewInformerFactory(&fake.Clientset{}, 0*time.Second) var registerPluginFuncs []st.RegisterPluginFunc for name, entrypoint := range tt.entrypoints { registerPluginFuncs = append(registerPluginFuncs, // anything supported by TestPlugin is fine st.RegisterFilterPlugin(name, entrypoint), ) } // we always need this registerPluginFuncs = append(registerPluginFuncs, st.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), st.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), ) _, err := st.NewFramework(registerPluginFuncs, "test", frameworkruntime.WithInformerFactory(fakeInformerFactory)) if len(tt.wantErr) > 0 { if err == nil || !strings.Contains(err.Error(), tt.wantErr) { t.Errorf("got error %q, want %q", err, tt.wantErr) } return } if err != nil { t.Fatalf("Failed to create scheduler: %v", err) } }) } } func indexByPodSpecNodeName(obj interface{}) ([]string, error) { pod, ok := obj.(*v1.Pod) if !ok { return []string{}, nil } if len(pod.Spec.NodeName) == 0 { return []string{}, nil } return []string{pod.Spec.NodeName}, nil } func indexByPodAnnotationNodeName(obj interface{}) ([]string, error) { pod, ok := obj.(*v1.Pod) if !ok { return []string{}, nil } if len(pod.Annotations) == 0 { return []string{}, nil } nodeName, ok := pod.Annotations["node-name"] if !ok { return []string{}, nil } return []string{nodeName}, nil } </ins>


visible at runtime in the container. release: v1.9 file: test/e2e/common/node/downwardapi.go <ins> - testname: Ephemeral Container Creation codename: '[sig-node] Ephemeral Containers [NodeConformance] will start an ephemeral container in an existing pod [Conformance]' description: Adding an ephemeral container to pod.spec MUST result in the container running. release: "1.25" file: test/e2e/common/node/ephemeral_containers.go </ins> - testname: init-container-starts-app-restartalways-pod codename: '[sig-node] InitContainer [NodeConformance] should invoke init containers on a RestartAlways pod [Conformance]'


podClient = f.PodClient() }) <del> ginkgo.It("will start an ephemeral container in an existing pod", func() { </del> <ins> // Release: 1.25 // Testname: Ephemeral Container Creation // Description: Adding an ephemeral container to pod.spec MUST result in the container running. framework.ConformanceIt("will start an ephemeral container in an existing pod", func() { </ins> ginkgo.By("creating a target pod") pod := podClient.CreateSync(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "ephemeral-containers-target-pod"},


"k8s.io/client-go/openapi" cachedopenapi "k8s.io/client-go/openapi/cached" restclient "k8s.io/client-go/rest" <ins> "k8s.io/klog/v2" </ins> ) type cacheEntry struct {


ErrCacheNotFound = errors.New("not found") ) <ins> // Server returning empty ResourceList for Group/Version. type emptyResponseError struct { gv string } func (e *emptyResponseError) Error() string { return fmt.Sprintf("received empty response for: %s", e.gv) } </ins> var _ discovery.CachedDiscoveryInterface = &memCacheClient{} // isTransientConnectionError checks whether given error is "Connection refused" or


if cachedVal.err != nil && isTransientError(cachedVal.err) { r, err := d.serverResourcesForGroupVersion(groupVersion) if err != nil { <del> utilruntime.HandleError(fmt.Errorf("couldn't get resource list for %v: %v", groupVersion, err)) </del> <ins> // Don't log "empty response" as an error; it is a common response for metrics. if _, emptyErr := err.(*emptyResponseError); emptyErr { // Log at same verbosity as disk cache. klog.V(3).Infof("%v", err) } else { utilruntime.HandleError(fmt.Errorf("couldn't get resource list for %v: %v", groupVersion, err)) } </ins> } cachedVal = &cacheEntry{r, err} d.groupToServerResources[groupVersion] = cachedVal


r, err := d.serverResourcesForGroupVersion(gv) if err != nil { <del> utilruntime.HandleError(fmt.Errorf("couldn't get resource list for %v: %v", gv, err)) </del> <ins> // Don't log "empty response" as an error; it is a common response for metrics. if _, emptyErr := err.(*emptyResponseError); emptyErr { // Log at same verbosity as disk cache. klog.V(3).Infof("%v", err) } else { utilruntime.HandleError(fmt.Errorf("couldn't get resource list for %v: %v", gv, err)) } </ins> } resultLock.Lock()


return r, err } if len(r.APIResources) == 0 { <del> return r, fmt.Errorf("Got empty response for: %v", groupVersion) </del> <ins> return r, &emptyResponseError{gv: groupVersion} </ins> } return r, nil }


return finalURL } <ins> // finalURLTemplate is similar to URL(), but will make all specific parameter values equal // - instead of name or namespace, "{name}" and "{namespace}" will be used, and all query // parameters will be reset. This creates a copy of the url so as not to change the // underlying object. func (r Request) finalURLTemplate() url.URL { newParams := url.Values{} v := []string{"{value}"} for k := range r.params { newParams[k] = v } r.params = newParams u := r.URL() if u == nil { return url.URL{} } segments := strings.Split(u.Path, "/") groupIndex := 0 index := 0 trimmedBasePath := "" if r.c.base != nil && strings.Contains(u.Path, r.c.base.Path) { p := strings.TrimPrefix(u.Path, r.c.base.Path) if !strings.HasPrefix(p, "/") { p = "/" + p } // store the base path that we have trimmed so we can append it // before returning the URL trimmedBasePath = r.c.base.Path segments = strings.Split(p, "/") groupIndex = 1 } if len(segments) <= 2 { return *u } const CoreGroupPrefix = "api" const NamedGroupPrefix = "apis" isCoreGroup := segments[groupIndex] == CoreGroupPrefix isNamedGroup := segments[groupIndex] == NamedGroupPrefix if isCoreGroup { // checking the case of core group with /api/v1/... format index = groupIndex + 2 } else if isNamedGroup { // checking the case of named group with /apis/apps/v1/... format index = groupIndex + 3 } else { // this should not happen that the only two possibilities are /api... and /apis..., just want to put an // outlet here in case more API groups are added in future if ever possible: // https://kubernetes.io/docs/concepts/overview/kubernetes-api/#api-groups // if a wrong API groups name is encountered, return the {prefix} for url.Path u.Path = "/{prefix}" u.RawQuery = "" return *u } // switch segLength := len(segments) - index; segLength { switch { // case len(segments) - index == 1: // resource (with no name) do nothing case len(segments)-index == 2: // /$RESOURCE/$NAME: replace $NAME with {name} segments[index+1] = "{name}" case len(segments)-index == 3: if segments[index+2] == "finalize" || segments[index+2] == "status" { // /$RESOURCE/$NAME/$SUBRESOURCE: replace $NAME with {name} segments[index+1] = "{name}" } else { // /namespace/$NAMESPACE/$RESOURCE: replace $NAMESPACE with {namespace} segments[index+1] = "{namespace}" } case len(segments)-index >= 4: segments[index+1] = "{namespace}" // /namespace/$NAMESPACE/$RESOURCE/$NAME: replace $NAMESPACE with {namespace},  $NAME with {name} if segments[index+3] != "finalize" && segments[index+3] != "status" { // /$RESOURCE/$NAME/$SUBRESOURCE: replace $NAME with {name} segments[index+3] = "{name}" } } u.Path = path.Join(trimmedBasePath, path.Join(segments...)) return *u } </ins> func (r *Request) tryThrottleWithInfo(ctx context.Context, retryInfo string) error { if r.rateLimiter == nil { return nil


// but we use a throttled logger to prevent spamming. globalThrottledLogger.Infof("%s", message) } <del> metrics.RateLimiterLatency.Observe(ctx, r.verb, *r.URL(), latency) </del> <ins> metrics.RateLimiterLatency.Observe(ctx, r.verb, r.finalURLTemplate(), latency) </ins> return err }


// Metrics for total request latency start := time.Now() defer func() { <del> metrics.RequestLatency.Observe(ctx, r.verb, *r.URL(), time.Since(start)) </del> <ins> metrics.RequestLatency.Observe(ctx, r.verb, r.finalURLTemplate(), time.Since(start)) </ins> }() if r.err != nil {


} } <ins> func TestURLTemplate(t *testing.T) { uri, _ := url.Parse("http://localhost/some/base/url/path") uriSingleSlash, _ := url.Parse("http://localhost/") testCases := []struct { Request          *Request ExpectedFullURL  string ExpectedFinalURL string }{ { // non dynamic client Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("POST"). Prefix("api", "v1").Resource("r1").Namespace("ns").Name("nm").Param("p0", "v0"), ExpectedFullURL:  "http://localhost/some/base/url/path/api/v1/namespaces/ns/r1/nm?p0=v0", ExpectedFinalURL: "http://localhost/some/base/url/path/api/v1/namespaces/%7Bnamespace%7D/r1/%7Bname%7D?p0=%7Bvalue%7D", }, { // non dynamic client with wrong api group Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("POST"). Prefix("pre1", "v1").Resource("r1").Namespace("ns").Name("nm").Param("p0", "v0"), ExpectedFullURL:  "http://localhost/some/base/url/path/pre1/v1/namespaces/ns/r1/nm?p0=v0", ExpectedFinalURL: "http://localhost/%7Bprefix%7D", }, { // dynamic client with core group + namespace + resourceResource (with name) // /api/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/api/v1/namespaces/ns/r1/name1"), ExpectedFullURL:  "http://localhost/some/base/url/path/api/v1/namespaces/ns/r1/name1", ExpectedFinalURL: "http://localhost/some/base/url/path/api/v1/namespaces/%7Bnamespace%7D/r1/%7Bname%7D", }, { // dynamic client with named group + namespace + resourceResource (with name) // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/g1/v1/namespaces/ns/r1/name1"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/g1/v1/namespaces/ns/r1/name1", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/g1/v1/namespaces/%7Bnamespace%7D/r1/%7Bname%7D", }, { // dynamic client with core group + namespace + resourceResource (with NO name) // /api/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/api/v1/namespaces/ns/r1"), ExpectedFullURL:  "http://localhost/some/base/url/path/api/v1/namespaces/ns/r1", ExpectedFinalURL: "http://localhost/some/base/url/path/api/v1/namespaces/%7Bnamespace%7D/r1", }, { // dynamic client with named group + namespace + resourceResource (with NO name) // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/g1/v1/namespaces/ns/r1"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/g1/v1/namespaces/ns/r1", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/g1/v1/namespaces/%7Bnamespace%7D/r1", }, { // dynamic client with core group + resourceResource (with name) // /api/$RESOURCEVERSION/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/api/v1/r1/name1"), ExpectedFullURL:  "http://localhost/some/base/url/path/api/v1/r1/name1", ExpectedFinalURL: "http://localhost/some/base/url/path/api/v1/r1/%7Bname%7D", }, { // dynamic client with named group + resourceResource (with name) // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/g1/v1/r1/name1"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/g1/v1/r1/name1", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/g1/v1/r1/%7Bname%7D", }, { // dynamic client with named group + namespace + resourceResource (with name) + subresource // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%NAME/$SUBRESOURCE Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/namespaces/namespaces/namespaces/namespaces/namespaces/namespaces/finalize"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/namespaces/namespaces/namespaces/finalize", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/%7Bnamespace%7D/namespaces/%7Bname%7D/finalize", }, { // dynamic client with named group + namespace + resourceResource (with name) // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/namespaces/namespaces/namespaces/namespaces/namespaces/namespaces"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/namespaces/namespaces/namespaces", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/%7Bnamespace%7D/namespaces/%7Bname%7D", }, { // dynamic client with named group + namespace + resourceResource (with NO name) + subresource // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%SUBRESOURCE Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/namespaces/namespaces/namespaces/namespaces/namespaces/finalize"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/namespaces/namespaces/finalize", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/%7Bnamespace%7D/namespaces/finalize", }, { // dynamic client with named group + namespace + resourceResource (with NO name) + subresource // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%SUBRESOURCE Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/namespaces/namespaces/namespaces/namespaces/namespaces/status"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/namespaces/namespaces/status", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/%7Bnamespace%7D/namespaces/status", }, { // dynamic client with named group + namespace + resourceResource (with no name) // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/namespaces/namespaces/namespaces/namespaces/namespaces"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/namespaces/namespaces", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/%7Bnamespace%7D/namespaces", }, { // dynamic client with named group + resourceResource (with name) + subresource // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/namespaces/namespaces/namespaces/namespaces/finalize"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/namespaces/finalize", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/%7Bname%7D/finalize", }, { // dynamic client with named group + resourceResource (with name) + subresource // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/namespaces/namespaces/namespaces/namespaces/status"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/namespaces/status", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/%7Bname%7D/status", }, { // dynamic client with named group + resourceResource (with name) // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/namespaces/namespaces/namespaces/namespaces"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/namespaces", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces/%7Bname%7D", }, { // dynamic client with named group + resourceResource (with no name) // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/apis/namespaces/namespaces/namespaces"), ExpectedFullURL:  "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces", ExpectedFinalURL: "http://localhost/some/base/url/path/apis/namespaces/namespaces/namespaces", }, { // dynamic client with wrong api group + namespace + resourceResource (with name) + subresource // /apis/$NAMEDGROUPNAME/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%NAME/$SUBRESOURCE Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/pre1/namespaces/namespaces/namespaces/namespaces/namespaces/namespaces/finalize"), ExpectedFullURL:  "http://localhost/some/base/url/path/pre1/namespaces/namespaces/namespaces/namespaces/namespaces/namespaces/finalize", ExpectedFinalURL: "http://localhost/%7Bprefix%7D", }, { // dynamic client with core group + namespace + resourceResource (with name) where baseURL is a single / // /api/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%NAME Request: NewRequestWithClient(uriSingleSlash, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/api/v1/namespaces/ns/r2/name1"), ExpectedFullURL:  "http://localhost/api/v1/namespaces/ns/r2/name1", ExpectedFinalURL: "http://localhost/api/v1/namespaces/%7Bnamespace%7D/r2/%7Bname%7D", }, { // dynamic client with core group + namespace + resourceResource (with name) where baseURL is 'some/base/url/path' // /api/$RESOURCEVERSION/namespaces/$NAMESPACE/$RESOURCE/%NAME Request: NewRequestWithClient(uri, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/api/v1/namespaces/ns/r3/name1"), ExpectedFullURL:  "http://localhost/some/base/url/path/api/v1/namespaces/ns/r3/name1", ExpectedFinalURL: "http://localhost/some/base/url/path/api/v1/namespaces/%7Bnamespace%7D/r3/%7Bname%7D", }, { // dynamic client where baseURL is a single / // / Request: NewRequestWithClient(uriSingleSlash, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/"), ExpectedFullURL:  "http://localhost/", ExpectedFinalURL: "http://localhost/", }, { // dynamic client where baseURL is a single / // /version Request: NewRequestWithClient(uriSingleSlash, "", ClientContentConfig{GroupVersion: schema.GroupVersion{Group: "test"}}, nil).Verb("DELETE"). Prefix("/version"), ExpectedFullURL:  "http://localhost/version", ExpectedFinalURL: "http://localhost/version", }, } for i, testCase := range testCases { r := testCase.Request full := r.URL() if full.String() != testCase.ExpectedFullURL { t.Errorf("%d: unexpected initial URL: %s %s", i, full, testCase.ExpectedFullURL) } actualURL := r.finalURLTemplate() actual := actualURL.String() if actual != testCase.ExpectedFinalURL { t.Errorf("%d: unexpected URL template: %s %s", i, actual, testCase.ExpectedFinalURL) } if r.URL().String() != full.String() { t.Errorf("%d, creating URL template changed request: %s -> %s", i, full.String(), r.URL().String()) } } } </ins> func TestTransformResponse(t *testing.T) { invalid := []byte("aaaaa") uri, _ := url.Parse("http://localhost")


"k8s.io/apiserver/pkg/storage/value" kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2alpha1" "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics" <ins> "k8s.io/klog/v2" </ins> "k8s.io/utils/lru" )


value.RecordCacheMiss() } uid := string(uuid.NewUUID()) <ins> klog.V(6).InfoS("Decrypting content using envelope service", "uid", uid, "key", string(dataCtx.AuthenticatedData())) </ins> key, err := t.envelopeService.Decrypt(ctx, uid, &DecryptRequest{ Ciphertext:  encryptedObject.EncryptedDEK, KeyID:       encryptedObject.KeyID,


} uid := string(uuid.NewUUID()) <ins> klog.V(6).InfoS("Encrypting content using envelope service", "uid", uid, "key", string(dataCtx.AuthenticatedData())) </ins> resp, err := t.envelopeService.Encrypt(ctx, uid, newKey) if err != nil { return nil, fmt.Errorf("failed to encrypt DEK, error: %w", err)


"k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" <ins> genericapirequest "k8s.io/apiserver/pkg/endpoints/request" </ins> "k8s.io/apiserver/pkg/storage/value" kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2alpha1" "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics"


if transformer == nil { value.RecordCacheMiss() <ins> requestInfo := getRequestInfoFromContext(ctx) </ins> uid := string(uuid.NewUUID()) <del> klog.V(6).InfoS("Decrypting content using envelope service", "uid", uid, "key", string(dataCtx.AuthenticatedData())) </del> <ins> klog.V(6).InfoS("decrypting content using envelope service", "uid", uid, "key", string(dataCtx.AuthenticatedData()), "group", requestInfo.APIGroup, "version", requestInfo.APIVersion, "resource", requestInfo.Resource, "subresource", requestInfo.Subresource, "verb", requestInfo.Verb, "namespace", requestInfo.Namespace, "name", requestInfo.Name) </ins> key, err := t.envelopeService.Decrypt(ctx, uid, &kmsservice.DecryptRequest{ Ciphertext:  encryptedObject.EncryptedDEK, KeyID:       encryptedObject.KeyID,


return nil, err } <ins> requestInfo := getRequestInfoFromContext(ctx) </ins> uid := string(uuid.NewUUID()) <del> klog.V(6).InfoS("encrypting content using envelope service", "uid", uid, "key", string(dataCtx.AuthenticatedData())) </del> <ins> klog.V(6).InfoS("encrypting content using envelope service", "uid", uid, "key", string(dataCtx.AuthenticatedData()), "group", requestInfo.APIGroup, "version", requestInfo.APIVersion, "resource", requestInfo.Resource, "subresource", requestInfo.Subresource, "verb", requestInfo.Verb, "namespace", requestInfo.Namespace, "name", requestInfo.Name) </ins> resp, err := t.envelopeService.Encrypt(ctx, uid, newKey) if err != nil { return nil, fmt.Errorf("failed to encrypt DEK, error: %w", err)


} return errKeyIDOKCode, nil } <ins> func getRequestInfoFromContext(ctx context.Context) *genericapirequest.RequestInfo { if reqInfo, found := genericapirequest.RequestInfoFrom(ctx); found { return reqInfo } return &genericapirequest.RequestInfo{} } </ins>


"bytes" "context" "encoding/base64" <ins> "flag" </ins> "fmt" "reflect" <ins> "regexp" </ins> "strconv" "strings" "testing" "time" <ins> genericapirequest "k8s.io/apiserver/pkg/endpoints/request" </ins> "k8s.io/apiserver/pkg/storage/value" aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes" kmstypes "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2/v2alpha1" "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/metrics" "k8s.io/component-base/metrics/legacyregistry" "k8s.io/component-base/metrics/testutil" <ins> "k8s.io/klog/v2" </ins> kmsservice "k8s.io/kms/pkg/service" testingclock "k8s.io/utils/clock/testing" )


}) } } <ins> func TestEnvelopeLogging(t *testing.T) { klog.InitFlags(nil) flag.Set("v", "6") flag.Parse() testCases := []struct { desc     string ctx      context.Context wantLogs []string }{ { desc: "no request info in context", ctx:  testContext(t), wantLogs: []string{ `"encrypting content using envelope service" uid="UID" key="0123456789" group="" version="" resource="" subresource="" verb="" namespace="" name=""`, `"decrypting content using envelope service" uid="UID" key="0123456789" group="" version="" resource="" subresource="" verb="" namespace="" name=""`, }, }, { desc: "request info in context", ctx: genericapirequest.WithRequestInfo(testContext(t), &genericapirequest.RequestInfo{ APIGroup:    "awesome.bears.com", APIVersion:  "v1", Resource:    "pandas", Subresource: "status", Namespace:   "kube-system", Name:        "panda", Verb:        "update", }), wantLogs: []string{ `"encrypting content using envelope service" uid="UID" key="0123456789" group="awesome.bears.com" version="v1" resource="pandas" subresource="status" verb="update" namespace="kube-system" name="panda"`, `"decrypting content using envelope service" uid="UID" key="0123456789" group="awesome.bears.com" version="v1" resource="pandas" subresource="status" verb="update" namespace="kube-system" name="panda"`, }, }, } for _, tc := range testCases { tc := tc t.Run(tc.desc, func(t *testing.T) { var buf bytes.Buffer klog.SetOutput(&buf) klog.LogToStderr(false) defer klog.LogToStderr(true) envelopeService := newTestEnvelopeService() fakeClock := testingclock.NewFakeClock(time.Now()) envelopeTransformer := newEnvelopeTransformerWithClock(envelopeService, testProviderName, func(ctx context.Context) (string, error) { return "1", nil }, func(ctx context.Context) error { return nil }, aestransformer.NewGCMTransformer, 1*time.Second, fakeClock) dataCtx := value.DefaultContext([]byte(testContextText)) originalText := []byte(testText) transformedData, err := envelopeTransformer.TransformToStorage(tc.ctx, originalText, dataCtx) if err != nil { t.Fatalf("envelopeTransformer: error while transforming data to storage: %v", err) } // advance the clock to trigger cache to expire, so we make a decrypt call that will log fakeClock.Step(2 * time.Second) _, _, err = envelopeTransformer.TransformFromStorage(tc.ctx, transformedData, dataCtx) if err != nil { t.Fatalf("could not decrypt Envelope transformer's encrypted data even once: %v", err) } klog.Flush() klog.SetOutput(&bytes.Buffer{}) // prevent further writes into buf capturedOutput := buf.String() // replace the uid with a constant to make the test output stable and assertable capturedOutput = regexp.MustCompile(`uid="[^"]+"`).ReplaceAllString(capturedOutput, `uid="UID"`) for _, wantLog := range tc.wantLogs { if !strings.Contains(capturedOutput, wantLog) { t.Errorf("expected log %q, got %q", wantLog, capturedOutput) } } }) } } </ins>


import ( "context" <ins> "encoding/json" </ins> "fmt" "reflect" "strconv" "time" v1 "k8s.io/api/core/v1" <ins> apiequality "k8s.io/apimachinery/pkg/api/equality" </ins> "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" <ins> "k8s.io/apimachinery/pkg/types" utilrand "k8s.io/apimachinery/pkg/util/rand" </ins> "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch"