framework.ExpectNoError(waitErr, "some pods failed to complete within %v", completeTimeout) }) }) <del> ginkgo.Context("Pods sharing a single local PV [Serial]", func() { var ( pv *v1.PersistentVolume ) ginkgo.BeforeEach(func(ctx context.Context) { localVolume := &localTestVolume{ ltr: &utils.LocalTestResource{ Node: config.randomNode, Path: "/tmp", }, localVolumeType: DirectoryLocalVolumeType, } pvConfig := makeLocalPVConfig(config, localVolume) var err error pv, err = e2epv.CreatePV(ctx, config.client, f.Timeouts, e2epv.MakePersistentVolume(pvConfig)) framework.ExpectNoError(err) }) ginkgo.AfterEach(func(ctx context.Context) { if pv == nil { return } ginkgo.By(fmt.Sprintf("Clean PV %s", pv.Name)) err := config.client.CoreV1().PersistentVolumes().Delete(ctx, pv.Name, metav1.DeleteOptions{}) framework.ExpectNoError(err) }) ginkgo.It("all pods should be running", func(ctx context.Context) { var ( pvc   *v1.PersistentVolumeClaim pods  = map[string]*v1.Pod{} count = 2 err   error ) pvc = e2epv.MakePersistentVolumeClaim(makeLocalPVCConfig(config, DirectoryLocalVolumeType), config.ns) ginkgo.By(fmt.Sprintf("Create a PVC %s", pvc.Name)) pvc, err = e2epv.CreatePVC(ctx, config.client, config.ns, pvc) framework.ExpectNoError(err) ginkgo.By(fmt.Sprintf("Create %d pods to use this PVC", count)) podConfig := e2epod.Config{ NS:           config.ns, PVCs:         []*v1.PersistentVolumeClaim{pvc}, SeLinuxLabel: selinuxLabel, } for i := 0; i < count; i++ { pod, err := e2epod.MakeSecPod(&podConfig) framework.ExpectNoError(err) pod, err = config.client.CoreV1().Pods(config.ns).Create(ctx, pod, metav1.CreateOptions{}) framework.ExpectNoError(err) pods[pod.Name] = pod } ginkgo.By("Wait for all pods are running") const runningTimeout = 5 * time.Minute waitErr := wait.PollImmediate(time.Second, runningTimeout, func() (done bool, err error) { podsList, err := config.client.CoreV1().Pods(config.ns).List(ctx, metav1.ListOptions{}) if err != nil { return false, err } runningPods := 0 for _, pod := range podsList.Items { switch pod.Status.Phase { case v1.PodRunning: runningPods++ } } return runningPods == count, nil }) framework.ExpectNoError(waitErr, "Some pods are not running within %v", runningTimeout) }) }) </del> }) func deletePodAndPVCs(ctx context.Context, config *localTestConfig, pod *v1.Pod) error {


See our documentation on [kubernetes.io]. <del> Try our [interactive tutorial]. </del> Take a free course on [Scalable Microservices with Kubernetes]. To use Kubernetes code as a library in other applications, see the [list of published components](https://git.k8s.io/kubernetes/staging/README.md).


[developer's documentation]: https://git.k8s.io/community/contributors/devel#readme [Docker environment]: https://docs.docker.com/engine [Go environment]: https://go.dev/doc/install <del> [interactive tutorial]: https://kubernetes.io/docs/tutorials/kubernetes-basics </del> [kubernetes.io]: https://kubernetes.io [Scalable Microservices with Kubernetes]: https://www.udacity.com/course/scalable-microservices-with-kubernetes--ud615 [troubleshooting guide]: https://kubernetes.io/docs/tasks/debug/


return retval, nil }, WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { <del> w := watch.NewFake() </del> <ins> w := watch.NewRaceFreeFake() </ins> if options.ResourceVersion == "1" { go func() { // Close with a "Gone" error when trying to start a watch from the first list


}, } _, _, w, done := NewIndexerInformerWatcher(lw, &corev1.Secret{}) <ins> defer w.Stop() </ins> // Expect secret add select {


} } <del> // This is the spec for the volume that this plugin wraps. var wrappedVolumeSpec = volume.Spec{ // This should be on a tmpfs instead of the local disk; the problem is // charging the memory for the tmpfs to the right cgroup.  We should make // this a tmpfs when we can do the accounting correctly. Volume: &api.Volume{VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}}, </del> <ins> func wrappedVolumeSpec() volume.Spec { // This is the spec for the volume that this plugin wraps. return volume.Spec{ // This should be on a tmpfs instead of the local disk; the problem is // charging the memory for the tmpfs to the right cgroup.  We should make // this a tmpfs when we can do the accounting correctly. Volume: &api.Volume{VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}}, } </ins> } func (b *configMapVolumeMounter) SetUp(fsGroup *int64) error {


var _ volume.VolumePlugin = &downwardAPIPlugin{} <del> var wrappedVolumeSpec = volume.Spec{ Volume: &api.Volume{VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{Medium: api.StorageMediumMemory}}}, </del> <ins> func wrappedVolumeSpec() volume.Spec { return volume.Spec{ Volume: &api.Volume{VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{Medium: api.StorageMediumMemory}}}, } </ins> } func (plugin *downwardAPIPlugin) Init(host volume.VolumeHost) error {


func (b *downwardAPIVolumeMounter) SetUpAt(dir string, fsGroup *int64) error { glog.V(3).Infof("Setting up a downwardAPI volume %v for pod %v/%v at %v", b.volName, b.pod.Namespace, b.pod.Name, dir) // Wrap EmptyDir. Here we rely on the idempotency of the wrapped plugin to avoid repeatedly mounting <del> wrapped, err := b.plugin.host.NewWrapperMounter(b.volName, wrappedVolumeSpec, b.pod, *b.opts) </del> <ins> wrapped, err := b.plugin.host.NewWrapperMounter(b.volName, wrappedVolumeSpec(), b.pod, *b.opts) </ins> if err != nil { glog.Errorf("Couldn't setup downwardAPI volume %v for pod %v/%v: %s", b.volName, b.pod.Namespace, b.pod.Name, err.Error()) return err


var _ volume.VolumePlugin = &gitRepoPlugin{} <del> var wrappedVolumeSpec = volume.Spec{ Volume: &api.Volume{VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}}, </del> <ins> func wrappedVolumeSpec() volume.Spec { return volume.Spec{ Volume: &api.Volume{VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{}}}, } </ins> } const (


} // Wrap EmptyDir, let it do the setup. <del> wrapped, err := b.plugin.host.NewWrapperMounter(b.volName, wrappedVolumeSpec, &b.pod, b.opts) </del> <ins> wrapped, err := b.plugin.host.NewWrapperMounter(b.volName, wrappedVolumeSpec(), &b.pod, b.opts) </ins> if err != nil { return err }


func (c *gitRepoVolumeUnmounter) TearDownAt(dir string) error { // Wrap EmptyDir, let it do the teardown. <del> wrapped, err := c.plugin.host.NewWrapperUnmounter(c.volName, wrappedVolumeSpec, c.podUID) </del> <ins> wrapped, err := c.plugin.host.NewWrapperUnmounter(c.volName, wrappedVolumeSpec(), c.podUID) </ins> if err != nil { return err }


var _ volume.VolumePlugin = &secretPlugin{} <del> var wrappedVolumeSpec = volume.Spec{ Volume: &api.Volume{VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{Medium: api.StorageMediumMemory}}}, </del> <ins> func wrappedVolumeSpec() volume.Spec { return volume.Spec{ Volume: &api.Volume{VolumeSource: api.VolumeSource{EmptyDir: &api.EmptyDirVolumeSource{Medium: api.StorageMediumMemory}}}, } </ins> } func getPath(uid types.UID, volName string, host volume.VolumeHost) string {


glog.V(3).Infof("Setting up volume %v for pod %v at %v", b.volName, b.pod.UID, dir) // Wrap EmptyDir, let it do the setup. <del> wrapped, err := b.plugin.host.NewWrapperMounter(b.volName, wrappedVolumeSpec, &b.pod, *b.opts) </del> <ins> wrapped, err := b.plugin.host.NewWrapperMounter(b.volName, wrappedVolumeSpec(), &b.pod, *b.opts) </ins> if err != nil { return err }


glog.V(3).Infof("Tearing down volume %v for pod %v at %v", c.volName, c.podUID, dir) // Wrap EmptyDir, let it do the teardown. <del> wrapped, err := c.plugin.host.NewWrapperUnmounter(c.volName, wrappedVolumeSpec, c.podUID) </del> <ins> wrapped, err := c.plugin.host.NewWrapperUnmounter(c.volName, wrappedVolumeSpec(), c.podUID) </ins> if err != nil { return err }


- [Debugging clusters](#debugging-clusters) - [Local clusters](#local-clusters) - [Testing against local clusters](#testing-against-local-clusters) <ins> - [Version-skewed and upgrade testing](#version-skewed-and-upgrade-testing) </ins> - [Kinds of tests](#kinds-of-tests) - [Conformance tests](#conformance-tests) - [Defining Conformance Subset](#defining-conformance-subset)


go run hack/e2e.go -v --test --check_node_count=false --test_args="--host=http://127.0.0.1:8080" --ginkgo.focus="Secrets" ``` <ins> ### Version-skewed and upgrade testing We run version-skewed tests to check that newer versions of Kubernetes work similarly enough to older versions.  The general strategy is to cover the following cases: 1. One version of `kubectl` with another version of the cluster and tests (e.g. that v1.2 and v1.4 `kubectl` doesn't break v1.3 tests running against a v1.3 cluster). 1. A newer version of the Kubernetes master with older nodes and tests (e.g. that upgrading a master to v1.3 with nodes at v1.2 still passes v1.2 tests). 1. A newer version of the whole cluster with older tests (e.g. that a cluster upgraded---master and nodes---to v1.3 still passes v1.2 tests). 1. That an upgraded cluster functions the same as a brand-new cluster of the same version (e.g. a cluster upgraded to v1.3 passes the same v1.3 tests as a newly-created v1.3 cluster). [hack/e2e-runner.sh](http://releases.k8s.io/HEAD/hack/jenkins/e2e-runner.sh) is the authoritative source on how to run version-skewed tests, but below is a quick-and-dirty tutorial. ```sh # Assume you have two copies of the Kubernetes repository checked out, at # ./kubernetes and ./kubernetes_old # If using GKE: export KUBERNETES_PROVIDER=gke export CLUSTER_API_VERSION=${OLD_VERSION} # Deploy a cluster at the old version; see above for more details cd ./kubernetes_old go run ./hack/e2e.go -v --up # Upgrade the cluster to the new version # # If using GKE, add --upgrade-target=${NEW_VERSION} # # You can target Feature:MasterUpgrade or Feature:ClusterUpgrade cd ../kubernetes go run ./hack/e2e.go -v --test --check_version_skew=false --test_args="--ginkgo.focus=[Feature:MasterUpgrade]" # Run old tests with new kubectl cd ../kubernetes_old go run ./hack/e2e.go -v --test --test_args="--kubectl-path=$(pwd)/../kubernetes/cluster/kubectl.sh" ``` If you are just testing version-skew, you may want to just deploy at one version and then test at another version, instead of going through the whole upgrade process: ```sh # With the same setup as above # Deploy a cluster at the new version cd ./kubernetes go run ./hack/e2e.go -v --up # Run new tests with old kubectl go run ./hack/e2e.go -v --test --test_args="--kubectl-path=$(pwd)/../kubernetes_old/cluster/kubectl.sh" # Run old tests with new kubectl cd ../kubernetes_old go run ./hack/e2e.go -v --test --test_args="--kubectl-path=$(pwd)/../kubernetes/cluster/kubectl.sh" ``` </ins> ## Kinds of tests We are working on implementing clearer partitioning of our e2e tests to make


configs[ResourceConsumer] = Config{list.PromoterE2eRegistry, "resource-consumer", "1.13"} configs[SdDummyExporter] = Config{list.GcRegistry, "sd-dummy-exporter", "v0.2.0"} configs[VolumeNFSServer] = Config{list.PromoterE2eRegistry, "volume/nfs", "1.3"} <del> configs[VolumeISCSIServer] = Config{list.PromoterE2eRegistry, "volume/iscsi", "2.5"} </del> <ins> configs[VolumeISCSIServer] = Config{list.PromoterE2eRegistry, "volume/iscsi", "2.6"} </ins> configs[VolumeRBDServer] = Config{list.PromoterE2eRegistry, "volume/rbd", "1.0.6"} configs[WindowsServer] = Config{list.MicrosoftRegistry, "windows", "1809"}


// TODO (#113700): patch claim := claim.DeepCopy() claim.Status.ReservedFor = valid <ins> // When a ResourceClaim uses delayed allocation, then it makes sense to // deallocate the claim as soon as the last consumer stops using // it. This ensures that the claim can be allocated again as needed by // some future consumer instead of trying to schedule that consumer // onto the node that was chosen for the previous consumer. It also // releases the underlying resources for use by other claims. // // This has to be triggered by the transition from "was being used" to // "is not used anymore" because a DRA driver is not required to set // `status.reservedFor` together with `status.allocation`, i.e. a claim // that is "currently unused" should not get deallocated. // // This does not matter for claims that were created for a pod. For // those, the resource claim controller will trigger deletion when the // pod is done. However, it doesn't hurt to also trigger deallocation // for such claims and not checking for them keeps this code simpler. if len(valid) == 0 && claim.Spec.AllocationMode == resourcev1alpha2.AllocationModeWaitForFirstConsumer { claim.Status.DeallocationRequested = true } </ins> _, err := ec.kubeClient.ResourceV1alpha2().ResourceClaims(claim.Namespace).UpdateStatus(ctx, claim, metav1.UpdateOptions{}) if err != nil { return err


plugin, err := app.StartPlugin(logger, "/cdi", d.Name, nodename, app.FileOperations{ Create: func(name string, content []byte) error { <del> ginkgo.By(fmt.Sprintf("creating CDI file %s on node %s:n%s", name, nodename, string(content))) </del> <ins> klog.Background().Info("creating CDI file", "node", nodename, "filename", name, "content", string(content)) </ins> return d.createFile(&pod, name, content) }, Remove: func(name string) error { <del> ginkgo.By(fmt.Sprintf("deleting CDI file %s on node %s", name, nodename)) </del> <ins> klog.Background().Info("deleting CDI file", "node", nodename, "filename", name) </ins> return d.removeFile(&pod, name) }, },


ginkgo.Context("with immediate allocation", func() { claimTests(resourcev1alpha2.AllocationModeImmediate) }) <ins> ginkgo.It("must deallocate after use when using delayed allocation", func(ctx context.Context) { parameters := b.parameters() pod := b.podExternal() claim := b.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer) b.create(ctx, parameters, claim, pod) gomega.Eventually(ctx, func(ctx context.Context) (*resourcev1alpha2.ResourceClaim, error) { return b.f.ClientSet.ResourceV1alpha2().ResourceClaims(b.f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{}) }).WithTimeout(f.Timeouts.PodDelete).ShouldNot(gomega.HaveField("Status.Allocation", (*resourcev1alpha2.AllocationResult)(nil))) b.testPod(ctx, f.ClientSet, pod) ginkgo.By(fmt.Sprintf("deleting pod %s", klog.KObj(pod))) framework.ExpectNoError(b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).Delete(ctx, pod.Name, metav1.DeleteOptions{})) ginkgo.By("waiting for claim to get deallocated") gomega.Eventually(ctx, func(ctx context.Context) (*resourcev1alpha2.ResourceClaim, error) { return b.f.ClientSet.ResourceV1alpha2().ResourceClaims(b.f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{}) }).WithTimeout(f.Timeouts.PodDelete).Should(gomega.HaveField("Status.Allocation", (*resourcev1alpha2.AllocationResult)(nil))) }) </ins> }) ginkgo.Context("multiple nodes", func() {


// TODO: move it to k8s.io/utils/net, this is the same as current AddIPOffset() // but using netip.Addr instead of net.IP func addOffsetAddress(address netip.Addr, offset uint64) (netip.Addr, error) { <del> addressBig := big.NewInt(0).SetBytes(address.AsSlice()) r := big.NewInt(0).Add(addressBig, big.NewInt(int64(offset))) addr, ok := netip.AddrFromSlice(r.Bytes()) </del> <ins> addressBytes := address.AsSlice() addressBig := big.NewInt(0).SetBytes(addressBytes) r := big.NewInt(0).Add(addressBig, big.NewInt(int64(offset))).Bytes() // r must be 4 or 16 bytes depending of the ip family // bigInt conversion to bytes will not take this into consideration // and drop the leading zeros, so we have to take this into account. lenDiff := len(addressBytes) - len(r) if lenDiff > 0 { r = append(make([]byte, lenDiff), r...) } else if lenDiff < 0 { return netip.Addr{}, fmt.Errorf("invalid address %v", r) } addr, ok := netip.AddrFromSlice(r) </ins> if !ok { <del> return netip.Addr{}, fmt.Errorf("invalid address %v", r.Bytes()) </del> <ins> return netip.Addr{}, fmt.Errorf("invalid address %v", r) </ins> } return addr, nil }


want:    netip.MustParseAddr("192.168.0.128"), }, { <ins> name:    "IPv4 with leading zeros", address: netip.MustParseAddr("0.0.1.8"), offset:  138, want:    netip.MustParseAddr("0.0.1.146"), }, { name:    "IPv6 with leading zeros", address: netip.MustParseAddr("00fc::1"), offset:  255, want:    netip.MustParseAddr("fc::100"), }, { </ins> name:    "IPv6 offset 255", address: netip.MustParseAddr("2001:db8:1::101"), offset:  255,


subnet: netip.MustParsePrefix("fd00:1:2:3::/64"), want:   netip.MustParseAddr("fd00:1:2:3:FFFF:FFFF:FFFF:FFFF"), }, <ins> { name:   "ipv6 00fc::/112", subnet: netip.MustParsePrefix("00fc::/112"), want:   netip.MustParseAddr("fc::ffff"), }, { name:   "ipv6 fc00::/112", subnet: netip.MustParsePrefix("fc00::/112"), want:   netip.MustParseAddr("fc00::ffff"), }, </ins> } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) {


} } <ins> func TestAllocateNextFC(t *testing.T) { _, cidr, err := netutils.ParseCIDRSloppy("fc::/112") if err != nil { t.Fatal(err) } t.Logf("CIDR %s", cidr) r, err := newTestAllocator(cidr) if err != nil { t.Fatal(err) } defer r.Destroy() ip, err := r.AllocateNext() if err != nil { t.Fatalf("wrong ip %s : %v", ip, err) } t.Log(ip.String()) } </ins> func BenchmarkIPAllocatorAllocateNextIPv4Size1048574(b *testing.B) { _, cidr, err := netutils.ParseCIDRSloppy("10.0.0.0/12") if err != nil {


testingclock "k8s.io/utils/clock/testing" "k8s.io/utils/pointer" <ins> "github.com/google/go-cmp/cmp" </ins> "github.com/stretchr/testify/assert" )


gotNames.Insert(pod.Name) } <del> assert.Equal(t, 0, expectedNames.Difference(gotNames).Len(), "expected %v, got %v", expectedNames.List(), gotNames.List()) assert.Equal(t, 0, gotNames.Difference(expectedNames).Len(), "expected %v, got %v", expectedNames.List(), gotNames.List()) </del> <ins> if diff := cmp.Diff(expectedNames.List(), gotNames.List()); diff != "" { t.Errorf("Active pod names (-want,+got):n%s", diff) } </ins> } func TestSortingActivePods(t *testing.T) {


gotNames.Insert(rs.Name) } <del> assert.Equal(t, 0, expectedNames.Difference(gotNames).Len(), "expected %v, got %v", expectedNames.List(), gotNames.List()) assert.Equal(t, 0, gotNames.Difference(expectedNames).Len(), "expected %v, got %v", expectedNames.List(), gotNames.List()) </del> <ins> if diff := cmp.Diff(expectedNames.List(), gotNames.List()); diff != "" { t.Errorf("Active replica set names (-want,+got):n%s", diff) } </ins> } func TestComputeHash(t *testing.T) {


metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" <ins> "k8s.io/apimachinery/pkg/types" </ins> "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/uuid" utilfeature "k8s.io/apiserver/pkg/util/feature"


} } <del> func newReplicaSet(name string, replicas int) *apps.ReplicaSet { </del> <ins> func newReplicaSet(name string, replicas int, rsUuid types.UID) *apps.ReplicaSet { </ins> return &apps.ReplicaSet{ TypeMeta: metav1.TypeMeta{APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ <del> UID:             uuid.NewUUID(), </del> <ins> UID:             rsUuid, </ins> Name:            name, Namespace:       metav1.NamespaceDefault, ResourceVersion: "18",


} wg.Wait() <del> // Expectations have been surpassed podExp, exists, err := e.GetExpectations(rcKey) assert.NoError(t, err, "Could not get expectations for rc, exists %v and err %v", exists, err) assert.True(t, exists, "Could not get expectations for rc, exists %v and err %v", exists, err) add, del := podExp.GetExpectations() assert.Equal(t, int64(-1), add, "Unexpected pod expectations %#v", podExp) assert.Equal(t, int64(-1), del, "Unexpected pod expectations %#v", podExp) assert.True(t, e.SatisfiedExpectations(logger, rcKey), "Expectations are met but the rc will not sync") // Next round of rc sync, old expectations are cleared e.SetExpectations(logger, rcKey, 1, 2) podExp, exists, err = e.GetExpectations(rcKey) assert.NoError(t, err, "Could not get expectations for rc, exists %v and err %v", exists, err) assert.True(t, exists, "Could not get expectations for rc, exists %v and err %v", exists, err) add, del = podExp.GetExpectations() assert.Equal(t, int64(1), add, "Unexpected pod expectations %#v", podExp) assert.Equal(t, int64(2), del, "Unexpected pod expectations %#v", podExp) // Expectations have expired because of ttl fakeClock.Step(ttl + 1) assert.True(t, e.SatisfiedExpectations(logger, rcKey), "Expectations should have expired but didn't") </del> <ins> tests := []struct { name                      string expectationsToSet         []int expireExpectations        bool wantPodExpectations       []int64 wantExpectationsSatisfied bool }{ { name:                      "Expectations have been surpassed", expireExpectations:        false, wantPodExpectations:       []int64{int64(-1), int64(-1)}, wantExpectationsSatisfied: true, }, { name:                      "Old expectations are cleared because of ttl", expectationsToSet:         []int{1, 2}, expireExpectations:        true, wantPodExpectations:       []int64{int64(1), int64(2)}, wantExpectationsSatisfied: false, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { if len(test.expectationsToSet) > 0 { e.SetExpectations(logger, rcKey, test.expectationsToSet[0], test.expectationsToSet[1]) } podExp, exists, err := e.GetExpectations(rcKey) assert.NoError(t, err, "Could not get expectations for rc, exists %v and err %v", exists, err) assert.True(t, exists, "Could not get expectations for rc, exists %v and err %v", exists, err) add, del := podExp.GetExpectations() assert.Equal(t, test.wantPodExpectations[0], add, "Unexpected pod expectations %#v", podExp) assert.Equal(t, test.wantPodExpectations[1], del, "Unexpected pod expectations %#v", podExp) assert.Equal(t, test.wantExpectationsSatisfied, e.SatisfiedExpectations(logger, rcKey), "Expectations are met but the rc will not sync") if test.expireExpectations { fakeClock.Step(ttl + 1) assert.True(t, e.SatisfiedExpectations(logger, rcKey), "Expectations should have expired but didn't") } }) } </ins> } func TestUIDExpectations(t *testing.T) { logger, _ := ktesting.NewTestContext(t) uidExp := NewUIDTrackingControllerExpectations(NewControllerExpectations()) <del> rcList := []*v1.ReplicationController{ newReplicationController(2), newReplicationController(1), newReplicationController(0), newReplicationController(5), </del> <ins> type test struct { name        string numReplicas int } shuffleTests := func(tests []test) { for i := range tests { j := rand.Intn(i + 1) tests[i], tests[j] = tests[j], tests[i] } </ins> } <del> rcToPods := map[string][]string{} rcKeys := []string{} for i := range rcList { rc := rcList[i] rcName := fmt.Sprintf("rc-%v", i) </del> <ins> getRcDataFrom := func(test test) (string, []string) { rc := newReplicationController(test.numReplicas) rcName := fmt.Sprintf("rc-%v", test.numReplicas) </ins> rc.Name = rcName rc.Spec.Selector[rcName] = rcName podList := newPodList(nil, 5, v1.PodRunning, rc) rcKey, err := KeyFunc(rc) if err != nil { t.Fatalf("Couldn't get key for object %#v: %v", rc, err) } <del> rcKeys = append(rcKeys, rcKey) </del> rcPodNames := []string{} for i := range podList.Items { p := &podList.Items[i] p.Name = fmt.Sprintf("%v-%v", p.Name, rc.Name) rcPodNames = append(rcPodNames, PodKey(p)) } <del> rcToPods[rcKey] = rcPodNames </del> uidExp.ExpectDeletions(logger, rcKey, rcPodNames) <ins> return rcKey, rcPodNames </ins> } <del> for i := range rcKeys { j := rand.Intn(i + 1) rcKeys[i], rcKeys[j] = rcKeys[j], rcKeys[i] </del> <ins> tests := []test{ {name: "Replication controller with 2 replicas", numReplicas: 2}, {name: "Replication controller with 1 replica", numReplicas: 1}, {name: "Replication controller with no replicas", numReplicas: 0}, {name: "Replication controller with 5 replicas", numReplicas: 5}, </ins> } <del> for _, rcKey := range rcKeys { assert.False(t, uidExp.SatisfiedExpectations(logger, rcKey), "Controller %v satisfied expectations before deletion", rcKey) for _, p := range rcToPods[rcKey] { uidExp.DeletionObserved(logger, rcKey, p) } </del> <ins> shuffleTests(tests) for _, test := range tests { t.Run(test.name, func(t *testing.T) { </ins> <del> assert.True(t, uidExp.SatisfiedExpectations(logger, rcKey), "Controller %v didn't satisfy expectations after deletion", rcKey) </del> <ins> rcKey, rcPodNames := getRcDataFrom(test) assert.False(t, uidExp.SatisfiedExpectations(logger, rcKey), "Controller %v satisfied expectations before deletion", rcKey) </ins> <del> uidExp.DeleteExpectations(logger, rcKey) </del> <ins> for _, p := range rcPodNames { uidExp.DeletionObserved(logger, rcKey, p) } </ins> <del> assert.Nil(t, uidExp.GetUIDs(rcKey), "Failed to delete uid expectations for %v", rcKey) } } </del> <ins> assert.True(t, uidExp.SatisfiedExpectations(logger, rcKey), "Controller %v didn't satisfy expectations after deletion", rcKey) </ins> <del> func TestCreatePods(t *testing.T) { ns := metav1.NamespaceDefault body := runtime.EncodeOrDie(clientscheme.Codecs.LegacyCodec(v1.SchemeGroupVersion), &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "empty_pod"}}) fakeHandler := utiltesting.FakeHandler{ StatusCode:   200, ResponseBody: string(body), } testServer := httptest.NewServer(&fakeHandler) defer testServer.Close() clientset := clientset.NewForConfigOrDie(&restclient.Config{Host: testServer.URL, ContentConfig: restclient.ContentConfig{GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}}}) </del> <ins> uidExp.DeleteExpectations(logger, rcKey) </ins> <del> podControl := RealPodControl{ KubeClient: clientset, Recorder:   &record.FakeRecorder{}, </del> <ins> assert.Nil(t, uidExp.GetUIDs(rcKey), "Failed to delete uid expectations for %v", rcKey) }) </ins> } <ins> } func TestCreatePodsWithGenerateName(t *testing.T) { ns := metav1.NamespaceDefault generateName := "hello-" </ins> controllerSpec := newReplicationController(1) controllerRef := metav1.NewControllerRef(controllerSpec, v1.SchemeGroupVersion.WithKind("ReplicationController")) <del> // Make sure createReplica sends a POST to the apiserver with a pod from the controllers pod template err := podControl.CreatePods(context.TODO(), ns, controllerSpec.Spec.Template, controllerSpec, controllerRef) assert.NoError(t, err, "unexpected error: %v", err) expectedPod := v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Labels:       controllerSpec.Spec.Template.Labels, GenerateName: fmt.Sprintf("%s-", controllerSpec.Name), }, Spec: controllerSpec.Spec.Template.Spec, </del> <ins> type test struct { name            string podCreationFunc func(podControl RealPodControl) error wantPod         *v1.Pod </ins> } <del> fakeHandler.ValidateRequest(t, "/api/v1/namespaces/default/pods", "POST", nil) var actualPod = &v1.Pod{} err = json.Unmarshal([]byte(fakeHandler.RequestBody), actualPod) assert.NoError(t, err, "unexpected error: %v", err) assert.True(t, apiequality.Semantic.DeepDerivative(&expectedPod, actualPod), "Body: %s", fakeHandler.RequestBody) } func TestCreatePodsWithGenerateName(t *testing.T) { ns := metav1.NamespaceDefault body := runtime.EncodeOrDie(clientscheme.Codecs.LegacyCodec(v1.SchemeGroupVersion), &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "empty_pod"}}) fakeHandler := utiltesting.FakeHandler{ StatusCode:   200, ResponseBody: string(body), </del> <ins> var tests = []test{ { name: "Create pod", podCreationFunc: func(podControl RealPodControl) error { return podControl.CreatePods(context.TODO(), ns, controllerSpec.Spec.Template, controllerSpec, controllerRef) }, wantPod: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Labels:       controllerSpec.Spec.Template.Labels, GenerateName: fmt.Sprintf("%s-", controllerSpec.Name), }, Spec: controllerSpec.Spec.Template.Spec, }, }, { name: "Create pod with generate name", podCreationFunc: func(podControl RealPodControl) error { // Make sure createReplica sends a POST to the apiserver with a pod from the controllers pod template return podControl.CreatePodsWithGenerateName(context.TODO(), ns, controllerSpec.Spec.Template, controllerSpec, controllerRef, generateName) }, wantPod: &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Labels:          controllerSpec.Spec.Template.Labels, GenerateName:    generateName, OwnerReferences: []metav1.OwnerReference{*controllerRef}, }, Spec: controllerSpec.Spec.Template.Spec, }, }, </ins> } <del> testServer := httptest.NewServer(&fakeHandler) defer testServer.Close() clientset := clientset.NewForConfigOrDie(&restclient.Config{Host: testServer.URL, ContentConfig: restclient.ContentConfig{GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}}}) podControl := RealPodControl{ KubeClient: clientset, Recorder:   &record.FakeRecorder{}, } </del> <ins> for _, test := range tests { t.Run(test.name, func(t *testing.T) { body := runtime.EncodeOrDie(clientscheme.Codecs.LegacyCodec(v1.SchemeGroupVersion), &v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "empty_pod"}}) fakeHandler := utiltesting.FakeHandler{ StatusCode:   200, ResponseBody: string(body), } testServer := httptest.NewServer(&fakeHandler) defer testServer.Close() clientset := clientset.NewForConfigOrDie(&restclient.Config{Host: testServer.URL, ContentConfig: restclient.ContentConfig{GroupVersion: &schema.GroupVersion{Group: "", Version: "v1"}}}) </ins> <del> controllerSpec := newReplicationController(1) controllerRef := metav1.NewControllerRef(controllerSpec, v1.SchemeGroupVersion.WithKind("ReplicationController")) </del> <ins> podControl := RealPodControl{ KubeClient: clientset, Recorder:   &record.FakeRecorder{}, } </ins> <del> // Make sure createReplica sends a POST to the apiserver with a pod from the controllers pod template generateName := "hello-" err := podControl.CreatePodsWithGenerateName(context.TODO(), ns, controllerSpec.Spec.Template, controllerSpec, controllerRef, generateName) assert.NoError(t, err, "unexpected error: %v", err) </del> <ins> err := test.podCreationFunc(podControl) assert.NoError(t, err, "unexpected error: %v", err) </ins> <del> expectedPod := v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Labels:          controllerSpec.Spec.Template.Labels, GenerateName:    generateName, OwnerReferences: []metav1.OwnerReference{*controllerRef}, }, Spec: controllerSpec.Spec.Template.Spec, </del> <ins> fakeHandler.ValidateRequest(t, "/api/v1/namespaces/default/pods", "POST", nil) var actualPod = &v1.Pod{} err = json.Unmarshal([]byte(fakeHandler.RequestBody), actualPod) assert.NoError(t, err, "unexpected error: %v", err) assert.True(t, apiequality.Semantic.DeepDerivative(test.wantPod, actualPod), "Body: %s", fakeHandler.RequestBody) }) </ins> } <del> fakeHandler.ValidateRequest(t, "/api/v1/namespaces/default/pods", "POST", nil) var actualPod = &v1.Pod{} err = json.Unmarshal([]byte(fakeHandler.RequestBody), actualPod) assert.NoError(t, err, "unexpected error: %v", err) assert.True(t, apiequality.Semantic.DeepDerivative(&expectedPod, actualPod), "Body: %s", fakeHandler.RequestBody) </del> } func TestDeletePodsAllowsMissing(t *testing.T) {


func TestActivePodFiltering(t *testing.T) { logger, _ := ktesting.NewTestContext(t) <del> // This rc is not needed by the test, only the newPodList to give the pods labels/a namespace. rc := newReplicationController(0) podList := newPodList(nil, 5, v1.PodRunning, rc) podList.Items[0].Status.Phase = v1.PodSucceeded podList.Items[1].Status.Phase = v1.PodFailed expectedNames := sets.NewString() for _, pod := range podList.Items[2:] { expectedNames.Insert(pod.Name) </del> <ins> type podData struct { podName  string podPhase v1.PodPhase </ins> } <del> var podPointers []*v1.Pod for i := range podList.Items { podPointers = append(podPointers, &podList.Items[i]) </del> <ins> type test struct { name         string pods         []podData wantPodNames []string </ins> } <del> got := FilterActivePods(logger, podPointers) gotNames := sets.NewString() for _, pod := range got { gotNames.Insert(pod.Name) </del> <ins> tests := []test{ { name: "Filters active pods", pods: []podData{ {podName: "pod-1", podPhase: v1.PodSucceeded}, {podName: "pod-2", podPhase: v1.PodFailed}, {podName: "pod-3"}, {podName: "pod-4"}, {podName: "pod-5"}, }, wantPodNames: []string{"pod-3", "pod-4", "pod-5"}, }, </ins> } <del> if diff := cmp.Diff(expectedNames.List(), gotNames.List()); diff != "" { t.Errorf("Active pod names (-want,+got):n%s", diff) </del> <ins> for _, test := range tests { t.Run(test.name, func(t *testing.T) { // This rc is not needed by the test, only the newPodList to give the pods labels/a namespace. rc := newReplicationController(0) podList := newPodList(nil, 5, v1.PodRunning, rc) for idx, testPod := range test.pods { podList.Items[idx].Name = testPod.podName podList.Items[idx].Status.Phase = testPod.podPhase } var podPointers []*v1.Pod for i := range podList.Items { podPointers = append(podPointers, &podList.Items[i]) } got := FilterActivePods(logger, podPointers) gotNames := sets.NewString() for _, pod := range got { gotNames.Insert(pod.Name) } if diff := cmp.Diff(test.wantPodNames, gotNames.List()); diff != "" { t.Errorf("Active pod names (-want,+got):n%s", diff) } }) </ins> } } func TestSortingActivePods(t *testing.T) { <del> numPods := 9 // This rc is not needed by the test, only the newPodList to give the pods labels/a namespace. rc := newReplicationController(0) podList := newPodList(nil, numPods, v1.PodRunning, rc) pods := make([]*v1.Pod, len(podList.Items)) for i := range podList.Items { pods[i] = &podList.Items[i] } // pods[0] is not scheduled yet. pods[0].Spec.NodeName = "" pods[0].Status.Phase = v1.PodPending // pods[1] is scheduled but pending. pods[1].Spec.NodeName = "bar" pods[1].Status.Phase = v1.PodPending // pods[2] is unknown. pods[2].Spec.NodeName = "foo" pods[2].Status.Phase = v1.PodUnknown // pods[3] is running but not ready. pods[3].Spec.NodeName = "foo" pods[3].Status.Phase = v1.PodRunning // pods[4] is running and ready but without LastTransitionTime. </del> now := metav1.Now() <del> pods[4].Spec.NodeName = "foo" pods[4].Status.Phase = v1.PodRunning pods[4].Status.Conditions = []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue}} pods[4].Status.ContainerStatuses = []v1.ContainerStatus{{RestartCount: 3}, {RestartCount: 0}} // pods[5] is running and ready and with LastTransitionTime. pods[5].Spec.NodeName = "foo" pods[5].Status.Phase = v1.PodRunning pods[5].Status.Conditions = []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue, LastTransitionTime: now}} pods[5].Status.ContainerStatuses = []v1.ContainerStatus{{RestartCount: 3}, {RestartCount: 0}} // pods[6] is running ready for a longer time than pods[5]. </del> then := metav1.Time{Time: now.AddDate(0, -1, 0)} <del> pods[6].Spec.NodeName = "foo" pods[6].Status.Phase = v1.PodRunning pods[6].Status.Conditions = []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue, LastTransitionTime: then}} pods[6].Status.ContainerStatuses = []v1.ContainerStatus{{RestartCount: 3}, {RestartCount: 0}} // pods[7] has lower container restart count than pods[6]. pods[7].Spec.NodeName = "foo" pods[7].Status.Phase = v1.PodRunning pods[7].Status.Conditions = []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue, LastTransitionTime: then}} pods[7].Status.ContainerStatuses = []v1.ContainerStatus{{RestartCount: 2}, {RestartCount: 1}} pods[7].CreationTimestamp = now // pods[8] is older than pods[7]. pods[8].Spec.NodeName = "foo" pods[8].Status.Phase = v1.PodRunning pods[8].Status.Conditions = []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue, LastTransitionTime: then}} pods[8].Status.ContainerStatuses = []v1.ContainerStatus{{RestartCount: 2}, {RestartCount: 1}} pods[8].CreationTimestamp = then getOrder := func(pods []*v1.Pod) []string { names := make([]string, len(pods)) for i := range pods { names[i] = pods[i].Name } return names </del> <ins> tests := []struct { name      string pods      []v1.Pod wantOrder []string }{ { name: "Sorts by active pod", pods: []v1.Pod{ { ObjectMeta: metav1.ObjectMeta{Name: "unscheduled"}, Spec:       v1.PodSpec{NodeName: ""}, Status:     v1.PodStatus{Phase: v1.PodPending}, }, { ObjectMeta: metav1.ObjectMeta{Name: "scheduledButPending"}, Spec:       v1.PodSpec{NodeName: "bar"}, Status:     v1.PodStatus{Phase: v1.PodPending}, }, { ObjectMeta: metav1.ObjectMeta{Name: "unknownPhase"}, Spec:       v1.PodSpec{NodeName: "foo"}, Status:     v1.PodStatus{Phase: v1.PodUnknown}, }, { ObjectMeta: metav1.ObjectMeta{Name: "runningButNotReady"}, Spec:       v1.PodSpec{NodeName: "foo"}, Status:     v1.PodStatus{Phase: v1.PodRunning}, }, { ObjectMeta: metav1.ObjectMeta{Name: "runningNoLastTransitionTime"}, Spec:       v1.PodSpec{NodeName: "foo"}, Status: v1.PodStatus{ Phase:             v1.PodRunning, Conditions:        []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue}}, ContainerStatuses: []v1.ContainerStatus{{RestartCount: 3}, {RestartCount: 0}}, }, }, { ObjectMeta: metav1.ObjectMeta{Name: "runningWithLastTransitionTime"}, Spec:       v1.PodSpec{NodeName: "foo"}, Status: v1.PodStatus{ Phase:             v1.PodRunning, Conditions:        []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue, LastTransitionTime: now}}, ContainerStatuses: []v1.ContainerStatus{{RestartCount: 3}, {RestartCount: 0}}, }, }, { ObjectMeta: metav1.ObjectMeta{Name: "runningLongerTime"}, Spec:       v1.PodSpec{NodeName: "foo"}, Status: v1.PodStatus{ Phase:             v1.PodRunning, Conditions:        []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue, LastTransitionTime: then}}, ContainerStatuses: []v1.ContainerStatus{{RestartCount: 3}, {RestartCount: 0}}, }, }, { ObjectMeta: metav1.ObjectMeta{Name: "lowerContainerRestartCount", CreationTimestamp: now}, Spec:       v1.PodSpec{NodeName: "foo"}, Status: v1.PodStatus{ Phase:             v1.PodRunning, Conditions:        []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue, LastTransitionTime: then}}, ContainerStatuses: []v1.ContainerStatus{{RestartCount: 2}, {RestartCount: 1}}, }, }, { ObjectMeta: metav1.ObjectMeta{Name: "oldest", CreationTimestamp: then}, Spec:       v1.PodSpec{NodeName: "foo"}, Status: v1.PodStatus{ Phase:             v1.PodRunning, Conditions:        []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue, LastTransitionTime: then}}, ContainerStatuses: []v1.ContainerStatus{{RestartCount: 2}, {RestartCount: 1}}, }, }, }, wantOrder: []string{ "unscheduled", "scheduledButPending", "unknownPhase", "runningButNotReady", "runningNoLastTransitionTime", "runningWithLastTransitionTime", "runningLongerTime", "lowerContainerRestartCount", "oldest", }, }, </ins> } <del> expected := getOrder(pods) </del> <ins> for _, test := range tests { t.Run(test.name, func(t *testing.T) { numPods := len(test.pods) </ins> <del> for i := 0; i < 20; i++ { idx := rand.Perm(numPods) randomizedPods := make([]*v1.Pod, numPods) for j := 0; j < numPods; j++ { randomizedPods[j] = pods[idx[j]] } sort.Sort(ActivePods(randomizedPods)) actual := getOrder(randomizedPods) </del> <ins> for i := 0; i < 20; i++ { idx := rand.Perm(numPods) randomizedPods := make([]*v1.Pod, numPods) for j := 0; j < numPods; j++ { randomizedPods[j] = &test.pods[idx[j]] } </ins> <del> assert.EqualValues(t, expected, actual, "expected %v, got %v", expected, actual) </del> <ins> sort.Sort(ActivePods(randomizedPods)) gotOrder := make([]string, len(randomizedPods)) for i := range randomizedPods { gotOrder[i] = randomizedPods[i].Name } if diff := cmp.Diff(test.wantOrder, gotOrder); diff != "" { t.Errorf("Sorted active pod names (-want,+got):n%s", diff) } } }) </ins> } }


{p1: unscheduled5Hours, p2: unscheduled8Hours}, {p1: ready5Hours, p2: ready10Hours}, } <del> for _, tc := range equalityTests { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LogarithmicScaleDown, !tc.disableLogarithmicScaleDown)() if tc.p2 == nil { tc.p2 = tc.p1 } podsWithRanks := ActivePodsWithRanks{ Pods: []*v1.Pod{tc.p1, tc.p2}, Rank: []int{1, 1}, Now:  now, } if podsWithRanks.Less(0, 1) || podsWithRanks.Less(1, 0) { t.Errorf("expected pod %q to be equivalent to %q", tc.p1.Name, tc.p2.Name) } </del> <ins> for i, test := range equalityTests { t.Run(fmt.Sprintf("Equality tests %d", i), func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LogarithmicScaleDown, !test.disableLogarithmicScaleDown)() if test.p2 == nil { test.p2 = test.p1 } podsWithRanks := ActivePodsWithRanks{ Pods: []*v1.Pod{test.p1, test.p2}, Rank: []int{1, 1}, Now:  now, } if podsWithRanks.Less(0, 1) || podsWithRanks.Less(1, 0) { t.Errorf("expected pod %q to be equivalent to %q", test.p1.Name, test.p2.Name) } }) </ins> } type podWithRank struct { pod  *v1.Pod rank int


{lesser: podWithRank{highPodDeletionCost, 2}, greater: podWithRank{lowPodDeletionCost, 1}, disablePodDeletioncost: true}, {lesser: podWithRank{ready2Hours, 1}, greater: podWithRank{ready5Hours, 1}}, } for i, test := range inequalityTests { <del> t.Run(fmt.Sprintf("test%d", i), func(t *testing.T) { </del> <ins> t.Run(fmt.Sprintf("Inequality tests %d", i), func(t *testing.T) { </ins> defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodDeletionCost, !test.disablePodDeletioncost)() defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.LogarithmicScaleDown, !test.disableLogarithmicScaleDown)()


} func TestActiveReplicaSetsFiltering(t *testing.T) { <del> var replicaSets []*apps.ReplicaSet replicaSets = append(replicaSets, newReplicaSet("zero", 0)) replicaSets = append(replicaSets, nil) replicaSets = append(replicaSets, newReplicaSet("foo", 1)) replicaSets = append(replicaSets, newReplicaSet("bar", 2)) expectedNames := sets.NewString() for _, rs := range replicaSets[2:] { expectedNames.Insert(rs.Name) } got := FilterActiveReplicaSets(replicaSets) gotNames := sets.NewString() for _, rs := range got { gotNames.Insert(rs.Name) </del> <ins> rsUuid := uuid.NewUUID() tests := []struct { name            string replicaSets     []*apps.ReplicaSet wantReplicaSets []*apps.ReplicaSet }{ { name: "Filters active replica sets", replicaSets: []*apps.ReplicaSet{ newReplicaSet("zero", 0, rsUuid), nil, newReplicaSet("foo", 1, rsUuid), newReplicaSet("bar", 2, rsUuid), }, wantReplicaSets: []*apps.ReplicaSet{ newReplicaSet("foo", 1, rsUuid), newReplicaSet("bar", 2, rsUuid), }, }, </ins> } <del> if diff := cmp.Diff(expectedNames.List(), gotNames.List()); diff != "" { t.Errorf("Active replica set names (-want,+got):n%s", diff) </del> <ins> for _, test := range tests { t.Run(test.name, func(t *testing.T) { gotReplicaSets := FilterActiveReplicaSets(test.replicaSets) if diff := cmp.Diff(test.wantReplicaSets, gotReplicaSets); diff != "" { t.Errorf("Active replica set names (-want,+got):n%s", diff) } }) </ins> } }


return false, pod, fmt.Errorf("invalid pod: %#v", obj) } <ins> if newPod.Name == "" { return true, pod, fmt.Errorf("invalid pod: name is needed for the pod") } </ins> // Apply default values and validate the pod. if err = defaultFn(newPod); err != nil { return true, pod, err


// Apply default values and validate pods. for i := range newPods.Items { newPod := &newPods.Items[i] <ins> if newPod.Name == "" { return true, pods, fmt.Errorf("invalid pod: name is needed for the pod") } </ins> if err = defaultFn(newPod); err != nil { return true, pods, err }


driver := NewDriver(f, nodes, func() app.Resources { return app.Resources{ NodeLocal:      true, <del> MaxAllocations: 2, </del> <ins> MaxAllocations: 1, </ins> Nodes:          nodes.NodeNames, } }) driver2 := NewDriver(f, nodes, func() app.Resources { return app.Resources{ NodeLocal:      true, <del> MaxAllocations: 2, </del> <ins> MaxAllocations: 1, </ins> Nodes:          nodes.NodeNames, AllocateWrapper: func(


parameters1 := b.parameters() parameters2 := b2.parameters() <del> pod1 := b.podExternal() pod2 := b2.podExternal() </del> <ins> // Order is relevant here: each pod must be matched with its own claim. </ins> pod1claim1 := b.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer) <ins> pod1 := b.podExternal() </ins> pod2claim1 := b2.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer) <del> pod1claim2 := b2.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer) pod1claim3 := b2.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer) </del> <ins> pod2 := b2.podExternal() // Add another claim to pod1. pod1claim2 := b2.externalClaim(resourcev1alpha2.AllocationModeWaitForFirstConsumer) </ins> pod1.Spec.ResourceClaims = append(pod1.Spec.ResourceClaims, v1.PodResourceClaim{ <del> Name: "claim-other1", </del> <ins> Name: "claim-other", </ins> Source: v1.ClaimSource{ ResourceClaimName: &pod1claim2.Name, }, }, <del> v1.PodResourceClaim{ Name: "claim-other2", Source: v1.ClaimSource{ ResourceClaimName: &pod1claim3.Name, }, }, </del> ) <del> // Block on the second, third external claim from driver2 that is to be allocated. </del> <ins> // Allocating the second claim in pod1 has to wait until pod2 has // consumed the available resources on the node. </ins> blockClaim, cancelBlockClaim := context.WithCancel(ctx) defer cancelBlockClaim() <del> b2.create(ctx, parameters2, pod1claim2, pod1claim3) b.create(ctx, parameters1, pod1claim1, pod1) </del> allocateWrapper2 = func(ctx context.Context, claimAllocations []*controller.ClaimAllocation, selectedNode string,


claimAllocations []*controller.ClaimAllocation, selectedNode string), ) { <del> // pod1 will have only external-claim[2-3], it has to wait // for pod2 to consume resources from driver2 if claimAllocations[0].Claim.Name != "external-claim-other" { </del> <ins> if claimAllocations[0].Claim.Name == pod1claim2.Name { </ins> <-blockClaim.Done() } handler(ctx, claimAllocations, selectedNode) return } <ins> b.create(ctx, parameters1, parameters2, pod1claim1, pod1claim2, pod1) </ins> ginkgo.By("waiting for one claim from driver1 to be allocated") var nodeSelector *v1.NodeSelector gomega.Eventually(ctx, func(ctx context.Context) (int, error) {


b2.create(ctx, pod2claim1, pod2) framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod2), "start pod 2") <del> // Allow allocation of pod1 claim2, claim3 to proceed. It should fail now </del> <ins> // Allow allocation of second claim in pod1 to proceed. It should fail now </ins> // and the other node must be used instead, after deallocating // the first claim. ginkgo.By("move first pod to other node")


} } <del> if !probeGracePeriodInUse(oldPodSpec) { // Set pod-level terminationGracePeriodSeconds to nil if the feature is disabled and it is not used VisitContainers(podSpec, AllContainers, func(c *api.Container, containerType ContainerType) bool { if c.LivenessProbe != nil { c.LivenessProbe.TerminationGracePeriodSeconds = nil } if c.StartupProbe != nil { c.StartupProbe.TerminationGracePeriodSeconds = nil } // cannot be set for readiness probes return true }) } </del> // If the feature is disabled and not in use, drop the hostUsers field. if !utilfeature.DefaultFeatureGate.Enabled(features.UserNamespacesSupport) && !hostUsersInUse(oldPodSpec) { // Drop the field in podSpec only if SecurityContext is not nil.


return false } <del> // probeGracePeriodInUse returns true if the pod spec is non-nil and has a probe that makes use // of the probe-level terminationGracePeriodSeconds feature func probeGracePeriodInUse(podSpec *api.PodSpec) bool { if podSpec == nil { return false } var inUse bool VisitContainers(podSpec, AllContainers, func(c *api.Container, containerType ContainerType) bool { // cannot be set for readiness probes if (c.LivenessProbe != nil && c.LivenessProbe.TerminationGracePeriodSeconds != nil) || (c.StartupProbe != nil && c.StartupProbe.TerminationGracePeriodSeconds != nil) { inUse = true return false } return true }) return inUse } </del> // schedulingGatesInUse returns true if the pod spec is non-nil and it has SchedulingGates field set. func schedulingGatesInUse(podSpec *api.PodSpec) bool { if podSpec == nil {


"k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" <ins> "k8s.io/apimachinery/pkg/util/sets" </ins> "k8s.io/apimachinery/pkg/util/validation/field" celconfig "k8s.io/apiserver/pkg/apis/cel" "k8s.io/apiserver/pkg/features"


} // WarningsOnCreate returns warnings for the creation of the given object. <del> func (customResourceStrategy) WarningsOnCreate(ctx context.Context, obj runtime.Object) []string { return nil </del> <ins> func (a customResourceStrategy) WarningsOnCreate(ctx context.Context, obj runtime.Object) []string { return generateWarningsFromObj(obj, nil) } func generateWarningsFromObj(obj, old runtime.Object) []string { var allWarnings []string fldPath := field.NewPath("metadata", "finalizers") newObjAccessor, err := meta.Accessor(obj) if err != nil { return allWarnings } newAdded := sets.NewString(newObjAccessor.GetFinalizers()...) if old != nil { oldObjAccessor, err := meta.Accessor(old) if err != nil { return allWarnings } newAdded = newAdded.Difference(sets.NewString(oldObjAccessor.GetFinalizers()...)) } for _, finalizer := range newAdded.List() { allWarnings = append(allWarnings, validateKubeFinalizerName(finalizer, fldPath)...) } return allWarnings </ins> } // Canonicalize normalizes the object after validation.


} // WarningsOnUpdate returns warnings for the given update. <del> func (customResourceStrategy) WarningsOnUpdate(ctx context.Context, obj, old runtime.Object) []string { return nil </del> <ins> func (a customResourceStrategy) WarningsOnUpdate(ctx context.Context, obj, old runtime.Object) []string { return generateWarningsFromObj(obj, old) </ins> } // GetAttrs returns labels and fields of a given object for filtering purposes.


"math" "strings" <ins> corev1 "k8s.io/api/core/v1" </ins> "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/validation" <ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" </ins> "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" <ins> "k8s.io/apimachinery/pkg/util/sets" apimachineryvalidation "k8s.io/apimachinery/pkg/util/validation" </ins> "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"


return allErrs } <del> // WarningsOnUpdate returns warnings for the given update. func (customResourceValidator) WarningsOnUpdate(ctx context.Context, obj, old runtime.Object) []string { return nil </del> <ins> var standardFinalizers = sets.NewString( metav1.FinalizerOrphanDependents, metav1.FinalizerDeleteDependents, string(corev1.FinalizerKubernetes), ) func validateKubeFinalizerName(stringValue string, fldPath *field.Path) []string { var allWarnings []string for _, msg := range apimachineryvalidation.IsQualifiedName(stringValue) { allWarnings = append(allWarnings, fmt.Sprintf("%s: %q: %s", fldPath.String(), stringValue, msg)) } if len(strings.Split(stringValue, "/")) == 1 { if !standardFinalizers.Has(stringValue) { allWarnings = append(allWarnings, fmt.Sprintf("%s: %q: prefer a domain-qualified finalizer name to avoid accidental conflicts with other finalizer writers", fldPath.String(), stringValue)) } } return allWarnings </ins> } func (a customResourceValidator) ValidateStatusUpdate(ctx context.Context, obj, old runtime.Object, scale *apiextensions.CustomResourceSubresourceScale) field.ErrorList {


} }` <ins> crdApplyFinalizerBody = ` { "apiVersion": "%s", "kind": "%s", "metadata": { "name": "%s", "finalizers": %s }, "spec": { "knownField1": "val1", "ports": [{ "name": "portName", "containerPort": 8080, "protocol": "TCP", "hostPort": 8082 }], "embeddedObj": { "apiVersion": "v1", "kind": "ConfigMap", "metadata": { "name": "my-cm", "namespace": "my-ns" } } } }` </ins> patchYAMLBody = ` apiVersion: %s kind: %s metadata: name: %s finalizers: <del> - test-finalizer </del> <ins> - test/finalizer </ins> spec: cronSpec: "* * * * */5" ports:


t.Run("PatchCRDSchemaless", func(t *testing.T) { testFieldValidationPatchCRDSchemaless(t, rest, schemalessGVK, schemalessGVR) }) t.Run("ApplyCreateCRDSchemaless", func(t *testing.T) { testFieldValidationApplyCreateCRDSchemaless(t, rest, schemalessGVK, schemalessGVR) }) t.Run("ApplyUpdateCRDSchemaless", func(t *testing.T) { testFieldValidationApplyUpdateCRDSchemaless(t, rest, schemalessGVK, schemalessGVR) }) <ins> t.Run("testFinalizerValidationApplyCreateCRD", func(t *testing.T) { testFinalizerValidationApplyCreateAndUpdateCRD(t, rest, schemalessGVK, schemalessGVR) }) </ins> } // testFieldValidationPost tests POST requests containing unknown fields with


} } <ins> func testFinalizerValidationApplyCreateAndUpdateCRD(t *testing.T, rest rest.Interface, gvk schema.GroupVersionKind, gvr schema.GroupVersionResource) { var testcases = []struct { name                 string finalizer            []string updatedFinalizer     []string opts                 metav1.PatchOptions expectUpdateWarnings []string expectCreateWarnings []string }{ { name:      "create-crd-with-invalid-finalizer", finalizer: []string{"invalid-finalizer"}, expectCreateWarnings: []string{ `metadata.finalizers: "invalid-finalizer": prefer a domain-qualified finalizer name to avoid accidental conflicts with other finalizer writers`, }, }, { name:      "create-crd-with-valid-finalizer", finalizer: []string{"kubernetes.io/valid-finalizer"}, }, { name:             "update-crd-with-invalid-finalizer", finalizer:        []string{"invalid-finalizer"}, updatedFinalizer: []string{"another-invalid-finalizer"}, expectCreateWarnings: []string{ `metadata.finalizers: "invalid-finalizer": prefer a domain-qualified finalizer name to avoid accidental conflicts with other finalizer writers`, }, expectUpdateWarnings: []string{ `metadata.finalizers: "another-invalid-finalizer": prefer a domain-qualified finalizer name to avoid accidental conflicts with other finalizer writers`, }, }, { name:             "update-crd-with-valid-finalizer", finalizer:        []string{"kubernetes.io/valid-finalizer"}, updatedFinalizer: []string{"kubernetes.io/another-valid-finalizer"}, }, { name:             "update-crd-with-valid-finalizer-leaving-an-existing-invalid-finalizer", finalizer:        []string{"invalid-finalizer"}, updatedFinalizer: []string{"kubernetes.io/another-valid-finalizer"}, expectCreateWarnings: []string{ `metadata.finalizers: "invalid-finalizer": prefer a domain-qualified finalizer name to avoid accidental conflicts with other finalizer writers`, }, }, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { kind := gvk.Kind apiVersion := gvk.Group + "/" + gvk.Version // create the CR as specified by the test case name := fmt.Sprintf("apply-create-crd-%s", tc.name) finalizerVal, _ := json.Marshal(tc.finalizer) applyCreateBody := []byte(fmt.Sprintf(crdApplyFinalizerBody, apiVersion, kind, name, finalizerVal)) req := rest.Patch(types.ApplyPatchType). AbsPath("/apis", gvr.Group, gvr.Version, gvr.Resource). Name(name). Param("fieldManager", "apply_test"). VersionedParams(&tc.opts, metav1.ParameterCodec) result := req.Body(applyCreateBody).Do(context.TODO()) if result.Error() != nil { t.Fatalf("unexpected error: %v", result.Error()) } if len(result.Warnings()) != len(tc.expectCreateWarnings) { for _, r := range result.Warnings() { t.Logf("received warning: %v", r) } t.Fatalf("unexpected number of warnings, expected: %d, got: %d", len(tc.expectCreateWarnings), len(result.Warnings())) } for i, expectedWarning := range tc.expectCreateWarnings { if expectedWarning != result.Warnings()[i].Text { t.Fatalf("expected warning: %s, got warning: %s", expectedWarning, result.Warnings()[i].Text) } } if len(tc.updatedFinalizer) != 0 { finalizerVal, _ := json.Marshal(tc.updatedFinalizer) applyUpdateBody := []byte(fmt.Sprintf(crdApplyFinalizerBody, apiVersion, kind, name, finalizerVal)) updateReq := rest.Patch(types.ApplyPatchType). AbsPath("/apis", gvr.Group, gvr.Version, gvr.Resource). Name(name). Param("fieldManager", "apply_test"). VersionedParams(&tc.opts, metav1.ParameterCodec) result = updateReq.Body(applyUpdateBody).Do(context.TODO()) if result.Error() != nil { t.Fatalf("unexpected error: %v", result.Error()) } if len(result.Warnings()) != len(tc.expectUpdateWarnings) { t.Fatalf("unexpected number of warnings, expected: %d, got: %d", len(tc.expectUpdateWarnings), len(result.Warnings())) } for i, expectedWarning := range tc.expectUpdateWarnings { if expectedWarning != result.Warnings()[i].Text { t.Fatalf("expected warning: %s, got warning: %s", expectedWarning, result.Warnings()[i].Text) } } } }) } } </ins> func setupCRD(t testing.TB, config *rest.Config, apiGroup string, schemaless bool) *apiextensionsv1.CustomResourceDefinition { apiExtensionClient, err := apiextensionsclient.NewForConfig(config) if err != nil {


metadata: name: %s finalizers: <del> - test-finalizer </del> <ins> - test/finalizer </ins> spec: cronSpec: "* * * * */5" ports:


Labels: testSvcLabels, }, Spec: v1.ServiceSpec{ <del> Type: "ClusterIP", </del> <ins> Type: "LoadBalancer", </ins> Ports: []v1.ServicePort{{ Name:       "http", Protocol:   v1.ProtocolTCP, Port:       int32(80), TargetPort: intstr.FromInt(80), }}, <ins> LoadBalancerClass: utilpointer.String("example.com/internal-vip"), </ins> }, } _, err = cs.CoreV1().Services(ns).Create(ctx, &testService, metav1.CreateOptions{})


// Error returns detailed information of why the pod failed to fit on each node. // A message format is "0/X nodes are available: <PreFilterMsg>. <FilterMsg>. <PostFilterMsg>." func (f *FitError) Error() string { <del> reasons := make(map[string]int) for _, status := range f.Diagnosis.NodeToStatusMap { for _, reason := range status.Reasons() { reasons[reason]++ } } </del> reasonMsg := fmt.Sprintf(NoNodeAvailableMsg+":", f.NumAllNodes) <del> // Add the messages from PreFilter plugins to reasonMsg. </del> preFilterMsg := f.Diagnosis.PreFilterMsg if preFilterMsg != "" { <ins> // PreFilter plugin returns unschedulable. // Add the messages from PreFilter plugins to reasonMsg. </ins> reasonMsg += fmt.Sprintf(SeparatorFormat, preFilterMsg) } <del> sortReasonsHistogram := func() []string { var reasonStrings []string for k, v := range reasons { reasonStrings = append(reasonStrings, fmt.Sprintf("%v %v", v, k)) </del> <ins> if preFilterMsg == "" { // the scheduling cycle went through PreFilter extension point successfully. // // When the prefilter plugin returns unschedulable, // the scheduling framework inserts the same unschedulable status to all nodes in NodeToStatusMap. // So, we shouldn't add the message from NodeToStatusMap when the PreFilter failed. // Otherwise, we will have duplicated reasons in the error message. reasons := make(map[string]int) for _, status := range f.Diagnosis.NodeToStatusMap { for _, reason := range status.Reasons() { reasons[reason]++ } } sortReasonsHistogram := func() []string { var reasonStrings []string for k, v := range reasons { reasonStrings = append(reasonStrings, fmt.Sprintf("%v %v", v, k)) } sort.Strings(reasonStrings) return reasonStrings } sortedFilterMsg := sortReasonsHistogram() if len(sortedFilterMsg) != 0 { reasonMsg += fmt.Sprintf(SeparatorFormat, strings.Join(sortedFilterMsg, ", ")) </ins> } <del> sort.Strings(reasonStrings) return reasonStrings } sortedFilterMsg := sortReasonsHistogram() if len(sortedFilterMsg) != 0 { reasonMsg += fmt.Sprintf(SeparatorFormat, strings.Join(sortedFilterMsg, ", ")) </del> } // Add the messages from PostFilter plugins to reasonMsg. <ins> // We can add this message regardless of whether the scheduling cycle fails at PreFilter or Filter // since we may run PostFilter (if enabled) in both cases. </ins> postFilterMsg := f.Diagnosis.PostFilterMsg if postFilterMsg != "" { reasonMsg += fmt.Sprintf(SeparatorFormat, postFilterMsg)


numAllNodes: 3, diagnosis: Diagnosis{ PreFilterMsg: "Node(s) failed PreFilter plugin FalsePreFilter", <ins> NodeToStatusMap: NodeToStatusMap{ // They're inserted by the framework. // We don't include them in the reason message because they'd be just duplicates. "node1": NewStatus(Unschedulable, "Node(s) failed PreFilter plugin FalsePreFilter"), "node2": NewStatus(Unschedulable, "Node(s) failed PreFilter plugin FalsePreFilter"), "node3": NewStatus(Unschedulable, "Node(s) failed PreFilter plugin FalsePreFilter"), }, </ins> }, wantReasonMsg: "0/3 nodes are available: Node(s) failed PreFilter plugin FalsePreFilter.", }, { <ins> name:        "nodes failed Prefilter plugin and the preemption also failed", numAllNodes: 3, diagnosis: Diagnosis{ PreFilterMsg: "Node(s) failed PreFilter plugin FalsePreFilter", NodeToStatusMap: NodeToStatusMap{ // They're inserted by the framework. // We don't include them in the reason message because they'd be just duplicates. "node1": NewStatus(Unschedulable, "Node(s) failed PreFilter plugin FalsePreFilter"), "node2": NewStatus(Unschedulable, "Node(s) failed PreFilter plugin FalsePreFilter"), "node3": NewStatus(Unschedulable, "Node(s) failed PreFilter plugin FalsePreFilter"), }, // PostFilterMsg will be included. PostFilterMsg: "Error running PostFilter plugin FailedPostFilter", }, wantReasonMsg: "0/3 nodes are available: Node(s) failed PreFilter plugin FalsePreFilter. Error running PostFilter plugin FailedPostFilter.", }, { </ins> name:        "nodes failed one Filter plugin with an empty PostFilterMsg", numAllNodes: 3, diagnosis: Diagnosis{


if !s.IsUnschedulable() { return nil, diagnosis, s.AsError() } <ins> // All nodes in NodeToStatusMap will have the same status so that they can be handled in the preemption. // Some non trivial refactoring is needed to avoid this copy. for _, n := range allNodes { diagnosis.NodeToStatusMap[n.Node().Name] = s } </ins> // Record the messages from PreFilter in Diagnosis.PreFilterMsg. msg := s.Message() diagnosis.PreFilterMsg = msg


Pod:         st.MakePod().Name("ignore").UID("ignore").PVC("unknownPVC").Obj(), NumAllNodes: 2, Diagnosis: framework.Diagnosis{ <del> NodeToStatusMap:      framework.NodeToStatusMap{}, </del> <ins> NodeToStatusMap: framework.NodeToStatusMap{ "node1": framework.NewStatus(framework.UnschedulableAndUnresolvable, `persistentvolumeclaim "unknownPVC" not found`).WithFailedPlugin("VolumeBinding"), "node2": framework.NewStatus(framework.UnschedulableAndUnresolvable, `persistentvolumeclaim "unknownPVC" not found`).WithFailedPlugin("VolumeBinding"), }, </ins> PreFilterMsg:         `persistentvolumeclaim "unknownPVC" not found`, UnschedulablePlugins: sets.New(volumebinding.Name), },


Pod:         st.MakePod().Name("ignore").UID("ignore").Namespace(v1.NamespaceDefault).PVC("existingPVC").Obj(), NumAllNodes: 2, Diagnosis: framework.Diagnosis{ <del> NodeToStatusMap:      framework.NodeToStatusMap{}, </del> <ins> NodeToStatusMap: framework.NodeToStatusMap{ "node1": framework.NewStatus(framework.UnschedulableAndUnresolvable, `persistentvolumeclaim "existingPVC" is being deleted`).WithFailedPlugin("VolumeBinding"), "node2": framework.NewStatus(framework.UnschedulableAndUnresolvable, `persistentvolumeclaim "existingPVC" is being deleted`).WithFailedPlugin("VolumeBinding"), }, </ins> PreFilterMsg:         `persistentvolumeclaim "existingPVC" is being deleted`, UnschedulablePlugins: sets.New(volumebinding.Name), },


Pod:         st.MakePod().Name("test-prefilter").UID("test-prefilter").Obj(), NumAllNodes: 2, Diagnosis: framework.Diagnosis{ <del> NodeToStatusMap:      framework.NodeToStatusMap{}, </del> <ins> NodeToStatusMap: framework.NodeToStatusMap{ "1": framework.NewStatus(framework.UnschedulableAndUnresolvable, "injected unschedulable status").WithFailedPlugin("FakePreFilter"), "2": framework.NewStatus(framework.UnschedulableAndUnresolvable, "injected unschedulable status").WithFailedPlugin("FakePreFilter"), }, </ins> PreFilterMsg:         "injected unschedulable status", UnschedulablePlugins: sets.New("FakePreFilter"), },


Pod:         st.MakePod().Name("test-prefilter").UID("test-prefilter").Obj(), NumAllNodes: 3, Diagnosis: framework.Diagnosis{ <del> NodeToStatusMap:      framework.NodeToStatusMap{}, </del> <ins> NodeToStatusMap: framework.NodeToStatusMap{ "node1": framework.NewStatus(framework.Unschedulable, "node(s) didn't satisfy plugin(s) [FakePreFilter2 FakePreFilter3] simultaneously"), "node2": framework.NewStatus(framework.Unschedulable, "node(s) didn't satisfy plugin(s) [FakePreFilter2 FakePreFilter3] simultaneously"), "node3": framework.NewStatus(framework.Unschedulable, "node(s) didn't satisfy plugin(s) [FakePreFilter2 FakePreFilter3] simultaneously"), }, </ins> UnschedulablePlugins: sets.Set[string]{}, PreFilterMsg:         "node(s) didn't satisfy plugin(s) [FakePreFilter2 FakePreFilter3] simultaneously", },


Pod:         st.MakePod().Name("test-prefilter").UID("test-prefilter").Obj(), NumAllNodes: 1, Diagnosis: framework.Diagnosis{ <del> NodeToStatusMap:      framework.NodeToStatusMap{}, </del> <ins> NodeToStatusMap: framework.NodeToStatusMap{ "node1": framework.NewStatus(framework.Unschedulable, "node(s) didn't satisfy plugin FakePreFilter2"), }, </ins> UnschedulablePlugins: sets.Set[string]{}, PreFilterMsg:         "node(s) didn't satisfy plugin FakePreFilter2", },


} // Complete completes all the required options. <del> func (o *Options) Complete() error { </del> <ins> func (o *Options) Complete(fs *pflag.FlagSet) error { </ins> if len(o.ConfigFile) == 0 && len(o.WriteConfigTo) == 0 { o.config.HealthzBindAddress = addressFromDeprecatedFlags(o.config.HealthzBindAddress, o.healthzPort) o.config.MetricsBindAddress = addressFromDeprecatedFlags(o.config.MetricsBindAddress, o.metricsPort)


if err != nil { return err } <ins> // Before we overwrite the config which holds the parsed // command line parameters, we need to copy all modified // logging settings over to the loaded config (i.e.  logging // command line flags have priority). Otherwise `--config // ... -v=5` doesn't work (config resets verbosity even // when it contains no logging settings). copyLogsFromFlags(fs, &c.Logging) </ins> o.config = c if err := o.initWatcher(); err != nil {


return utilfeature.DefaultMutableFeatureGate.SetFromMap(o.config.FeatureGates) } <ins> // copyLogsFromFlags applies the logging flags from the given flag set to the given // configuration. Fields for which the corresponding flag was not used are left // unmodified. For fields that have multiple values (like vmodule), the values from // the flags get joined so that the command line flags have priority. // // TODO (pohly): move this to logsapi func copyLogsFromFlags(from *pflag.FlagSet, to *logsapi.LoggingConfiguration) error { var cloneFS pflag.FlagSet logsapi.AddFlags(to, &cloneFS) vmodule := to.VModule to.VModule = nil var err error cloneFS.VisitAll(func(f *pflag.Flag) { if err != nil { return } fsFlag := from.Lookup(f.Name) if fsFlag == nil { err = fmt.Errorf("logging flag %s not found in flag set", f.Name) return } if !fsFlag.Changed { return } if setErr := f.Value.Set(fsFlag.Value.String()); setErr != nil { err = fmt.Errorf("copying flag %s value: %v", f.Name, setErr) return } }) to.VModule = append(to.VModule, vmodule...) return err } </ins> // Creates a new filesystem watcher and adds watches for the config file. func (o *Options) initWatcher() error { fswatcher := filesystem.NewFsnotifyWatcher()


return fmt.Errorf("failed os init: %w", err) } <del> if err := opts.Complete(); err != nil { </del> <ins> if err := opts.Complete(cmd.Flags()); err != nil { </ins> return fmt.Errorf("failed complete: %w", err) }


"testing" "time" <ins> "github.com/spf13/pflag" </ins> v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime"


opt := NewOptions() opt.ConfigFile = file.Name() <del> err = opt.Complete() </del> <ins> err = opt.Complete(new(pflag.FlagSet)) </ins> if err != nil { t.Fatal(err) }


import ( "errors" "fmt" <ins> "io/ioutil" "path" </ins> "testing" "time" "github.com/google/go-cmp/cmp" <ins> "github.com/spf13/pflag" </ins> "github.com/stretchr/testify/assert" <ins> "github.com/stretchr/testify/require" </ins> v1 "k8s.io/api/core/v1" <ins> "k8s.io/apimachinery/pkg/api/resource" </ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientsetfake "k8s.io/client-go/kubernetes/fake" componentbaseconfig "k8s.io/component-base/config"


} } <ins> // TestOptionsComplete checks that command line flags are combined with a // config properly. func TestOptionsComplete(t *testing.T) { header := `apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration ` // Determine default config (depends on platform defaults). o := NewOptions() require.NoError(t, o.Complete(new(pflag.FlagSet))) expected := o.config config := header + `logging: format: json flushFrequency: 1s verbosity: 10 vmodule: - filePattern: foo.go verbosity: 6 - filePattern: bar.go verbosity: 8 ` expectedLoggingConfig := logsapi.LoggingConfiguration{ Format:         "json", FlushFrequency: logsapi.TimeOrMetaDuration{Duration: metav1.Duration{Duration: time.Second}, SerializeAsString: true}, Verbosity:      10, VModule: []logsapi.VModuleItem{ { FilePattern: "foo.go", Verbosity:   6, }, { FilePattern: "bar.go", Verbosity:   8, }, }, Options: logsapi.FormatOptions{ JSON: logsapi.JSONOptions{ InfoBufferSize: resource.QuantityValue{Quantity: resource.MustParse("0")}, }, }, } for name, tc := range map[string]struct { config   string flags    []string expected *kubeproxyconfig.KubeProxyConfiguration }{ "empty": { expected: expected, }, "empty-config": { config:   header, expected: expected, }, "logging-config": { config: config, expected: func() *kubeproxyconfig.KubeProxyConfiguration { c := expected.DeepCopy() c.Logging = *expectedLoggingConfig.DeepCopy() return c }(), }, "flags": { flags: []string{ "-v=7", "--vmodule", "goo.go=8", }, expected: func() *kubeproxyconfig.KubeProxyConfiguration { c := expected.DeepCopy() c.Logging.Verbosity = 7 c.Logging.VModule = append(c.Logging.VModule, logsapi.VModuleItem{ FilePattern: "goo.go", Verbosity:   8, }) return c }(), }, "both": { config: config, flags: []string{ "-v=7", "--vmodule", "goo.go=8", "--ipvs-scheduler", "some-scheduler", // Overwritten by config. }, expected: func() *kubeproxyconfig.KubeProxyConfiguration { c := expected.DeepCopy() c.Logging = *expectedLoggingConfig.DeepCopy() // Flag wins. c.Logging.Verbosity = 7 // Flag and config get merged with command line flags first. c.Logging.VModule = append([]logsapi.VModuleItem{ { FilePattern: "goo.go", Verbosity:   8, }, }, c.Logging.VModule...) return c }(), }, } { t.Run(name, func(t *testing.T) { options := NewOptions() fs := new(pflag.FlagSet) options.AddFlags(fs) flags := tc.flags if len(tc.config) > 0 { tmp := t.TempDir() configFile := path.Join(tmp, "kube-proxy.conf") require.NoError(t, ioutil.WriteFile(configFile, []byte(tc.config), 0666)) flags = append(flags, "--config", configFile) } require.NoError(t, fs.Parse(flags)) require.NoError(t, options.Complete(fs)) assert.Equal(t, tc.expected, options.config) }) } } </ins> type fakeProxyServerLongRun struct{} // Run runs the specified ProxyServer.


- baseImportPath: "./vendor/k8s.io/kms/" allowedImports: <del> - k8s.io/apimachinery - k8s.io/klog </del> - k8s.io/kms - baseImportPath: "./vendor/k8s.io/endpointslice/"


require ( github.com/gogo/protobuf v1.3.2 google.golang.org/grpc v1.56.3 <del> k8s.io/apimachinery v0.0.0 </del> ) require ( <del> github.com/go-logr/logr v1.2.4 // indirect </del> github.com/golang/protobuf v1.5.3 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect google.golang.org/protobuf v1.31.0 // indirect <del> k8s.io/klog/v2 v2.100.1 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect </del> ) <del> replace ( k8s.io/apimachinery => ../apimachinery k8s.io/kms => ../kms ) </del> <ins> replace k8s.io/kms => ../kms </ins>


cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= <del> github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= </del> github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= <del> github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= </del> github.com/envoyproxy/go-control-plane v0.11.1-0.20230524094728-9239064ad72f/go.mod h1:sfYdkwUW4BA3PbKjySwjJy+O4Pu0h62rlqCMHNk+K+Q= github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= <del> github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= </del> github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= <del> github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= </del> github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= <del> github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= </del> github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= <del> github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= </del> github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= <del> github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= github.com/onsi/gomega v1.28.0/go.mod h1:A1H2JE76sI14WIP57LMKj7FVfCHx3g3BcZVjJG8bjX8= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= </del> github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=


golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= <del> golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= </del> golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= <del> golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= </del> <ins> golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= </ins> golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=


google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= <del> gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= </del>


github.com/ThalesIgnite/crypto11 v1.2.5/go.mod h1:ILDKtnCKiQ7zRoNxcp36Y1ZR8LBPmR2E23+wTQe/MlE= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= <del> github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= </del> github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=


google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= <del> k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= </del>


"google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" <del> "k8s.io/apimachinery/pkg/util/wait" </del> kmsapi "k8s.io/kms/apis/v2" )


client := newClient(t, address) // make sure the gRPC server is up before running tests <del> if err := wait.PollImmediateUntilWithContext(ctx, time.Second, func(ctx context.Context) (bool, error) { ctx, cancel := context.WithTimeout(ctx, 3*time.Second) defer cancel() _, err := client.Status(ctx, &kmsapi.StatusRequest{}) if err != nil { t.Logf("failed to get kms status: %v", err) </del> <ins> ready: for { select { case <-ctx.Done(): t.Fatalf("server failed to start in time: %v", ctx.Err()) default: if done := func() bool { ctx, cancel := context.WithTimeout(ctx, 3*time.Second) defer cancel() _, err := client.Status(ctx, &kmsapi.StatusRequest{}) if err != nil { t.Logf("failed to get kms status: %v", err) } return err == nil }(); done { break ready } time.Sleep(time.Second) </ins> } <del> return err == nil, nil }); err != nil { t.Fatal(err) </del> } t.Run("should be able to encrypt and decrypt through unix domain sockets", func(t *testing.T) {


if !reflect.DeepEqual(n.podCIDRs, podCIDRs) { klog.ErrorS(nil, "Using NodeCIDR LocalDetector mode, current PodCIDRs are different than previous PodCIDRs, restarting", "node", klog.KObj(node), "newPodCIDRs", podCIDRs, "oldPodCIDRs", n.podCIDRs) <del> panic("Current Node PodCIDRs are different than previous PodCIDRs, restarting") </del> <ins> klog.FlushAndExit(klog.ExitFlushTimeout, 1) </ins> } }


if !reflect.DeepEqual(n.podCIDRs, podCIDRs) { klog.ErrorS(nil, "Using NodeCIDR LocalDetector mode, current PodCIDRs are different than previous PodCIDRs, restarting", "node", klog.KObj(node), "newPodCIDRs", podCIDRs, "oldPODCIDRs", n.podCIDRs) <del> panic("Current Node PodCIDRs are different than previous PodCIDRs, restarting") </del> <ins> klog.FlushAndExit(klog.ExitFlushTimeout, 1) </ins> } }


package proxy import ( <ins> "strconv" </ins> "testing" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" <ins> "k8s.io/klog/v2" </ins> ) func TestNodePodCIDRHandlerAdd(t *testing.T) { <ins> oldKlogOsExit := klog.OsExit defer func() { klog.OsExit = oldKlogOsExit }() klog.OsExit = customExit </ins> tests := []struct { name            string oldNodePodCIDRs []string


t.Errorf("The code did panic") } }() n.OnNodeAdd(node) }) } } func TestNodePodCIDRHandlerUpdate(t *testing.T) { <ins> oldKlogOsExit := klog.OsExit defer func() { klog.OsExit = oldKlogOsExit }() klog.OsExit = customExit </ins> tests := []struct { name            string oldNodePodCIDRs []string


t.Errorf("The code did panic") } }() n.OnNodeUpdate(oldNode, node) }) } } <ins> func customExit(exitCode int) { panic(strconv.Itoa(exitCode)) } </ins>


<ins> rules: - selectorRegexp: k8s[.]io/kubernetes allowedPrefixes: - k8s.io/kubernetes/cmd/kube-apiserver - k8s.io/kubernetes/pkg - k8s.io/kubernetes/plugin - k8s.io/kubernetes/test/utils - k8s.io/kubernetes/third_party </ins>


import ( "context" <ins> "crypto/ecdsa" "crypto/elliptic" "crypto/rand" </ins> "crypto/rsa" "crypto/x509" <ins> "crypto/x509/pkix" "encoding/pem" </ins> "fmt" <ins> "math" "math/big" </ins> "net" "os" "path/filepath"


restclient "k8s.io/client-go/rest" clientgotransport "k8s.io/client-go/transport" "k8s.io/client-go/util/cert" <ins> "k8s.io/client-go/util/keyutil" </ins> logsapi "k8s.io/component-base/logs/api/v1" "k8s.io/klog/v2" "k8s.io/kube-aggregator/pkg/apiserver"


"k8s.io/kubernetes/cmd/kube-apiserver/app" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" <del> "k8s.io/kubernetes/cmd/kubeadm/app/util/pkiutil" </del> testutil "k8s.io/kubernetes/test/utils" )


// give the kube api server an "identity" it can use to for request header auth // so that aggregated api servers can understand who the calling user is s.Authentication.RequestHeader.AllowedNames = []string{"ash", "misty", "brock"} <ins> // create private key signer, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { return result, err } </ins> // make a client certificate for the api server - common name has to match one of our defined names above <ins> serial, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64-1)) if err != nil { return result, err } serial = new(big.Int).Add(serial, big.NewInt(1)) </ins> tenThousandHoursLater := time.Now().Add(10_000 * time.Hour) <del> clientCrtOfAPIServer, signer, err := pkiutil.NewCertAndKey(proxySigningCert, proxySigningKey, &pkiutil.CertConfig{ Config: cert.Config{ </del> <ins> certTmpl := x509.Certificate{ Subject: pkix.Name{ </ins> CommonName: "misty", <del> Usages: []x509.ExtKeyUsage{ x509.ExtKeyUsageClientAuth, }, </del> }, <del> NotAfter:           &tenThousandHoursLater, PublicKeyAlgorithm: x509.ECDSA, }) </del> <ins> SerialNumber: serial, NotBefore:    proxySigningCert.NotBefore, NotAfter:     tenThousandHoursLater, KeyUsage:     x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{ x509.ExtKeyUsageClientAuth, }, BasicConstraintsValid: true, } certDERBytes, err := x509.CreateCertificate(rand.Reader, &certTmpl, proxySigningCert, signer.Public(), proxySigningKey) </ins> if err != nil { return result, err } <del> if err := pkiutil.WriteCertAndKey(s.SecureServing.ServerCert.CertDirectory, "misty-crt", clientCrtOfAPIServer, signer); err != nil { </del> <ins> clientCrtOfAPIServer, err := x509.ParseCertificate(certDERBytes) if err != nil { </ins> return result, err } <ins> // write the cert to disk certificatePath := filepath.Join(s.SecureServing.ServerCert.CertDirectory, "misty-crt.crt") certBlock := pem.Block{ Type:  "CERTIFICATE", Bytes: clientCrtOfAPIServer.Raw, } certBytes := pem.EncodeToMemory(&certBlock) if err := cert.WriteCert(certificatePath, certBytes); err != nil { return result, err } // write the key to disk privateKeyPath := filepath.Join(s.SecureServing.ServerCert.CertDirectory, "misty-crt.key") encodedPrivateKey, err := keyutil.MarshalPrivateKeyToPEM(signer) if err != nil { return result, err } if err := keyutil.WriteKey(privateKeyPath, encodedPrivateKey); err != nil { return result, err } </ins> s.ProxyClientKeyFile = filepath.Join(s.SecureServing.ServerCert.CertDirectory, "misty-crt.key") s.ProxyClientCertFile = filepath.Join(s.SecureServing.ServerCert.CertDirectory, "misty-crt.crt")


"k8s.io/kubernetes/test/e2e/framework" e2edaemonset "k8s.io/kubernetes/test/e2e/framework/daemonset" e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment" <del> e2ekubesystem "k8s.io/kubernetes/test/e2e/framework/kubesystem" </del> e2enetwork "k8s.io/kubernetes/test/e2e/framework/network" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod"


e2eservice "k8s.io/kubernetes/test/e2e/framework/service" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" "k8s.io/kubernetes/test/e2e/network/common" <del> gcecloud "k8s.io/legacy-cloud-providers/gce" </del> admissionapi "k8s.io/pod-security-admission/api" netutils "k8s.io/utils/net" utilpointer "k8s.io/utils/pointer"


} }) <del> // This test creates a load balancer, make sure its health check interval // equals to gceHcCheckIntervalSeconds. Then the interval is manipulated // to be something else, see if the interval will be reconciled. ginkgo.It("should reconcile LB health check interval [Slow][Serial][Disruptive]", func(ctx context.Context) { const gceHcCheckIntervalSeconds = int64(8) // This test is for clusters on GCE. // (It restarts kube-controller-manager, which we don't support on GKE) e2eskipper.SkipUnlessProviderIs("gce") e2eskipper.SkipUnlessSSHKeyPresent() clusterID, err := gce.GetClusterID(ctx, cs) if err != nil { framework.Failf("framework.GetClusterID(cs) = _, %v; want nil", err) } gceCloud, err := gce.GetGCECloud() if err != nil { framework.Failf("framework.GetGCECloud() = _, %v; want nil", err) } namespace := f.Namespace.Name serviceName := "lb-hc-int" jig := e2eservice.NewTestJig(cs, namespace, serviceName) ginkgo.By("create load balancer service") // Create loadbalancer service with source range from node[0] and podAccept svc, err := jig.CreateTCPService(ctx, func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeLoadBalancer }) framework.ExpectNoError(err) ginkgo.DeferCleanup(func(ctx context.Context) { ginkgo.By("Clean up loadbalancer service") e2eservice.WaitForServiceDeletedWithFinalizer(ctx, cs, svc.Namespace, svc.Name) }) svc, err = jig.WaitForLoadBalancer(ctx, e2eservice.GetServiceLoadBalancerCreationTimeout(ctx, cs)) framework.ExpectNoError(err) hcName := gcecloud.MakeNodesHealthCheckName(clusterID) hc, err := gceCloud.GetHTTPHealthCheck(hcName) if err != nil { framework.Failf("gceCloud.GetHttpHealthCheck(%q) = _, %v; want nil", hcName, err) } gomega.Expect(hc.CheckIntervalSec).To(gomega.Equal(gceHcCheckIntervalSeconds)) ginkgo.By("modify the health check interval") hc.CheckIntervalSec = gceHcCheckIntervalSeconds - 1 if err = gceCloud.UpdateHTTPHealthCheck(hc); err != nil { framework.Failf("gcecloud.UpdateHttpHealthCheck(%#v) = %v; want nil", hc, err) } ginkgo.By("restart kube-controller-manager") if err := e2ekubesystem.RestartControllerManager(ctx); err != nil { framework.Failf("e2ekubesystem.RestartControllerManager() = %v; want nil", err) } if err := e2ekubesystem.WaitForControllerManagerUp(ctx); err != nil { framework.Failf("e2ekubesystem.WaitForControllerManagerUp() = %v; want nil", err) } ginkgo.By("health check should be reconciled") pollInterval := framework.Poll * 10 loadBalancerPropagationTimeout := e2eservice.GetServiceLoadBalancerPropagationTimeout(ctx, cs) if pollErr := wait.PollImmediate(pollInterval, loadBalancerPropagationTimeout, func() (bool, error) { hc, err := gceCloud.GetHTTPHealthCheck(hcName) if err != nil { framework.Logf("ginkgo.Failed to get HttpHealthCheck(%q): %v", hcName, err) return false, err } framework.Logf("hc.CheckIntervalSec = %v", hc.CheckIntervalSec) return hc.CheckIntervalSec == gceHcCheckIntervalSeconds, nil }); pollErr != nil { framework.Failf("Health check %q does not reconcile its check interval to %d.", hcName, gceHcCheckIntervalSeconds) } }) </del> // [LinuxOnly]: Windows does not support session affinity. ginkgo.It("should have session affinity work for LoadBalancer service with ESIPP on [Slow] [LinuxOnly]", func(ctx context.Context) { // L4 load balancer affinity `ClientIP` is not supported on AWS ELB.


indexer.Add(resourceQuota) // old service was a load balancer, but updated version is a node port. <del> oldService := &api.Service{ ObjectMeta: api.ObjectMeta{Name: "service", Namespace: "test"}, </del> <ins> existingService := &api.Service{ ObjectMeta: api.ObjectMeta{Name: "service", Namespace: "test", ResourceVersion: "1"}, </ins> Spec:       api.ServiceSpec{Type: api.ServiceTypeLoadBalancer}, } newService := &api.Service{


Ports: []api.ServicePort{{Port: 1234}}, }, } <del> err := handler.Admit(admission.NewAttributesRecord(newService, oldService, api.Kind("Service").WithVersion("version"), newService.Namespace, newService.Name, api.Resource("services").WithVersion("version"), "", admission.Update, nil)) </del> <ins> err := handler.Admit(admission.NewAttributesRecord(newService, existingService, api.Kind("Service").WithVersion("version"), newService.Namespace, newService.Name, api.Resource("services").WithVersion("version"), "", admission.Update, nil)) </ins> if err != nil { t.Errorf("Unexpected error: %v", err) }


if prevItem == nil { return nil, admission.NewForbidden(a, fmt.Errorf("unable to get previous usage since prior version of object was not found")) } <del> prevUsage := evaluator.Usage(prevItem) deltaUsage = quota.Subtract(deltaUsage, prevUsage) </del> <ins> // if we can definitively determine that this is not a case of "create on update", // then charge based on the delta.  Otherwise, bill the maximum metadata, err := meta.Accessor(prevItem) if err == nil && len(metadata.GetResourceVersion()) > 0 { prevUsage := evaluator.Usage(prevItem) deltaUsage = quota.Subtract(deltaUsage, prevUsage) } </ins> } if quota.IsZero(deltaUsage) { return quotas, nil


// Verify PodReady is not set (since sandboxcreation is blocked) _, err = getTransitionTimeForPodConditionWithStatus(p, v1.PodReady, false) framework.ExpectNoError(err) <ins> // this testcase is creating the missing volume that unblock the pod above, // and check PodReadyToStartContainer is setting correctly. ginkgo.By("checking pod condition for a pod when volumes source is created") configmap := v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "cm-that-unblock-pod-condition", }, Data: map[string]string{ "key": "value", }, BinaryData: map[string][]byte{ "binaryKey": []byte("value"), }, } _, err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, &configmap, metav1.CreateOptions{}) framework.ExpectNoError(err) defer func() { err = f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, "cm-that-unblock-pod-condition", metav1.DeleteOptions{}) framework.ExpectNoError(err, "unable to delete configmap") }() p2 := webserverPodSpec("pod2-"+string(uuid.NewUUID()), "web2", "init2", hasInitContainers) p2.Spec.Volumes = []v1.Volume{ { Name: "cm-2", VolumeSource: v1.VolumeSource{ ConfigMap: &v1.ConfigMapVolumeSource{ LocalObjectReference: v1.LocalObjectReference{Name: "cm-that-unblock-pod-condition"}, }, }, }, } p2.Spec.Containers[0].VolumeMounts = []v1.VolumeMount{ { Name:      "cm-2", MountPath: "/config", }, } p2 = e2epod.NewPodClient(f).Create(ctx, p2) framework.ExpectNoError(e2epod.WaitTimeoutForPodReadyInNamespace(ctx, f.ClientSet, p2.Name, p2.Namespace, framework.PodStartTimeout)) p2, err = e2epod.NewPodClient(f).Get(ctx, p2.Name, metav1.GetOptions{}) framework.ExpectNoError(err) _, err = getTransitionTimeForPodConditionWithStatus(p2, v1.PodScheduled, true) framework.ExpectNoError(err) _, err = getTransitionTimeForPodConditionWithStatus(p2, v1.PodInitialized, true) framework.ExpectNoError(err) // Verify PodReadyToStartContainers is set (since sandboxcreation is unblocked) if checkPodReadyToStart { _, err = getTransitionTimeForPodConditionWithStatus(p2, v1.PodReadyToStartContainers, true) framework.ExpectNoError(err) } </ins> } }


// Verify PodReady is not set (since sandboxcreation is blocked) _, err = getTransitionTimeForPodConditionWithStatus(p, v1.PodReady, false) framework.ExpectNoError(err) <del> ginkgo.By("update pod related volume resource to unblock sandbox creation") e2epod.NewPodClient(f).Update(ctx, p.Name, func(pod *v1.Pod) { pod.Spec.Volumes = []v1.Volume{ { Name: "secret", VolumeSource: v1.VolumeSource{ Secret: &v1.SecretVolumeSource{ SecretName: "secret", }, }, }, } pod.Spec.Containers[0].VolumeMounts = []v1.VolumeMount{ { Name:      "secret", MountPath: "/config", }, } }) // Verify PodReadyToStartContainers is set (since sandboxcreation is unblocked) if checkPodReadyToStart { _, err = getTransitionTimeForPodConditionWithStatus(p, v1.PodReadyToStartContainers, true) framework.ExpectNoError(err) } </del> } }


} func (f *fakeService) WaitForReady(ctx context.Context) error { <del> err := wait.PollWithContext(ctx, 1*time.Second, 200*time.Millisecond, func(ctx context.Context) (done bool, err error) { </del> <ins> err := wait.PollUntilContextTimeout(ctx, 200*time.Millisecond, time.Second, false, func(ctx context.Context) (done bool, err error) { </ins> return f.Port() != nil, nil })


fs.DurationVar(&o.NodeMonitorGracePeriod.Duration, "node-monitor-grace-period", o.NodeMonitorGracePeriod.Duration, "Amount of time which we allow running Node to be unresponsive before marking it unhealthy. "+ "Must be N times more than kubelet's nodeStatusUpdateFrequency, "+ <del> "where N means number of retries allowed for kubelet to post node status.") </del> <ins> "where N means number of retries allowed for kubelet to post node status. "+ "This value should also be greater than the sum of HTTP2_PING_TIMEOUT_SECONDS and HTTP2_READ_IDLE_TIMEOUT_SECONDS") </ins> fs.Float32Var(&o.NodeEvictionRate, "node-eviction-rate", 0.1, "Number of nodes per second on which pods are deleted in case of node failure when a zone is healthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters.") fs.Float32Var(&o.SecondaryNodeEvictionRate, "secondary-node-eviction-rate", 0.01, "Number of nodes per second on which pods are deleted in case of node failure when a zone is unhealthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters. This value is implicitly overridden to 0 if the cluster size is smaller than --large-cluster-size-threshold.") fs.Int32Var(&o.LargeClusterSizeThreshold, "large-cluster-size-threshold", 50, fmt.Sprintf("Number of nodes from which %s treats the cluster as large for the eviction logic purposes. --secondary-node-eviction-rate is implicitly overridden to 0 for clusters this size or smaller. Notice: If nodes reside in multiple zones, this threshold will be considered as zone node size threshold for each zone to determine node eviction rate independently.", names.NodeLifecycleController))


// NodeMonitorGracePeriod is the amount of time which we allow a running node to be // unresponsive before marking it unhealthy. Must be N times more than kubelet's // nodeStatusUpdateFrequency, where N means number of retries allowed for kubelet <del> // to post node status. </del> <ins> // to post node status. This value should also be greater than the sum of // HTTP2_PING_TIMEOUT_SECONDS and HTTP2_READ_IDLE_TIMEOUT_SECONDS. </ins> NodeMonitorGracePeriod metav1.Duration // secondaryNodeEvictionRate is implicitly overridden to 0 for clusters smaller than or equal to largeClusterSizeThreshold LargeClusterSizeThreshold int32


if obj.PodEvictionTimeout == zero { obj.PodEvictionTimeout = metav1.Duration{Duration: 5 * time.Minute} } <ins> // NodeMonitorGracePeriod is set to a default value of 50 seconds. // This value should be greater than the sum of HTTP2_PING_TIMEOUT_SECONDS (30s) // and HTTP2_READ_IDLE_TIMEOUT_SECONDS (15s) from the http2 health check // to ensure that the server has adequate time to handle slow or idle connections // properly before marking a node as unhealthy. </ins> if obj.NodeMonitorGracePeriod == zero { <del> obj.NodeMonitorGracePeriod = metav1.Duration{Duration: 40 * time.Second} </del> <ins> obj.NodeMonitorGracePeriod = metav1.Duration{Duration: 50 * time.Second} </ins> } if obj.NodeStartupGracePeriod == zero { obj.NodeStartupGracePeriod = metav1.Duration{Duration: 60 * time.Second}


//    be less than the node health signal update frequency, since there will //    only be fresh values from Kubelet at an interval of node health signal //    update frequency. <del> // 2. nodeMonitorGracePeriod can't be too large for user experience - larger </del> <ins> // 2. nodeMonitorGracePeriod should be greater than the sum of HTTP2_PING_TIMEOUT_SECONDS (30s) // 	  and HTTP2_READ_IDLE_TIMEOUT_SECONDS (15s) from the http2 health check // 	  to ensure that the server has adequate time to handle slow or idle connections //    properly before marking a node as unhealthy. // 3. nodeMonitorGracePeriod can't be too large for user experience - larger </ins> //    value takes longer for user to see up-to-date node health. nodeMonitorGracePeriod time.Duration