TEMP_DIR:=$(shell mktemp -d) ifeq ($(ARCH),amd64) <del> BASEIMAGE?=k8s.gcr.io/build-image/debian-base:buster-v1.3.0 </del> <ins> BASEIMAGE?=k8s.gcr.io/build-image/debian-base:buster-v1.4.0 </ins> endif ifeq ($(ARCH),arm) <del> BASEIMAGE?=k8s.gcr.io/build-image/debian-base-arm:buster-v1.3.0 </del> <ins> BASEIMAGE?=k8s.gcr.io/build-image/debian-base-arm:buster-v1.4.0 </ins> endif ifeq ($(ARCH),arm64) <del> BASEIMAGE?=k8s.gcr.io/build-image/debian-base-arm64:buster-v1.3.0 </del> <ins> BASEIMAGE?=k8s.gcr.io/build-image/debian-base-arm64:buster-v1.4.0 </ins> endif ifeq ($(ARCH),ppc64le) <del> BASEIMAGE?=k8s.gcr.io/build-image/debian-base-ppc64le:buster-v1.3.0 </del> <ins> BASEIMAGE?=k8s.gcr.io/build-image/debian-base-ppc64le:buster-v1.4.0 </ins> endif ifeq ($(ARCH),s390x) <del> BASEIMAGE?=k8s.gcr.io/build-image/debian-base-s390x:buster-v1.3.0 </del> <ins> BASEIMAGE?=k8s.gcr.io/build-image/debian-base-s390x:buster-v1.4.0 </ins> endif RUNNERIMAGE?=gcr.io/distroless/static:latest


configs[CheckMetadataConcealment] = Config{e2eRegistry, "metadata-concealment", "1.2"} configs[CudaVectorAdd] = Config{e2eRegistry, "cuda-vector-add", "1.0"} configs[CudaVectorAdd2] = Config{e2eRegistry, "cuda-vector-add", "2.0"} <del> configs[DebianIptables] = Config{buildImageRegistry, "debian-iptables", "buster-v1.4.0"} </del> <ins> configs[DebianIptables] = Config{buildImageRegistry, "debian-iptables", "buster-v1.5.0"} </ins> configs[EchoServer] = Config{e2eRegistry, "echoserver", "2.2"} configs[Etcd] = Config{gcRegistry, "etcd", "3.4.13-0"} configs[GlusterDynamicProvisioner] = Config{dockerGluster, "glusterdynamic-provisioner", "v1.0"}


for k, v := range labels { pv.Labels[k] = v var values []string <del> if k == v1.LabelFailureDomainBetaZone { </del> <ins> if k == v1.LabelTopologyZone || k == v1.LabelFailureDomainBetaZone { </ins> values, err = volumehelpers.LabelZonesToList(v) if err != nil { return nil, fmt.Errorf("failed to convert label string for Zone: %s to a List: %v", v, err)


translator := NewvSphereCSITranslator() topologySelectorTerm := v1.TopologySelectorTerm{[]v1.TopologySelectorLabelRequirement{ { <ins> Key:    v1.LabelTopologyZone, Values: []string{"zone-a"}, }, { Key:    v1.LabelTopologyRegion, Values: []string{"region-a"}, }, }} topologySelectorTermWithBetaLabels := v1.TopologySelectorTerm{[]v1.TopologySelectorLabelRequirement{ { </ins> Key:    v1.LabelFailureDomainBetaZone, Values: []string{"zone-a"}, },


expSc: NewStorageClass(map[string]string{"storagepolicyname": "test-policy-name", paramcsiMigration: "true"}, []v1.TopologySelectorTerm{topologySelectorTerm}), }, { <ins> name:  "translate with storagepolicyname and allowedTopology beta labels", sc:    NewStorageClass(map[string]string{"storagepolicyname": "test-policy-name"}, []v1.TopologySelectorTerm{topologySelectorTermWithBetaLabels}), expSc: NewStorageClass(map[string]string{"storagepolicyname": "test-policy-name", paramcsiMigration: "true"}, []v1.TopologySelectorTerm{topologySelectorTermWithBetaLabels}), }, { </ins> name:  "translate with raw vSAN policy parameters, datastore and diskformat", sc:    NewStorageClass(map[string]string{"hostfailurestotolerate": "2", "datastore": "vsanDatastore", "diskformat": "thin"}, []v1.TopologySelectorTerm{topologySelectorTerm}), expSc: NewStorageClass(map[string]string{"hostfailurestotolerate-migrationparam": "2", "datastore-migrationparam": "vsanDatastore", "diskformat-migrationparam": "thin", paramcsiMigration: "true"}, []v1.TopologySelectorTerm{topologySelectorTerm}),


} } // Get the node zone information <del> nodeFd := node.ObjectMeta.Labels[v1.LabelFailureDomainBetaZone] nodeRegion := node.ObjectMeta.Labels[v1.LabelFailureDomainBetaRegion] </del> <ins> nodeFd := node.ObjectMeta.Labels[v1.LabelTopologyZone] nodeRegion := node.ObjectMeta.Labels[v1.LabelTopologyRegion] </ins> nodeZone := &cloudprovider.Zone{FailureDomain: nodeFd, Region: nodeRegion} nodeInfo := &NodeInfo{dataCenter: res.datacenter, vm: vm, vcServer: res.vc, vmUUID: nodeUUID, zone: nodeZone} nm.addNodeInfo(node.ObjectMeta.Name, nodeInfo)


// FIXME: For now, pick the first zone of datastore as the zone of volume labels := make(map[string]string) if len(dsZones) > 0 { <del> labels[v1.LabelFailureDomainBetaRegion] = dsZones[0].Region labels[v1.LabelFailureDomainBetaZone] = dsZones[0].FailureDomain </del> <ins> labels[v1.LabelTopologyRegion] = dsZones[0].Region labels[v1.LabelTopologyZone] = dsZones[0].FailureDomain </ins> } return labels, nil }


term := v1.TopologySelectorTerm{ MatchLabelExpressions: []v1.TopologySelectorLabelRequirement{ { <del> Key:    v1.LabelFailureDomainBetaZone, </del> <ins> Key:    v1.LabelTopologyZone, </ins> Values: zones, }, },


zones = append(zones, zoneA) nodeSelectorMap := map[string]string{ // nodeSelector set as zoneB <del> v1.LabelFailureDomainBetaZone: zoneB, </del> <ins> v1.LabelTopologyZone: zoneB, </ins> } verifyPodSchedulingFails(client, namespace, nodeSelectorMap, scParameters, zones, storagev1.VolumeBindingWaitForFirstConsumer) })


ginkgo.By("Verify zone information is present in the volume labels") for _, pv := range persistentvolumes { // Multiple zones are separated with "__" <del> pvZoneLabels := strings.Split(pv.ObjectMeta.Labels["failure-domain.beta.kubernetes.io/zone"], "__") </del> <ins> pvZoneLabels := strings.Split(pv.ObjectMeta.Labels[v1.LabelTopologyZone], "__") </ins> for _, zone := range zones { gomega.Expect(pvZoneLabels).Should(gomega.ContainElement(zone), "Incorrect or missing zone labels in pv.") }


case err := <-injectedError: return nil, err } <del> // Never get here. return nil, nil </del> }


import ( "context" <ins> "strings" </ins> "time" "github.com/Microsoft/hcsshim"


// That will typically happen with init-containers in Exited state. Docker still knows about them but the HCS does not. // As we don't want to block stats retrieval for other containers, we only log errors. if !hcsshim.IsNotExist(err) && !hcsshim.IsAlreadyStopped(err) { <del> klog.Errorf("Error opening container (stats will be missing) '%s': %v", containerID, err) </del> <ins> klog.V(4).Infof("Error opening container (stats will be missing) '%s': %v", containerID, err) </ins> } return nil, nil }


stats, err := hcsshimContainer.Statistics() if err != nil { <ins> if strings.Contains(err.Error(), "0x5") || strings.Contains(err.Error(), "0xc0370105") { // When the container is just created, querying for stats causes access errors because it hasn't started yet // This is transient; skip container for now // // These hcs errors do not have helpers exposed in public package so need to query for the known codes // https://github.com/microsoft/hcsshim/blob/master/internal/hcs/errors.go // PR to expose helpers in hcsshim: https://github.com/microsoft/hcsshim/pull/933 klog.V(4).Infof("Container is not in a state that stats can be accessed '%s': %v. This occurs when the container is created but not started.", containerID, err) return nil, nil } </ins> return nil, err }


func (r *Reachability) PrintSummary(printExpected bool, printObserved bool, printComparison bool) { right, wrong, ignored, comparison := r.Summary(ignoreLoopback) if ignored > 0 { <del> framework.Logf("warning: the results of %d pod->pod cases have been ignored", ignored) </del> <ins> framework.Logf("warning: this test doesn't take into consideration hairpin traffic, i.e. traffic whose source and destination is the same pod: %d cases ignored", ignored) </ins> } framework.Logf("reachability: correct:%v, incorrect:%v, result=%tnn", right, wrong, wrong == 0) if printExpected {


f := framework.NewDefaultFramework("metrics-grabber") var c, ec clientset.Interface var grabber *e2emetrics.Grabber <ins> var masterRegistered bool </ins> ginkgo.BeforeEach(func() { var err error c = f.ClientSet ec = f.KubemarkExternalClusterClientSet <ins> // Check if master Node is registered nodes, err := c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) </ins> framework.ExpectNoError(err) <ins> for _, node := range nodes.Items { if strings.HasSuffix(node.Name, "master") { masterRegistered = true } } </ins> gomega.Eventually(func() error { grabber, err = e2emetrics.NewMetricsGrabber(c, ec, true, true, true, true, true) if err != nil { return fmt.Errorf("failed to create metrics grabber: %v", err) } <del> if !grabber.HasControlPlanePods() { </del> <ins> if masterRegistered && !grabber.HasControlPlanePods() { </ins> return fmt.Errorf("unable to get find control plane pods") } return nil


ginkgo.It("should grab all metrics from a Scheduler.", func() { ginkgo.By("Proxying to Pod through the API server") <del> // Check if master Node is registered nodes, err := c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) framework.ExpectNoError(err) var masterRegistered = false for _, node := range nodes.Items { if strings.HasSuffix(node.Name, "master") { masterRegistered = true } } </del> if !masterRegistered { framework.Logf("Master is node api.Registry. Skipping testing Scheduler metrics.") return


ginkgo.It("should grab all metrics from a ControllerManager.", func() { ginkgo.By("Proxying to Pod through the API server") <del> // Check if master Node is registered nodes, err := c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) framework.ExpectNoError(err) var masterRegistered = false for _, node := range nodes.Items { if strings.HasSuffix(node.Name, "master") { masterRegistered = true } } </del> if !masterRegistered { framework.Logf("Master is node api.Registry. Skipping testing ControllerManager metrics.") return


import ( "context" <ins> "encoding/json" </ins> "fmt" "time" appsv1 "k8s.io/api/apps/v1" <ins> autoscalingv1 "k8s.io/api/autoscaling/v1" "k8s.io/apimachinery/pkg/types" </ins> v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource"


framework.ConformanceIt("should adopt matching pods on creation and release no longer matching pods", func() { testRSAdoptMatchingAndReleaseNotMatching(f) }) <ins> ginkgo.It("Replicaset should have a working scale subresource", func() { testRSScaleSubresources(f) }) </ins> }) // A basic test to check the deployment of an image using a ReplicaSet. The


}) framework.ExpectNoError(err) } <ins> func testRSScaleSubresources(f *framework.Framework) { ns := f.Namespace.Name c := f.ClientSet // Create webserver pods. rsPodLabels := map[string]string{ "name": "sample-pod", "pod":  WebserverImageName, } rsName := "test-rs" replicas := int32(1) ginkgo.By(fmt.Sprintf("Creating replica set %q that asks for more than the allowed pod quota", rsName)) rs := newRS(rsName, replicas, rsPodLabels, WebserverImageName, WebserverImage, nil) _, err := c.AppsV1().ReplicaSets(ns).Create(context.TODO(), rs, metav1.CreateOptions{}) framework.ExpectNoError(err) // Verify that the required pods have come up. err = e2epod.VerifyPodsRunning(c, ns, "sample-pod", false, replicas) framework.ExpectNoError(err, "error in waiting for pods to come up: %s", err) ginkgo.By("getting scale subresource") scale, err := c.AppsV1().ReplicaSets(ns).GetScale(context.TODO(), rsName, metav1.GetOptions{}) if err != nil { framework.Failf("Failed to get scale subresource: %v", err) } framework.ExpectEqual(scale.Spec.Replicas, int32(1)) framework.ExpectEqual(scale.Status.Replicas, int32(1)) ginkgo.By("updating a scale subresource") scale.ResourceVersion = "" // indicate the scale update should be unconditional scale.Spec.Replicas = 2 scaleResult, err := c.AppsV1().ReplicaSets(ns).UpdateScale(context.TODO(), rsName, scale, metav1.UpdateOptions{}) if err != nil { framework.Failf("Failed to put scale subresource: %v", err) } framework.ExpectEqual(scaleResult.Spec.Replicas, int32(2)) ginkgo.By("verifying the replicaset Spec.Replicas was modified") rs, err = c.AppsV1().ReplicaSets(ns).Get(context.TODO(), rsName, metav1.GetOptions{}) if err != nil { framework.Failf("Failed to get statefulset resource: %v", err) } framework.ExpectEqual(*(rs.Spec.Replicas), int32(2)) ginkgo.By("Patch a scale subresource") scale.ResourceVersion = "" // indicate the scale update should be unconditional scale.Spec.Replicas = 4    // should be 2 after "UpdateScale" operation, now Patch to 4 rsScalePatchPayload, err := json.Marshal(autoscalingv1.Scale{ Spec: autoscalingv1.ScaleSpec{ Replicas: scale.Spec.Replicas, }, }) framework.ExpectNoError(err, "Could not Marshal JSON for patch payload") _, err = c.AppsV1().ReplicaSets(ns).Patch(context.TODO(), rsName, types.StrategicMergePatchType, []byte(rsScalePatchPayload), metav1.PatchOptions{}, "scale") framework.ExpectNoError(err, "Failed to patch replicaset: %v", err) rs, err = c.AppsV1().ReplicaSets(ns).Get(context.TODO(), rsName, metav1.GetOptions{}) framework.ExpectNoError(err, "Failed to get replicaset resource: %v", err) framework.ExpectEqual(*(rs.Spec.Replicas), int32(4), "replicaset should have 4 replicas") } </ins>


the Job MUST execute to completion. release: v1.16 file: test/e2e/apps/job.go <ins> - testname: ReplicaSet, completes the scaling of a ReplicaSet subresource codename: '[sig-apps] ReplicaSet Replicaset should have a working scale subresource [Conformance]' description: Create a ReplicaSet (RS) with a single Pod. The Pod MUST be verified that it is running. The RS MUST get and verify the scale subresource count. The RS MUST update and verify the scale subresource. The RS MUST patch and verify a scale subresource. release: v1.21 file: test/e2e/apps/replica_set.go </ins> - testname: Replica Set, adopt matching pods and release non matching pods codename: '[sig-apps] ReplicaSet should adopt matching pods on creation and release no longer matching pods [Conformance]'


testRSAdoptMatchingAndReleaseNotMatching(f) }) <del> ginkgo.It("Replicaset should have a working scale subresource", func() { </del> <ins> /* Release: v1.21 Testname: ReplicaSet, completes the scaling of a ReplicaSet subresource Description: Create a ReplicaSet (RS) with a single Pod. The Pod MUST be verified that it is running. The RS MUST get and verify the scale subresource count. The RS MUST update and verify the scale subresource. The RS MUST patch and verify a scale subresource. */ framework.ConformanceIt("Replicaset should have a working scale subresource", func() { </ins> testRSScaleSubresources(f) }) })


package oom import ( <ins> "os" </ins> "github.com/opencontainers/runc/libcontainer/cgroups" "testing"


func TestPidListerFailure(t *testing.T) { _, err := getPids("/does/not/exist") <del> assert.True(t, cgroups.IsNotFound(err), "expected getPids to return not exists error. Got %v", err) </del> <ins> assert.True(t, cgroups.IsNotFound(err) || os.IsNotExist(err), "expected getPids to return not exists error. Got %v", err) </ins> }


path string } <ins> func (v *VersionFile) nextPath() string { return fmt.Sprintf("%s-next", v.path) } </ins> // Exists returns true if a version.txt file exists on the file system. func (v *VersionFile) Exists() (bool, error) { return exists(v.path)


// Write creates or overwrites the contents of the version.txt file with the given EtcdVersionPair. func (v *VersionFile) Write(vp *EtcdVersionPair) error { <del> data := []byte(fmt.Sprintf("%s/%s", vp.version, vp.storageVersion)) return ioutil.WriteFile(v.path, data, 0666) </del> <ins> // We do write + rename instead of just write to protect from version.txt // corruption under full disk condition. // See https://github.com/kubernetes/kubernetes/issues/98989. err := ioutil.WriteFile(v.nextPath(), []byte(vp.String()), 0666) if err != nil { return fmt.Errorf("failed to write new version file %s: %v", v.nextPath(), err) } return os.Rename(v.nextPath(), v.path) </ins> } func exists(path string) (bool, error) {


name = "client-targets", conditioned_srcs = for_platforms(for_client = [ "//cmd/kubectl", <ins> "//cmd/kubectl-convert", </ins> ]), )


load("//staging/src/k8s.io/component-base/version:def.bzl", "version_x_defs") go_binary( <del> name = "kubectl", </del> <ins> name = "kubectl-convert", </ins> embed = [":go_default_library"], pure = "on", visibility = ["//visibility:public"],


# If you update this list, please also update build/BUILD. readonly KUBE_CLIENT_TARGETS=( cmd/kubectl <ins> cmd/kubectl-convert </ins> ) readonly KUBE_CLIENT_BINARIES=("${KUBE_CLIENT_TARGETS[@]##*/}") readonly KUBE_CLIENT_BINARIES_WIN=("${KUBE_CLIENT_BINARIES[@]/%/.exe}")


<ins> #!/usr/bin/env bash # Copyright 2021 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -o errexit set -o nounset set -o pipefail run_convert_tests() { set -o nounset set -o errexit ### Convert deployment YAML file locally without affecting the live deployment # Pre-condition: no deployments exist kube::test::get_object_assert deployment "{{range.items}}{{${id_field:?}}}:{{end}}" '' # Command # Create a deployment (revision 1) kubectl create -f hack/testdata/deployment-revision1.yaml "${kube_flags[@]:?}" kube::test::get_object_assert deployment "{{range.items}}{{${id_field:?}}}:{{end}}" 'nginx:' kube::test::get_object_assert deployment "{{range.items}}{{${image_field0:?}}}:{{end}}" "${IMAGE_DEPLOYMENT_R1}:" # Command output_message=$(kubectl convert --local -f hack/testdata/deployment-revision1.yaml --output-version=apps/v1beta1 -o yaml "${kube_flags[@]:?}") # Post-condition: apiVersion is still apps/v1 in the live deployment, but command output is the new value kube::test::get_object_assert 'deployment nginx' "{{ .apiVersion }}" 'apps/v1' kube::test::if_has_string "${output_message}" "apps/v1beta1" # Clean up kubectl delete deployment nginx "${kube_flags[@]:?}" ## Convert multiple busybox PODs recursively from directory of YAML files # Command output_message=$(! kubectl-convert -f hack/testdata/recursive/pod --recursive 2>&1 "${kube_flags[@]:?}") # Post-condition: busybox0 & busybox1 PODs are converted, and since busybox2 is malformed, it should error kube::test::if_has_string "${output_message}" "Object 'Kind' is missing" # check that convert command supports --template output output_message=$(kubectl-convert "${kube_flags[@]:?}" -f hack/testdata/deployment-revision1.yaml --output-version=apps/v1beta2 --template="{{ .metadata.name }}:") kube::test::if_has_string "${output_message}" 'nginx:' set +o nounset set +o errexit } </ins>


source "${KUBE_ROOT}/test/cmd/authorization.sh" source "${KUBE_ROOT}/test/cmd/batch.sh" source "${KUBE_ROOT}/test/cmd/certificate.sh" <ins> source "${KUBE_ROOT}/test/cmd/convert.sh" </ins> source "${KUBE_ROOT}/test/cmd/core.sh" source "${KUBE_ROOT}/test/cmd/crd.sh" source "${KUBE_ROOT}/test/cmd/create.sh"


kube::util::ensure-gnu-sed kube::log::status "Building kubectl" <del> make -C "${KUBE_ROOT}" WHAT="cmd/kubectl" </del> <ins> make -C "${KUBE_ROOT}" WHAT="cmd/kubectl cmd/kubectl-convert" </ins> # Check kubectl kube::log::status "Running kubectl with no options"


fi ###################### <ins> # Convert            # ###################### if kube::test::if_supports_resource "${deployments}"; then record_command run_convert_tests fi ###################### </ins> # Delete             # ###################### if kube::test::if_supports_resource "${configmaps}" ; then


return err != nil && os.IsNotExist(err) } <ins> // GetCapacity returns node capacity data for "cpu", "memory", "ephemeral-storage", and "huge-pages*" // At present this method is only invoked when introspecting ephemeral storage </ins> func (cm *containerManagerImpl) GetCapacity() v1.ResourceList { <ins> if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.LocalStorageCapacityIsolation) { // We store allocatable ephemeral-storage in the capacity property once we Start() the container manager if _, ok := cm.capacity[v1.ResourceEphemeralStorage]; !ok { // If we haven't yet stored the capacity for ephemeral-storage, we can try to fetch it directly from cAdvisor, if cm.cadvisorInterface != nil { rootfs, err := cm.cadvisorInterface.RootFsInfo() if err != nil { klog.ErrorS(err, "Unable to get rootfs data from cAdvisor interface") // If the rootfsinfo retrieval from cAdvisor fails for any reason, fallback to returning the capacity property with no ephemeral storage data return cm.capacity } // We don't want to mutate cm.capacity here so we'll manually construct a v1.ResourceList from it, // and add ephemeral-storage capacityWithEphemeralStorage := v1.ResourceList{} for rName, rQuant := range cm.capacity { capacityWithEphemeralStorage[rName] = rQuant } capacityWithEphemeralStorage[v1.ResourceEphemeralStorage] = cadvisor.EphemeralStorageCapacityFromFsInfo(rootfs)[v1.ResourceEphemeralStorage] return capacityWithEphemeralStorage } } } </ins> return cm.capacity }


package cm import ( <ins> "errors" </ins> "io/ioutil" "os" "path" "testing" <ins> gomock "github.com/golang/mock/gomock" cadvisorapiv2 "github.com/google/cadvisor/info/v2" utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" kubefeatures "k8s.io/kubernetes/pkg/features" </ins> "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" <ins> v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing" </ins> "k8s.io/mount-utils" )


assert.NoError(t, err) assert.True(t, f.cpuHardcapping, "cpu hardcapping is expected to be enabled") } <ins> func TestGetCapacity(t *testing.T) { ephemeralStorageFromCapacity := int64(2000) ephemeralStorageFromCadvisor := int64(8000) mockCtrl := gomock.NewController(t) defer mockCtrl.Finish() mockCtrlError := gomock.NewController(t) defer mockCtrlError.Finish() mockCadvisor := cadvisortest.NewMockInterface(mockCtrl) rootfs := cadvisorapiv2.FsInfo{ Capacity: 8000, } mockCadvisor.EXPECT().RootFsInfo().Return(rootfs, nil) mockCadvisorError := cadvisortest.NewMockInterface(mockCtrlError) mockCadvisorError.EXPECT().RootFsInfo().Return(cadvisorapiv2.FsInfo{}, errors.New("Unable to get rootfs data from cAdvisor interface")) cases := []struct { name                                string cm                                  *containerManagerImpl expectedResourceQuantity            *resource.Quantity expectedNoEphemeralStorage          bool enableLocalStorageCapacityIsolation bool }{ { name: "capacity property has ephemeral-storage", cm: &containerManagerImpl{ cadvisorInterface: mockCadvisor, capacity: v1.ResourceList{ v1.ResourceEphemeralStorage: *resource.NewQuantity(ephemeralStorageFromCapacity, resource.BinarySI), }, }, expectedResourceQuantity:            resource.NewQuantity(ephemeralStorageFromCapacity, resource.BinarySI), expectedNoEphemeralStorage:          false, enableLocalStorageCapacityIsolation: true, }, { name: "capacity property does not have ephemeral-storage", cm: &containerManagerImpl{ cadvisorInterface: mockCadvisor, capacity:          v1.ResourceList{}, }, expectedResourceQuantity:            resource.NewQuantity(ephemeralStorageFromCadvisor, resource.BinarySI), expectedNoEphemeralStorage:          false, enableLocalStorageCapacityIsolation: true, }, { name: "capacity property does not have ephemeral-storage, error from rootfs", cm: &containerManagerImpl{ cadvisorInterface: mockCadvisorError, capacity:          v1.ResourceList{}, }, expectedNoEphemeralStorage:          true, enableLocalStorageCapacityIsolation: true, }, { name: "capacity property does not have ephemeral-storage, cadvisor interface is nil", cm: &containerManagerImpl{ cadvisorInterface: nil, capacity:          v1.ResourceList{}, }, expectedNoEphemeralStorage:          true, enableLocalStorageCapacityIsolation: true, }, { name: "LocalStorageCapacityIsolation feature flag is disabled", cm: &containerManagerImpl{ cadvisorInterface: mockCadvisor, capacity: v1.ResourceList{ v1.ResourceCPU:    resource.MustParse("4"), v1.ResourceMemory: resource.MustParse("16G"), }, }, expectedNoEphemeralStorage:          true, enableLocalStorageCapacityIsolation: false, }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, kubefeatures.LocalStorageCapacityIsolation, c.enableLocalStorageCapacityIsolation)() ret := c.cm.GetCapacity() if v, exists := ret[v1.ResourceEphemeralStorage]; !exists { if !c.expectedNoEphemeralStorage { t.Errorf("did not get any ephemeral storage data") } } else { if v.Value() != c.expectedResourceQuantity.Value() { t.Errorf("got unexpected %s value, expected %d, got %d", v1.ResourceEphemeralStorage, c.expectedResourceQuantity.Value(), v.Value()) } } }) } } </ins>


<ins> // +build !providerless /* Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package vsphere import ( "context" "fmt" "github.com/vmware/govmomi/property" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" "k8s.io/klog/v2" "k8s.io/legacy-cloud-providers/vsphere/vclib" ) type sharedDatastore struct { nodeManager         *NodeManager candidateDatastores []*vclib.DatastoreInfo } type hostInfo struct { hostUUID   string hostMOID   string datacenter string } const ( summary       = "summary" runtimeHost   = "summary.runtime.host" hostsProperty = "host" nameProperty  = "name" ) func (shared *sharedDatastore) getSharedDatastore(ctcx context.Context) (*vclib.DatastoreInfo, error) { nodes := shared.nodeManager.getNodes() // Segregate nodes according to VC-DC dcNodes := make(map[string][]NodeInfo) nodeHosts := make(map[string]hostInfo) for nodeName, node := range nodes { nodeInfo, err := shared.nodeManager.GetNodeInfoWithNodeObject(node) if err != nil { return nil, fmt.Errorf("unable to find node %s: %v", nodeName, err) } vcDC := nodeInfo.vcServer + nodeInfo.dataCenter.String() dcNodes[vcDC] = append(dcNodes[vcDC], nodeInfo) } for vcDC, nodes := range dcNodes { var hostInfos []hostInfo var err error hostInfos, err = shared.getNodeHosts(ctcx, nodes, vcDC) if err != nil { if vclib.IsManagedObjectNotFoundError(err) { klog.Warningf("SharedHost.getSharedDatastore: batch fetching of hosts failed - switching to fetching them individually.") hostInfos, err = shared.getEachNodeHost(ctcx, nodes, vcDC) if err != nil { klog.Errorf("SharedHost.getSharedDatastore: error fetching node hosts individually: %v", err) return nil, err } } else { return nil, err } } for _, host := range hostInfos { hostDCName := fmt.Sprintf("%s/%s", host.datacenter, host.hostMOID) nodeHosts[hostDCName] = host } } if len(nodeHosts) < 1 { msg := fmt.Sprintf("SharedHost.getSharedDatastore unable to find hosts associated with nodes") klog.Error(msg) return nil, fmt.Errorf("") } for _, datastoreInfo := range shared.candidateDatastores { dataStoreHosts, err := shared.getAttachedHosts(ctcx, datastoreInfo.Datastore) if err != nil { msg := fmt.Sprintf("error finding attached hosts to datastore %s: %v", datastoreInfo.Name(), err) klog.Error(msg) return nil, fmt.Errorf(msg) } if shared.isIncluded(dataStoreHosts, nodeHosts) { return datastoreInfo, nil } } return nil, fmt.Errorf("SharedHost.getSharedDatastore: unable to find any shared datastores") } // check if all of the nodeHosts are included in the dataStoreHosts func (shared *sharedDatastore) isIncluded(dataStoreHosts []hostInfo, nodeHosts map[string]hostInfo) bool { result := true for _, host := range nodeHosts { hostFound := false for _, targetHost := range dataStoreHosts { if host.hostUUID == targetHost.hostUUID && host.hostMOID == targetHost.hostMOID { hostFound = true } } if !hostFound { result = false } } return result } func (shared *sharedDatastore) getEachNodeHost(ctx context.Context, nodes []NodeInfo, dcVC string) ([]hostInfo, error) { var hosts []hostInfo for _, node := range nodes { host, err := node.vm.GetHost(ctx) if err != nil { klog.Errorf("SharedHost.getEachNodeHost: unable to find host for vm %s: %v", node.vm.InventoryPath, err) return nil, err } hosts = append(hosts, hostInfo{ hostUUID:   host.Summary.Hardware.Uuid, hostMOID:   host.Summary.Host.String(), datacenter: node.dataCenter.String(), }) } return hosts, nil } func (shared *sharedDatastore) getNodeHosts(ctx context.Context, nodes []NodeInfo, dcVC string) ([]hostInfo, error) { var vmRefs []types.ManagedObjectReference if len(nodes) < 1 { return nil, fmt.Errorf("no nodes found for dc-vc: %s", dcVC) } var nodeInfo NodeInfo for _, n := range nodes { nodeInfo = n vmRefs = append(vmRefs, n.vm.Reference()) } pc := property.DefaultCollector(nodeInfo.dataCenter.Client()) var vmoList []mo.VirtualMachine err := pc.Retrieve(ctx, vmRefs, []string{nameProperty, runtimeHost}, &vmoList) if err != nil { klog.Errorf("SharedHost.getNodeHosts: unable to fetch vms from datacenter %s: %w", nodeInfo.dataCenter.String(), err) return nil, err } var hostMoList []mo.HostSystem var hostRefs []types.ManagedObjectReference for _, vmo := range vmoList { if vmo.Summary.Runtime.Host == nil { msg := fmt.Sprintf("SharedHost.getNodeHosts: no host associated with vm %s", vmo.Name) klog.Error(msg) return nil, fmt.Errorf(msg) } hostRefs = append(hostRefs, vmo.Summary.Runtime.Host.Reference()) } pc = property.DefaultCollector(nodeInfo.dataCenter.Client()) err = pc.Retrieve(ctx, hostRefs, []string{summary}, &hostMoList) if err != nil { klog.Errorf("SharedHost.getNodeHosts: unable to fetch hosts from datacenter %s: %w", nodeInfo.dataCenter.String(), err) return nil, err } var hosts []hostInfo for _, host := range hostMoList { hosts = append(hosts, hostInfo{hostMOID: host.Summary.Host.String(), hostUUID: host.Summary.Hardware.Uuid, datacenter: nodeInfo.dataCenter.String()}) } return hosts, nil } func (shared *sharedDatastore) getAttachedHosts(ctx context.Context, datastore *vclib.Datastore) ([]hostInfo, error) { var ds mo.Datastore pc := property.DefaultCollector(datastore.Client()) err := pc.RetrieveOne(ctx, datastore.Reference(), []string{hostsProperty}, &ds) if err != nil { return nil, err } mounts := make(map[types.ManagedObjectReference]types.DatastoreHostMount) var refs []types.ManagedObjectReference for _, host := range ds.Host { refs = append(refs, host.Key) mounts[host.Key] = host } var hs []mo.HostSystem err = pc.Retrieve(ctx, refs, []string{summary}, &hs) if err != nil { return nil, err } var hosts []hostInfo for _, h := range hs { hosts = append(hosts, hostInfo{hostUUID: h.Summary.Hardware.Uuid, hostMOID: h.Summary.Host.String()}) } return hosts, nil } </ins>


return diskUUID, nil } <ins> // GetHost returns host of the virtual machine func (vm *VirtualMachine) GetHost(ctx context.Context) (mo.HostSystem, error) { host, err := vm.HostSystem(ctx) var hostSystemMo mo.HostSystem if err != nil { klog.Errorf("Failed to get host system for VM: %q. err: %+v", vm.InventoryPath, err) return hostSystemMo, err } s := object.NewSearchIndex(vm.Client()) err = s.Properties(ctx, host.Reference(), []string{"summary"}, &hostSystemMo) if err != nil { klog.Errorf("Failed to retrieve datastores for host: %+v. err: %+v", host, err) return hostSystemMo, err } return hostSystemMo, nil } </ins> // DetachDisk detaches the disk specified by vmDiskPath func (vm *VirtualMachine) DetachDisk(ctx context.Context, vmDiskPath string) error { device, err := vm.getVirtualDeviceByPath(ctx, vmDiskPath)


if len(zonesToSearch) == 0 { // If zone is not provided, get the shared datastore across all node VMs. klog.V(4).Infof("Validating if datastore %s is shared across all node VMs", datastoreName) <del> sharedDsList, err = getSharedDatastoresInK8SCluster(ctx, vs.nodeManager) </del> <ins> sharedDSFinder := &sharedDatastore{ nodeManager:         vs.nodeManager, candidateDatastores: candidateDatastoreInfos, } datastoreInfo, err = sharedDSFinder.getSharedDatastore(ctx) </ins> if err != nil { klog.Errorf("Failed to get shared datastore: %+v", err) return "", err } <del> // Prepare error msg to be used later, if required. err = fmt.Errorf("The specified datastore %s is not a shared datastore across node VMs", datastoreName) </del> <ins> if datastoreInfo == nil { err = fmt.Errorf("The specified datastore %s is not a shared datastore across node VMs", datastoreName) klog.Error(err) return "", err } </ins> } else { // If zone is provided, get the shared datastores in that zone. klog.V(4).Infof("Validating if datastore %s is in zone %s ", datastoreName, zonesToSearch)


klog.Errorf("Failed to find a shared datastore matching zone %s. err: %+v", zonesToSearch, err) return "", err } <del> // Prepare error msg to be used later, if required. err = fmt.Errorf("The specified datastore %s does not match the provided zones : %s", datastoreName, zonesToSearch) } found := false // Check if the selected datastore belongs to the list of shared datastores computed. for _, sharedDs := range sharedDsList { if datastoreInfo, found = candidateDatastores[sharedDs.Info.Url]; found { klog.V(4).Infof("Datastore validation succeeded") found = true break </del> <ins> found := false for _, sharedDs := range sharedDsList { if datastoreInfo, found = candidateDatastores[sharedDs.Info.Url]; found { klog.V(4).Infof("Datastore validation succeeded") found = true break } } if !found { err = fmt.Errorf("The specified datastore %s does not match the provided zones : %s", datastoreName, zonesToSearch) klog.Error(err) return "", err </ins> } <del> } if !found { klog.Error(err) return "", err </del> } } }


LOG_FILE="${LOG_FILE:-${TMP_DIR}/update-vendor.log}" kube::log::status "logfile at ${LOG_FILE}" <ins> function finish { ret=$? if [[ ${ret} != 0 ]]; then echo "An error has occurred. Please see more details in ${LOG_FILE}" fi exit ${ret} } trap finish EXIT </ins> if [ -z "${BASH_XTRACEFD:-}" ]; then exec 19> "${LOG_FILE}" export BASH_XTRACEFD="19"


"context" "fmt" "math/rand" <del> "sort" </del> "sync" "sync/atomic" "time"


return feasibleNodes, nil } <del> type pluginScores struct { Plugin           string Score            int64 AverageNodeScore float64 } </del> // prioritizeNodes prioritizes the nodes by running the score plugins, // which return a score for each node from the call to RunScorePlugins(). // The scores from each plugin are added together to make the score for that node, then


return nil, scoreStatus.AsError() } <ins> if klog.V(10).Enabled() { for plugin, nodeScoreList := range scoresMap { for _, nodeScore := range nodeScoreList { klog.InfoS("Plugin scored node for pod", "pod", klog.KObj(pod), "plugin", plugin, "node", nodeScore.Name, "score", nodeScore.Score) } } } </ins> // Summarize all scores. result := make(framework.NodeScoreList, 0, len(nodes)) for i := range nodes { result = append(result, framework.NodeScore{Name: nodes[i].Name, Score: 0}) for j := range scoresMap {


} } <del> if klog.V(4).Enabled() { logPluginScores(nodes, scoresMap, pod) } </del> if len(g.extenders) != 0 && nodes != nil { var mu sync.Mutex var wg sync.WaitGroup


} } <del> if klog.V(4).Enabled() { </del> <ins> if klog.V(10).Enabled() { </ins> for i := range result { klog.InfoS("Calculated node's final score for pod", "pod", klog.KObj(pod), "node", result[i].Name, "score", result[i].Score) }


return result, nil } <del> // logPluginScores adds summarized plugin score logging. The goal of this block is to show the highest scoring plugins on // each node, and the average score for those plugins across all nodes. func logPluginScores(nodes []*v1.Node, scoresMap framework.PluginToNodeScores, pod *v1.Pod) { totalPluginScores := make(map[string]int64) for j := range scoresMap { for i := range nodes { totalPluginScores[j] += scoresMap[j][i].Score } } // Build a map of Nodes->PluginScores on that node nodeToPluginScores := make(framework.PluginToNodeScores, len(nodes)) for _, node := range nodes { nodeToPluginScores[node.Name] = make(framework.NodeScoreList, len(scoresMap)) } // Convert the scoresMap (which contains Plugins->NodeScores) to the Nodes->PluginScores map for plugin, nodeScoreList := range scoresMap { for _, nodeScore := range nodeScoreList { klog.V(10).InfoS("Plugin scored node for pod", "pod", klog.KObj(pod), "plugin", plugin, "node", nodeScore.Name, "score", nodeScore.Score) nodeToPluginScores[nodeScore.Name] = append(nodeToPluginScores[nodeScore.Name], framework.NodeScore{Name: plugin, Score: nodeScore.Score}) } } for node, scores := range nodeToPluginScores { // Get the top 3 scoring plugins for each node sort.Slice(scores, func(i, j int) bool { return scores[i].Score > scores[j].Score }) var topScores []pluginScores for _, score := range scores { pluginScore := pluginScores{ Plugin:           score.Name, Score:            score.Score, AverageNodeScore: float64(totalPluginScores[score.Name]) / float64(len(nodes)), } topScores = append(topScores, pluginScore) if len(topScores) == 3 { break } } klog.InfoS("Top 3 plugins for pod on node", "pod", klog.KObj(pod), "node", node, "scores", topScores, ) } } </del> // NewGenericScheduler creates a genericScheduler object. func NewGenericScheduler( cache internalcache.Cache,


}, "required": [ "currentReplicas", <del> "desiredReplicas", "conditions" </del> <ins> "desiredReplicas" </ins> ], "type": "object" },


// conditions is the set of conditions required for this autoscaler to scale its target, // and indicates whether or not those conditions are met. <ins> // +optional </ins> repeated HorizontalPodAutoscalerCondition conditions = 6; }


// conditions is the set of conditions required for this autoscaler to scale its target, // and indicates whether or not those conditions are met. <ins> // +optional </ins> Conditions []HorizontalPodAutoscalerCondition `json:"conditions" protobuf:"bytes,6,rep,name=conditions"` }


// Check that container has restarted ginkgo.By("Waiting for container to restart") restarts := int32(0) <del> err = wait.PollImmediate(10*time.Second, 2*time.Minute, func() (bool, error) { </del> <ins> err = wait.PollImmediate(10*time.Second, framework.PodStartTimeout, func() (bool, error) { </ins> pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), pod.Name, metav1.GetOptions{}) if err != nil { return false, err


ginkgo.By("Waiting for container to stop restarting") stableCount := int(0) stableThreshold := int(time.Minute / framework.Poll) <del> err = wait.PollImmediate(framework.Poll, 2*time.Minute, func() (bool, error) { </del> <ins> err = wait.PollImmediate(framework.Poll, framework.PodStartTimeout, func() (bool, error) { </ins> pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), pod.Name, metav1.GetOptions{}) if err != nil { return false, err


var ( // TODO: Deprecate gitMajor and gitMinor, use only gitVersion instead. gitMajor     string = "0"              // major version, always numeric <del> gitMinor     string = "2+"             // minor version, numeric possibly followed by "+" gitVersion   string = "v0.2-dev"       // version from git, output of $(git describe) </del> <ins> gitMinor     string = "3+"             // minor version, numeric possibly followed by "+" gitVersion   string = "v0.3-dev"       // version from git, output of $(git describe) </ins> gitCommit    string = ""               // sha1 from git, output of $(git rev-parse HEAD) gitTreeState string = "not a git tree" // state of git tree, either "clean" or "dirty" )


terminationGracePeriodSeconds: 600 hostNetwork: true containers: <del> - image: gcr.io/k8s-ingress-image-push/ingress-gce-glbc-amd64:v1.11.0 </del> <ins> - image: gcr.io/k8s-ingress-image-push/ingress-gce-glbc-amd64:v1.12.0 </ins> livenessProbe: httpGet: path: /healthz


metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" <ins> utilnet "k8s.io/apimachinery/pkg/util/net" </ins> utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait"


} kubeClientConfigOverrides(s, clientConfig) <del> closeAllConns, err := updateDialer(clientConfig) if err != nil { return nil, nil, err </del> <ins> // Kubelet needs to be able to recover from stale http connections. // HTTP2 has a mechanism to detect broken connections by sending periodical pings. // HTTP1 only can have one persistent connection, and it will close all Idle connections // once the Kubelet heartbeat fails. However, since there are many edge cases that we can't // control, users can still opt-in to the previous behavior for closing the connections by // setting the environment variable DISABLE_HTTP2. var closeAllConns func() if s := os.Getenv("DISABLE_HTTP2"); len(s) > 0 { klog.InfoS("HTTP2 has been explicitly disabled, updating Kubelet client Dialer to forcefully close active connections on heartbeat failures") closeAllConns, err = updateDialer(clientConfig) if err != nil { return nil, nil, err } } else { closeAllConns = func() { utilnet.CloseIdleConnectionsFor(clientConfig.Transport) } </ins> } return clientConfig, closeAllConns, nil }


} } <ins> // CloseIdleConnectionsFor close idles connections for the Transport. // If the Transport is wrapped it iterates over the wrapped round trippers // until it finds one that implements the CloseIdleConnections method. // If the Transport does not have a CloseIdleConnections method // then this function does nothing. func CloseIdleConnectionsFor(transport http.RoundTripper) { if transport == nil { return } type closeIdler interface { CloseIdleConnections() } switch transport := transport.(type) { case closeIdler: transport.CloseIdleConnections() case RoundTripperWrapper: CloseIdleConnectionsFor(transport.WrappedRoundTripper()) default: klog.Warningf("unknown transport type: %T", transport) } } </ins> type TLSClientConfigHolder interface { TLSClientConfig() *tls.Config }


"k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" utilnet "k8s.io/apimachinery/pkg/util/net" <ins> "k8s.io/apimachinery/pkg/util/wait" </ins> ) type tcpLB struct {


} } <ins> // 1. connect to https server with http1.1 using a TCP proxy // 2. the connection has keepalive enabled so it will be reused // 3. break the TCP connection stopping the proxy // 4. close the idle connection to force creating a new connection // 5. count that there are 2 connection to the server (we didn't reuse the original connection) func TestReconnectBrokenTCP_HTTP1(t *testing.T) { ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "Hello, %s", r.Proto) })) ts.EnableHTTP2 = false ts.StartTLS() defer ts.Close() u, err := url.Parse(ts.URL) if err != nil { t.Fatalf("failed to parse URL from %q: %v", ts.URL, err) } lb := newLB(t, u.Host) defer lb.ln.Close() stopCh := make(chan struct{}) go lb.serve(stopCh) transport, ok := ts.Client().Transport.(*http.Transport) if !ok { t.Fatal("failed to assert *http.Transport") } config := &Config{ Host:      "https://" + lb.ln.Addr().String(), Transport: utilnet.SetTransportDefaults(transport), // large timeout, otherwise the broken connection will be cleaned by it Timeout: wait.ForeverTestTimeout, // These fields are required to create a REST client. ContentConfig: ContentConfig{ GroupVersion:         &schema.GroupVersion{}, NegotiatedSerializer: &serializer.CodecFactory{}, }, } config.TLSClientConfig.NextProtos = []string{"http/1.1"} client, err := RESTClientFor(config) if err != nil { t.Fatalf("failed to create REST client: %v", err) } data, err := client.Get().AbsPath("/").DoRaw(context.TODO()) if err != nil { t.Fatalf("unexpected err: %s: %v", data, err) } if string(data) != "Hello, HTTP/1.1" { t.Fatalf("unexpected response: %s", data) } // Deliberately let the LB stop proxying traffic for the current // connection. This mimics a broken TCP connection that's not properly // closed. close(stopCh) stopCh = make(chan struct{}) go lb.serve(stopCh) // Close the idle connections utilnet.CloseIdleConnectionsFor(client.Client.Transport) // If the client didn't close the idle connections, the broken connection // would still be in the connection pool, the following request would // then reuse the broken connection instead of creating a new one, and // thus would fail. data, err = client.Get().AbsPath("/").DoRaw(context.TODO()) if err != nil { t.Fatalf("unexpected err: %v", err) } if string(data) != "Hello, HTTP/1.1" { t.Fatalf("unexpected response: %s", data) } dials := atomic.LoadInt32(&lb.dials) if dials != 2 { t.Fatalf("expected %d dials, got %d", 2, dials) } } // 1. connect to https server with http1.1 using a TCP proxy making the connection to timeout // 2. the connection has keepalive enabled so it will be reused // 3. close the in-flight connection to force creating a new connection // 4. count that there are 2 connection on the LB but only one succeeds func TestReconnectBrokenTCPInFlight_HTTP1(t *testing.T) { done := make(chan struct{}) defer close(done) received := make(chan struct{}) ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/hang" { conn, _, _ := w.(http.Hijacker).Hijack() close(received) <-done conn.Close() } fmt.Fprintf(w, "Hello, %s", r.Proto) })) ts.EnableHTTP2 = false ts.StartTLS() defer ts.Close() u, err := url.Parse(ts.URL) if err != nil { t.Fatalf("failed to parse URL from %q: %v", ts.URL, err) } lb := newLB(t, u.Host) defer lb.ln.Close() stopCh := make(chan struct{}) go lb.serve(stopCh) transport, ok := ts.Client().Transport.(*http.Transport) if !ok { t.Fatal("failed to assert *http.Transport") } config := &Config{ Host:      "https://" + lb.ln.Addr().String(), Transport: utilnet.SetTransportDefaults(transport), // Use something extraordinary large to not hit the timeout Timeout: wait.ForeverTestTimeout, // These fields are required to create a REST client. ContentConfig: ContentConfig{ GroupVersion:         &schema.GroupVersion{}, NegotiatedSerializer: &serializer.CodecFactory{}, }, } config.TLSClientConfig.NextProtos = []string{"http/1.1"} client, err := RESTClientFor(config) if err != nil { t.Fatalf("failed to create REST client: %v", err) } // The request will connect, hang and eventually time out // but we can use a context to close once the test is done // we are only interested in have an inflight connection ctx, cancel := context.WithCancel(context.Background()) reqErrCh := make(chan error, 1) defer close(reqErrCh) go func() { _, err = client.Get().AbsPath("/hang").DoRaw(ctx) reqErrCh <- err }() // wait until it connect to the server select { case <-received: case <-time.After(wait.ForeverTestTimeout): t.Fatal("Test timed out waiting for first request to fail") } // Deliberately let the LB stop proxying traffic for the current // connection. This mimics a broken TCP connection that's not properly // closed. close(stopCh) stopCh = make(chan struct{}) go lb.serve(stopCh) // New request will fail if tries to reuse the connection data, err := client.Get().AbsPath("/").DoRaw(context.Background()) if err != nil { t.Fatalf("unexpected err: %v", err) } if string(data) != "Hello, HTTP/1.1" { t.Fatalf("unexpected response: %s", data) } dials := atomic.LoadInt32(&lb.dials) if dials != 2 { t.Fatalf("expected %d dials, got %d", 2, dials) } // cancel the in-flight connection cancel() select { case <-reqErrCh: if err == nil { t.Fatal("Connection succeeded but was expected to timeout") } case <-time.After(10 * time.Second): t.Fatal("Test timed out waiting for the request to fail") } } </ins> func TestRestClientTimeout(t *testing.T) { ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { time.Sleep(2 * time.Second)


metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" <ins> "k8s.io/apimachinery/pkg/runtime/serializer" </ins> "k8s.io/apimachinery/pkg/runtime/serializer/streaming" "k8s.io/apimachinery/pkg/util/diff" "k8s.io/apimachinery/pkg/util/httpstream" "k8s.io/apimachinery/pkg/util/intstr" <ins> utilnet "k8s.io/apimachinery/pkg/util/net" </ins> "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes/scheme" restclientwatch "k8s.io/client-go/rest/watch"


func testRESTClientWithConfig(t testing.TB, srv *httptest.Server, contentConfig ClientContentConfig) *RESTClient { base, _ := url.Parse("http://localhost") <ins> var c *http.Client </ins> if srv != nil { var err error base, err = url.Parse(srv.URL) if err != nil { t.Fatalf("failed to parse test URL: %v", err) } <ins> c = srv.Client() </ins> } versionedAPIPath := defaultResourcePathWithPrefix("", "", "", "") <del> client, err := NewRESTClient(base, versionedAPIPath, contentConfig, nil, nil) </del> <ins> client, err := NewRESTClient(base, versionedAPIPath, contentConfig, nil, c) </ins> if err != nil { t.Fatalf("failed to create a client: %v", err) }


}) } } <ins> func TestReuseRequest(t *testing.T) { var tests = []struct { name        string enableHTTP2 bool }{ {"HTTP1", false}, {"HTTP2", true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(r.RemoteAddr)) })) ts.EnableHTTP2 = tt.enableHTTP2 ts.StartTLS() defer ts.Close() ctx, cancel := context.WithCancel(context.Background()) defer cancel() c := testRESTClient(t, ts) req1, err := c.Verb("GET"). Prefix("foo"). DoRaw(ctx) if err != nil { t.Fatalf("Unexpected error: %v", err) } req2, err := c.Verb("GET"). Prefix("foo"). DoRaw(ctx) if err != nil { t.Fatalf("Unexpected error: %v", err) } if string(req1) != string(req2) { t.Fatalf("Expected %v to be equal to %v", string(req1), string(req2)) } }) } } func TestHTTP1DoNotReuseRequestAfterTimeout(t *testing.T) { var tests = []struct { name        string enableHTTP2 bool }{ {"HTTP1", false}, {"HTTP2", true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { done := make(chan struct{}) ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Logf("TEST Connected from %v on %vn", r.RemoteAddr, r.URL.Path) if r.URL.Path == "/hang" { t.Logf("TEST hanging %vn", r.RemoteAddr) <-done } w.Write([]byte(r.RemoteAddr)) })) ts.EnableHTTP2 = tt.enableHTTP2 ts.StartTLS() defer ts.Close() // close hanging connection before shutting down the http server defer close(done) ctx, cancel := context.WithCancel(context.Background()) defer cancel() transport, ok := ts.Client().Transport.(*http.Transport) if !ok { t.Fatalf("failed to assert *http.Transport") } config := &Config{ Host:      ts.URL, Transport: utilnet.SetTransportDefaults(transport), Timeout:   100 * time.Millisecond, // These fields are required to create a REST client. ContentConfig: ContentConfig{ GroupVersion:         &schema.GroupVersion{}, NegotiatedSerializer: &serializer.CodecFactory{}, }, } if !tt.enableHTTP2 { config.TLSClientConfig.NextProtos = []string{"http/1.1"} } c, err := RESTClientFor(config) if err != nil { t.Fatalf("failed to create REST client: %v", err) } req1, err := c.Verb("GET"). Prefix("foo"). DoRaw(ctx) if err != nil { t.Fatalf("Unexpected error: %v", err) } _, err = c.Verb("GET"). Prefix("/hang"). DoRaw(ctx) if err == nil { t.Fatalf("Expected error") } req2, err := c.Verb("GET"). Prefix("foo"). DoRaw(ctx) if err != nil { t.Fatalf("Unexpected error: %v", err) } // http1 doesn't reuse the connection after it times if tt.enableHTTP2 != (string(req1) == string(req2)) { if tt.enableHTTP2 { t.Fatalf("Expected %v to be the same as %v", string(req1), string(req2)) } else { t.Fatalf("Expected %v to be different to %v", string(req1), string(req2)) } } }) } } func TestTransportConcurrency(t *testing.T) { const numReqs = 10 var tests = []struct { name        string enableHTTP2 bool }{ {"HTTP1", false}, {"HTTP2", true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Logf("Connected from %v %v", r.RemoteAddr, r.URL) fmt.Fprintf(w, "%v", r.FormValue("echo")) })) ts.EnableHTTP2 = tt.enableHTTP2 ts.StartTLS() defer ts.Close() var wg sync.WaitGroup wg.Add(numReqs) c := testRESTClient(t, ts) reqs := make(chan string) defer close(reqs) for i := 0; i < 4; i++ { go func() { for req := range reqs { res, err := c.Get().Param("echo", req).DoRaw(context.Background()) if err != nil { t.Errorf("error on req %s: %v", req, err) wg.Done() continue } if string(res) != req { t.Errorf("body of req %s = %q; want %q", req, res, req) } wg.Done() } }() } for i := 0; i < numReqs; i++ { reqs <- fmt.Sprintf("request-%d", i) } wg.Wait() }) } } </ins>


- [Deprecation of PodSecurityPolicy](#deprecation-of-podsecuritypolicy) - [Kubernetes API Reference Documentation](#kubernetes-api-reference-documentation) - [Kustomize Updates in Kubectl](#kustomize-updates-in-kubectl) <del> - [Default Container Labels](#default-container-labels) </del> <ins> - [Default Container Annotation](#default-container-annotation) </ins> - [Immutable Secrets and ConfigMaps](#immutable-secrets-and-configmaps) - [Structured Logging in Kubelet](#structured-logging-in-kubelet) - [Storage Capacity Tracking](#storage-capacity-tracking)


[Kustomize](https://github.com/kubernetes-sigs/kustomize) version in kubectl had a jump from v2.0.3 to [v4.0.5](https://github.com/kubernetes/kubernetes/pull/98946). Kustomize is now treated as a library and future updates will be less sporadic. <del> ### Default Container Labels </del> <ins> ### Default Container Annotation </ins> <del> Pod with multiple containers can use `kubectl.kubernetes.io/default-container` label to have a container preselected for kubectl commands. More can be read in [KEP-2227](https://github.com/kubernetes/enhancements/blob/master/keps/sig-cli/2227-kubectl-default-container/README.md). </del> <ins> Pod with multiple containers can use `kubectl.kubernetes.io/default-container` annotation to have a container preselected for kubectl commands. More can be read in [KEP-2227](https://github.com/kubernetes/enhancements/blob/master/keps/sig-cli/2227-kubectl-default-container/README.md). </ins> ### Immutable Secrets and ConfigMaps


w.Write(LEVEL_2, "NodeName:t%sn", nodeNameText) zoneText := "<unset>" <del> if endpoint.NodeName != nil { </del> <ins> if endpoint.Zone != nil { </ins> zoneText = *endpoint.Zone } w.Write(LEVEL_2, "Zone:t%sn", zoneText)


Addresses:  []string{"1.2.3.6", "1.2.3.7"}, Conditions: discoveryv1.EndpointConditions{Ready: utilpointer.BoolPtr(true)}, TargetRef:  &corev1.ObjectReference{Kind: "Pod", Name: "test-124"}, <ins> NodeName:   utilpointer.StringPtr("node-2"), </ins> }, }, Ports: []discoveryv1.EndpointPort{


Ready:    true Hostname:   <unset> TargetRef:  Pod/test-124 <del> NodeName:   <unset> </del> <ins> NodeName:   node-2 </ins> Zone:       <unset> Events:         <none>` + "n", },


# Windows path it'll use it instead of the default unzipper. # See: https://github.com/containerd/containerd/issues/1896 Add-MachineEnvironmentPath -Path $PIGZ_ROOT <del> # Add process exclusion for Windows Defender to boost performance. </del> <ins> # Add process exclusion for Windows Defender to boost performance. </ins> Add-MpPreference -ExclusionProcess "$PIGZ_ROOTunpigz.exe" Log-Output "Installed Pigz $PIGZ_VERSION" } else {


# TODO(pjh): move the logging agent code below into a separate # module; it was put here temporarily to avoid disrupting the file layout in # the K8s release machinery. <del> $LOGGINGAGENT_VERSION = '1.6.0' </del> <ins> $LOGGINGAGENT_VERSION = '1.7.3' </ins> $LOGGINGAGENT_ROOT = 'C:fluent-bit' $LOGGINGAGENT_SERVICE = 'fluent-bit' $LOGGINGAGENT_CMDLINE = '*fluent-bit.exe*' <del> $LOGGINGEXPORTER_VERSION = 'v0.10.3' </del> <ins> $LOGGINGEXPORTER_VERSION = 'v0.16.2' </ins> $LOGGINGEXPORTER_ROOT = 'C:flb-exporter' $LOGGINGEXPORTER_SERVICE = 'flb-exporter' $LOGGINGEXPORTER_CMDLINE = '*flb-exporter.exe*'


$fluentbit_parser_file = "$LOGGINGAGENT_ROOTconfparsers.conf" $PARSERS_CONFIG | Out-File -FilePath $fluentbit_parser_file -Encoding ASCII <ins> # Create directory for all the log position files. New-Item -Type Directory -Path "/var/run/google-fluentbit/pos-files/" </ins> Log-Output "Wrote logging config to $fluentbit_parser_file" }


[SERVICE] Flush         5 Grace         120 <del> Log_Level     debug </del> <ins> Log_Level     info </ins> Log_File      /var/log/fluentbit.log Daemon        off Parsers_File  parsers.conf


# # storage.backlog.mem_limit 5M [INPUT] Name         winlog Interval_Sec 2 # Channels Setup,Windows PowerShell Channels     application,system,security Tag          winevent.raw <del> DB           winlog.sqlite   # </del> <ins> DB           /var/run/google-fluentbit/pos-files/winlog.db </ins> # Json Log Example: # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text heren","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"}


Alias            kube_containers Tag              kube_<namespace_name>_<pod_name>_<container_name> Tag_Regex        (?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)- <del> Path             /var/log/containers/*.log </del> Mem_Buf_Limit    5MB Skip_Long_Lines  On Refresh_Interval 5 <del> DB               flb_kube.db # Settings from fluentd missing here. # tag reform.* # format json # time_key time # time_format %Y-%m-%dT%H:%M:%S.%NZ </del> <ins> Path             C:varlogcontainers*.log DB               /var/run/google-fluentbit/pos-files/flb_kube.db [FILTER] Name         parser Match        kube_* Key_Name     log Reserve_Data True Parser       docker Parser       containerd </ins> # Example: <del> # I0204 07:32:30.020537    3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537] </del> <ins> # I0928 03:15:50.440223    4880 main.go:51] Starting CSI-Proxy Server ... </ins> [INPUT] Name             tail <del> Alias            kubelet Tag              kubelet #Multiline        on #Multiline_Flush  5 </del> <ins> Alias            csi-proxy Tag              csi-proxy </ins> Mem_Buf_Limit    5MB Skip_Long_Lines  On Refresh_Interval 5 <del> Path /etc/kubernetes/logs/kubelet.log DB               /etc/kubernetes/logs/gcp-kubelet.db # Copied from fluentbit config. How is this used ? In match stages ? Parser_Firstline /^wd{4}/ Parser_1         ^(?<severity>w)(?<time>d{4} [^s]*)s+(?<pid>d+)s+(?<source>[^ ]]+)] (?<message>.*)/ # missing from fluentbit #   time_format %m%d %H:%M:%S.%N </del> <ins> Path             /etc/kubernetes/logs/csi-proxy.log DB               /var/run/google-fluentbit/pos-files/csi-proxy.db Multiline        On Parser_Firstline glog </ins> <del> # Example: # I0928 03:15:50.440223    4880 main.go:51] Starting CSI-Proxy Server ... </del> <ins> # I1118 21:26:53.975789       6 proxier.go:1096] Port "nodePort for kube-system/default-http-backend:http" (:31429/tcp) was open before and is still needed </ins> [INPUT] Name             tail <del> Alias            csi-proxy Tag              csi-proxy #Multiline        on #Multiline_Flush  5 </del> <ins> Alias            kube-proxy Tag              kube-proxy </ins> Mem_Buf_Limit    5MB Skip_Long_Lines  On Refresh_Interval 5 <del> Path             /etc/kubernetes/logs/csi-proxy.log DB               /etc/kubernetes/logs/gcp-csi-proxy.db # Copied from fluentbit config. How is this used ? In match stages ? Parser_Firstline /^wd{4}/ Parser_1         ^(?<severity>w)(?<time>d{4} [^s]*)s+(?<pid>d+)s+(?<source>[^ ]]+)] (?<message>.*)/ # missing from fluentbit #   time_format %m%d %H:%M:%S.%N </del> <ins> Path             /etc/kubernetes/logs/kube-proxy.log DB               /var/run/google-fluentbit/pos-files/kube-proxy.db Multiline        On Parser_Firstline glog </ins> # Example: # time="2019-12-10T21:27:59.836946700Z" level=info msg="loading plugin "io.containerd.grpc.v1.cri"..." type=io.containerd.grpc.v1 [INPUT] Name             tail Alias            container-runtime <del> Tag container-runtime #Multiline        on #Multiline_Flush  5 </del> <ins> Tag              container-runtime </ins> Mem_Buf_Limit    5MB Skip_Long_Lines  On Refresh_Interval 5 Path             /etc/kubernetes/logs/containerd.log <del> DB               /etc/kubernetes/logs/gcp-containerd.log.pos # Copied from fluentbit config. How is this used ? In match stages ? Parser_Firstline /^wd{4}/ Parser_1         ^(?<severity>w)(?<time>d{4} [^s]*)s+(?<pid>d+)s+(?<source>[^ ]]+)] (?<message>.*)/ </del> <ins> DB               /var/run/google-fluentbit/pos-files/container-runtime.db # TODO: Add custom parser for containerd logs once format is settled. </ins> <del> # missing from fluentbit #   time_format %m%d %H:%M:%S.%N # I1118 21:26:53.975789       6 proxier.go:1096] Port "nodePort for kube-system/default-http-backend:http" (:31429/tcp) was open before and is still needed </del> <ins> # Example: # I0204 07:32:30.020537    3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537] </ins> [INPUT] Name             tail <del> Alias            kube-proxy Tag              kube-proxy #Multiline        on #Multiline_Flush  5 </del> <ins> Alias            kubelet Tag              kubelet </ins> Mem_Buf_Limit    5MB Skip_Long_Lines  On Refresh_Interval 5 <del> Path             /etc/kubernetes/logs/kube-proxy.log DB               /etc/kubernetes/logs/gcp-kubeproxy.db # Copied from fluentbit config. How is this used ? In match stages ? Parser_Firstline /^wd{4}/ Parser_1         ^(?<severity>w)(?<time>d{4} [^s]*)s+(?<pid>d+)s+(?<source>[^ ]]+)] (?<message>.*)/ # missing from fluentbit #   time_format %m%d %H:%M:%S.%N </del> <ins> Path             /etc/kubernetes/logs/kubelet.log DB               /var/run/google-fluentbit/pos-files/kubelet.db Multiline        On Parser_Firstline glog </ins> [FILTER] Name        modify Match       * Hard_rename log message <del> # [OUTPUT] #    Name        http #    Match       * #    Host        127.0.0.1 #    Port        2021 #    URI         /logs #    header_tag  FLUENT-TAG #    Format      msgpack #    Retry_Limit 2 </del> <ins> [FILTER] Name         parser Match        kube_* Key_Name     message Reserve_Data True Parser       glog Parser       json </ins> [OUTPUT] <del> name  stackdriver match * </del> <ins> Name        http Match       * Host        127.0.0.1 Port        2021 URI         /logs header_tag  FLUENT-TAG Format      msgpack Retry_Limit 2 </ins> '@ # Fluentbit parsers config file $PARSERS_CONFIG = @' [PARSER] Name        docker Format      json


Time_Key    timestamp Time_Format %Y-%m-%dT%H:%M:%S.%L%z <del> # ---------- [PARSER] Name   json Format json Time_Key time Time_Format %d/%b/%Y:%H:%M:%S %z [PARSER] Name         docker Format       json Time_Key     time Time_Format  %Y-%m-%dT%H:%M:%S.%L Time_Keep    On </del> [PARSER] Name        syslog-rfc5424 Format      regex


$PARSERS_CONFIG | Out-File -FilePath $fluentbit_parser_file -Encoding ASCII # Create directory for all the log position files. <del> New-Item -Type Directory -Path "/var/run/google-fluentbit/pos-files/" </del> <ins> New-Item -Type Directory -Path "/var/run/google-fluentbit/pos-files/" -Force | Out-Null </ins> Log-Output "Wrote logging config to $fluentbit_parser_file" }


func updatePod(client clientset.Interface, pod *v1.Pod, condition *v1.PodCondition, nominatedNode string) error { klog.V(3).InfoS("Updating pod condition", "pod", klog.KObj(pod), "conditionType", condition.Type, "conditionStatus", condition.Status, "conditionReason", condition.Reason) <del> podCopy := pod.DeepCopy() </del> <ins> podStatusCopy := pod.Status.DeepCopy() </ins> // NominatedNodeName is updated only if we are trying to set it, and the value is // different from the existing one. <del> if !podutil.UpdatePodCondition(&podCopy.Status, condition) && </del> <ins> if !podutil.UpdatePodCondition(podStatusCopy, condition) && </ins> (len(nominatedNode) == 0 || pod.Status.NominatedNodeName == nominatedNode) { return nil } if nominatedNode != "" { <del> podCopy.Status.NominatedNodeName = nominatedNode </del> <ins> podStatusCopy.NominatedNodeName = nominatedNode </ins> } <del> return util.PatchPod(client, pod, podCopy) </del> <ins> return util.PatchPodStatus(client, pod, podStatusCopy) </ins> } // assume signals to the cache that a pod is already in the cache, so that binding can be asynchronous.


return GetPodStartTime(pod1).Before(GetPodStartTime(pod2)) } <del> // PatchPod calculates the delta bytes change from <old> to <new>, </del> <ins> // PatchPodStatus calculates the delta bytes change from <old.Status> to <newStatus>, </ins> // and then submit a request to API server to patch the pod changes. <del> func PatchPod(cs kubernetes.Interface, old *v1.Pod, new *v1.Pod) error { oldData, err := json.Marshal(old) </del> <ins> func PatchPodStatus(cs kubernetes.Interface, old *v1.Pod, newStatus *v1.PodStatus) error { if newStatus == nil { return nil } oldData, err := json.Marshal(v1.Pod{Status: old.Status}) </ins> if err != nil { return err } <del> newData, err := json.Marshal(new) </del> <ins> newData, err := json.Marshal(v1.Pod{Status: *newStatus}) </ins> if err != nil { return err }


if len(p.Status.NominatedNodeName) == 0 { continue } <del> podCopy := p.DeepCopy() podCopy.Status.NominatedNodeName = "" if err := PatchPod(cs, p, podCopy); err != nil { </del> <ins> podStatusCopy := p.Status.DeepCopy() podStatusCopy.NominatedNodeName = "" if err := PatchPodStatus(cs, p, podStatusCopy); err != nil { </ins> errs = append(errs, err) } }


package util import ( <ins> "context" </ins> "fmt" <ins> "github.com/google/go-cmp/cmp" </ins> "testing" "time"


}) } } <ins> func TestPatchPodStatus(t *testing.T) { tests := []struct { name           string pod            v1.Pod statusToUpdate v1.PodStatus }{ { name: "Should update pod conditions successfully", pod: v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", Name:      "pod1", }, Spec: v1.PodSpec{ ImagePullSecrets: []v1.LocalObjectReference{{Name: "foo"}}, }, }, statusToUpdate: v1.PodStatus{ Conditions: []v1.PodCondition{ { Type:   v1.PodScheduled, Status: v1.ConditionFalse, }, }, }, }, { // ref: #101697, #94626 - ImagePullSecrets are allowed to have empty secret names // which would fail the 2-way merge patch generation on Pod patches // due to the mergeKey being the name field name: "Should update pod conditions successfully on a pod Spec with secrets with empty name", pod: v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", Name:      "pod2", }, Spec: v1.PodSpec{ // this will serialize to imagePullSecrets:[{}] ImagePullSecrets: make([]v1.LocalObjectReference, 1), }, }, statusToUpdate: v1.PodStatus{ Conditions: []v1.PodCondition{ { Type:   v1.PodScheduled, Status: v1.ConditionFalse, }, }, }, }, } client := clientsetfake.NewSimpleClientset() for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { _, err := client.CoreV1().Pods(tc.pod.Namespace).Create(context.TODO(), &tc.pod, metav1.CreateOptions{}) if err != nil { t.Fatal(err) } err = PatchPodStatus(client, &tc.pod, &tc.statusToUpdate) if err != nil { t.Fatal(err) } retrievedPod, err := client.CoreV1().Pods(tc.pod.Namespace).Get(context.TODO(), tc.pod.Name, metav1.GetOptions{}) if err != nil { t.Fatal(err) } if diff := cmp.Diff(tc.statusToUpdate, retrievedPod.Status); diff != "" { t.Errorf("unexpected pod status (-want,+got):n%s", diff) } }) } } </ins>


// FieldImmutableErrorMsg is a error message for field is immutable. const FieldImmutableErrorMsg string = `field is immutable` <del> const totalAnnotationSizeLimitB int = 256 * (1 << 10) // 256 kB </del> <ins> const TotalAnnotationSizeLimitB int = 256 * (1 << 10) // 256 kB </ins> // BannedOwners is a black list of object that are not allowed to be owners. var BannedOwners = map[schema.GroupVersionKind]struct{}{


// ValidateAnnotations validates that a set of annotations are correctly defined. func ValidateAnnotations(annotations map[string]string, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} <del> var totalSize int64 for k, v := range annotations { </del> <ins> for k := range annotations { </ins> for _, msg := range validation.IsQualifiedName(strings.ToLower(k)) { allErrs = append(allErrs, field.Invalid(fldPath, k, msg)) } <del> totalSize += (int64)(len(k)) + (int64)(len(v)) </del> } <del> if totalSize > (int64)(totalAnnotationSizeLimitB) { allErrs = append(allErrs, field.TooLong(fldPath, "", totalAnnotationSizeLimitB)) </del> <ins> if err := ValidateAnnotationsSize(annotations); err != nil { allErrs = append(allErrs, field.TooLong(fldPath, "", TotalAnnotationSizeLimitB)) </ins> } return allErrs } <ins> func ValidateAnnotationsSize(annotations map[string]string) error { var totalSize int64 for k, v := range annotations { totalSize += (int64)(len(k)) + (int64)(len(v)) } if totalSize > (int64)(TotalAnnotationSizeLimitB) { return fmt.Errorf("annotations size %d is larger than limit %d", totalSize, TotalAnnotationSizeLimitB) } return nil } </ins> func validateOwnerReference(ownerReference metav1.OwnerReference, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} gvk := schema.FromAPIVersionAndKind(ownerReference.APIVersion, ownerReference.Kind)


{"1234/5678": "bar"}, {"1.2.3.4/5678": "bar"}, {"UpperCase123": "bar"}, <del> {"a": strings.Repeat("b", totalAnnotationSizeLimitB-1)}, </del> <ins> {"a": strings.Repeat("b", TotalAnnotationSizeLimitB-1)}, </ins> { <del> "a": strings.Repeat("b", totalAnnotationSizeLimitB/2-1), "c": strings.Repeat("d", totalAnnotationSizeLimitB/2-1), </del> <ins> "a": strings.Repeat("b", TotalAnnotationSizeLimitB/2-1), "c": strings.Repeat("d", TotalAnnotationSizeLimitB/2-1), </ins> }, } for i := range successCases {


} } totalSizeErrorCases := []map[string]string{ <del> {"a": strings.Repeat("b", totalAnnotationSizeLimitB)}, </del> <ins> {"a": strings.Repeat("b", TotalAnnotationSizeLimitB)}, </ins> { <del> "a": strings.Repeat("b", totalAnnotationSizeLimitB/2), "c": strings.Repeat("d", totalAnnotationSizeLimitB/2), </del> <ins> "a": strings.Repeat("b", TotalAnnotationSizeLimitB/2), "c": strings.Repeat("d", TotalAnnotationSizeLimitB/2), </ins> }, } for i := range totalSizeErrorCases {


corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" <ins> apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation" </ins> "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" ) <del> const totalAnnotationSizeLimitB int64 = 256 * (1 << 10) // 256 kB </del> type lastAppliedUpdater struct { fieldManager Manager }


annotations = map[string]string{} } annotations[corev1.LastAppliedConfigAnnotation] = value <del> if isAnnotationsValid(annotations) != nil { </del> <ins> if err := apimachineryvalidation.ValidateAnnotationsSize(annotations); err != nil { </ins> delete(annotations, corev1.LastAppliedConfigAnnotation) } accessor.SetAnnotations(annotations)


} return string(lastApplied), nil } <del> func isAnnotationsValid(annotations map[string]string) error { var totalSize int64 for k, v := range annotations { totalSize += (int64)(len(k)) + (int64)(len(v)) } if totalSize > (int64)(totalAnnotationSizeLimitB) { return fmt.Errorf("annotations size %d is larger than limit %d", totalSize, totalAnnotationSizeLimitB) } return nil } </del>


"encoding/json" "fmt" "math/rand" <ins> "strings" </ins> "time" "github.com/davecgh/go-spew/spew"


"k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" <ins> utilrand "k8s.io/apimachinery/pkg/util/rand" </ins> "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/dynamic" clientset "k8s.io/client-go/kubernetes" appsclient "k8s.io/client-go/kubernetes/typed/apps/v1" watchtools "k8s.io/client-go/tools/watch" <ins> "k8s.io/client-go/util/retry" </ins> appsinternal "k8s.io/kubernetes/pkg/apis/apps" deploymentutil "k8s.io/kubernetes/pkg/controller/deployment/util" "k8s.io/kubernetes/test/e2e/framework"


}) framework.ExpectNoError(err, "failed to see %v event", watch.Deleted) }) <ins> ginkgo.It("should validate Deployment Status endpoints", func() { dClient := c.AppsV1().Deployments(ns) dName := "test-deployment-" + utilrand.String(5) labelSelector := "e2e=testing" w := &cache.ListWatch{ WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { options.LabelSelector = labelSelector return dClient.Watch(context.TODO(), options) }, } dList, err := c.AppsV1().Deployments("").List(context.TODO(), metav1.ListOptions{LabelSelector: labelSelector}) framework.ExpectNoError(err, "failed to list Deployments") ginkgo.By("creating a Deployment") podLabels := map[string]string{"name": WebserverImageName, "e2e": "testing"} replicas := int32(1) framework.Logf("Creating simple deployment %s", dName) d := e2edeployment.NewDeployment(dName, replicas, podLabels, WebserverImageName, WebserverImage, appsv1.RollingUpdateDeploymentStrategyType) deploy, err := c.AppsV1().Deployments(ns).Create(context.TODO(), d, metav1.CreateOptions{}) framework.ExpectNoError(err) // Wait for it to be updated to revision 1 err = e2edeployment.WaitForDeploymentRevisionAndImage(c, ns, dName, "1", WebserverImage) framework.ExpectNoError(err) err = e2edeployment.WaitForDeploymentComplete(c, deploy) framework.ExpectNoError(err) testDeployment, err := dClient.Get(context.TODO(), dName, metav1.GetOptions{}) framework.ExpectNoError(err) ginkgo.By("Getting /status") dResource := schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} dStatusUnstructured, err := f.DynamicClient.Resource(dResource).Namespace(ns).Get(context.TODO(), dName, metav1.GetOptions{}, "status") framework.ExpectNoError(err, "Failed to fetch the status of deployment %s in namespace %s", dName, ns) dStatusBytes, err := json.Marshal(dStatusUnstructured) framework.ExpectNoError(err, "Failed to marshal unstructured response. %v", err) var dStatus appsv1.Deployment err = json.Unmarshal(dStatusBytes, &dStatus) framework.ExpectNoError(err, "Failed to unmarshal JSON bytes to a deployment object type") framework.Logf("Deployment %s has Conditions: %v", dName, dStatus.Status.Conditions) ginkgo.By("updating Deployment Status") var statusToUpdate, updatedStatus *appsv1.Deployment err = retry.RetryOnConflict(retry.DefaultRetry, func() error { statusToUpdate, err = dClient.Get(context.TODO(), dName, metav1.GetOptions{}) framework.ExpectNoError(err, "Unable to retrieve deployment %s", dName) statusToUpdate.Status.Conditions = append(statusToUpdate.Status.Conditions, appsv1.DeploymentCondition{ Type:    "StatusUpdate", Status:  "True", Reason:  "E2E", Message: "Set from e2e test", }) updatedStatus, err = dClient.UpdateStatus(context.TODO(), statusToUpdate, metav1.UpdateOptions{}) return err }) framework.ExpectNoError(err, "Failed to update status. %v", err) framework.Logf("updatedStatus.Conditions: %#v", updatedStatus.Status.Conditions) ginkgo.By("watching for the Deployment status to be updated") ctx, cancel := context.WithTimeout(context.Background(), dRetryTimeout) defer cancel() _, err = watchtools.Until(ctx, dList.ResourceVersion, w, func(event watch.Event) (bool, error) { if d, ok := event.Object.(*appsv1.Deployment); ok { found := d.ObjectMeta.Name == testDeployment.ObjectMeta.Name && d.ObjectMeta.Namespace == testDeployment.ObjectMeta.Namespace && d.Labels["e2e"] == "testing" if !found { framework.Logf("Observed Deployment %v in namespace %v with annotations: %v & Conditions: %vn", d.ObjectMeta.Name, d.ObjectMeta.Namespace, d.Annotations, d.Status.Conditions) return false, nil } for _, cond := range d.Status.Conditions { if cond.Type == "StatusUpdate" && cond.Reason == "E2E" && cond.Message == "Set from e2e test" { framework.Logf("Found Deployment %v in namespace %v with labels: %v annotations: %v & Conditions: %v", d.ObjectMeta.Name, d.ObjectMeta.Namespace, d.ObjectMeta.Labels, d.Annotations, cond) return found, nil } framework.Logf("Observed Deployment %v in namespace %v with annotations: %v & Conditions: %v", d.ObjectMeta.Name, d.ObjectMeta.Namespace, d.Annotations, cond) } } object := strings.Split(fmt.Sprintf("%v", event.Object), "{")[0] framework.Logf("Observed %v event: %+v", object, event.Type) return false, nil }) framework.ExpectNoError(err, "failed to locate Deployment %v in namespace %v", testDeployment.ObjectMeta.Name, ns) framework.Logf("Deployment %s has an updated status", dName) ginkgo.By("patching the Statefulset Status") payload := []byte(`{"status":{"conditions":[{"type":"StatusPatched","status":"True"}]}}`) framework.Logf("Patch payload: %v", string(payload)) patchedDeployment, err := dClient.Patch(context.TODO(), dName, types.MergePatchType, payload, metav1.PatchOptions{}, "status") framework.ExpectNoError(err, "Failed to patch status. %v", err) framework.Logf("Patched status conditions: %#v", patchedDeployment.Status.Conditions) ginkgo.By("watching for the Deployment status to be patched") ctx, cancel = context.WithTimeout(context.Background(), dRetryTimeout) defer cancel() _, err = watchtools.Until(ctx, dList.ResourceVersion, w, func(event watch.Event) (bool, error) { if e, ok := event.Object.(*appsv1.Deployment); ok { found := e.ObjectMeta.Name == testDeployment.ObjectMeta.Name && e.ObjectMeta.Namespace == testDeployment.ObjectMeta.Namespace && e.ObjectMeta.Labels["e2e"] == testDeployment.ObjectMeta.Labels["e2e"] if !found { framework.Logf("Observed deployment %v in namespace %v with annotations: %v & Conditions: %v", testDeployment.ObjectMeta.Name, testDeployment.ObjectMeta.Namespace, testDeployment.Annotations, testDeployment.Status.Conditions) return false, nil } for _, cond := range e.Status.Conditions { if cond.Type == "StatusPatched" { framework.Logf("Found deployment %v in namespace %v with labels: %v annotations: %v & Conditions: %v", testDeployment.ObjectMeta.Name, testDeployment.ObjectMeta.Namespace, testDeployment.ObjectMeta.Labels, testDeployment.Annotations, cond) return found, nil } framework.Logf("Observed deployment %v in namespace %v with annotations: %v & Conditions: %v", testDeployment.ObjectMeta.Name, testDeployment.ObjectMeta.Namespace, testDeployment.Annotations, cond) } } object := strings.Split(fmt.Sprintf("%v", event.Object), "{")[0] framework.Logf("Observed %v event: %+v", object, event.Type) return false, nil }) framework.ExpectNoError(err, "failed to locate deployment %v in namespace %v", testDeployment.ObjectMeta.Name, ns) framework.Logf("Deployment %s has a patched status", dName) }) </ins> }) func failureTrap(c clientset.Interface, ns string) {


MUST succeed. It MUST succeed when deleting the Deployment. release: v1.20 file: test/e2e/apps/deployment.go <ins> - testname: Deployment, status sub-resource codename: '[sig-apps] Deployment should validate Deployment Status endpoints [Conformance]' description: When a Deployment is created it MUST succeed. Attempt to read, update and patch its status sub-resource; all mutating sub-resource operations MUST be visible to subsequent reads. release: v1.22 file: test/e2e/apps/deployment.go </ins> - testname: 'PodDisruptionBudget: list and delete collection' codename: '[sig-apps] DisruptionController Listing PodDisruptionBudgets for all namespaces should list and delete a collection of PodDisruptionBudgets [Conformance]'


framework.ExpectNoError(err, "failed to see %v event", watch.Deleted) }) <del> ginkgo.It("should validate Deployment Status endpoints", func() { </del> <ins> /* Release: v1.22 Testname: Deployment, status sub-resource Description: When a Deployment is created it MUST succeed. Attempt to read, update and patch its status sub-resource; all mutating sub-resource operations MUST be visible to subsequent reads. */ framework.ConformanceIt("should validate Deployment Status endpoints", func() { </ins> dClient := c.AppsV1().Deployments(ns) dName := "test-deployment-" + utilrand.String(5) labelSelector := "e2e=testing"


clone, err := conversion.NewCloner().DeepCopy(&volume) if err != nil { glog.Errorf("error cloning volume %q: %v", volume.Name, err) <ins> continue </ins> } volumeClone := clone.(*api.PersistentVolume) ctrl.storeVolumeUpdate(volumeClone)


clone, err := conversion.NewCloner().DeepCopy(&claim) if err != nil { glog.Errorf("error cloning claim %q: %v", claimToClaimKey(&claim), err) <ins> continue </ins> } claimClone := clone.(*api.PersistentVolumeClaim) ctrl.storeClaimUpdate(claimClone)


return false } <del> accessModes := c.spec.PersistentVolume.Spec.AccessModes </del> <ins> if c.spec.PersistentVolume == nil { klog.V(4).Info(log("mounter.SetupAt Warning: skipping fsGroup permission change, no access mode available. The volume may only be accessible to root users.")) return false } </ins> if c.spec.PersistentVolume.Spec.AccessModes == nil { klog.V(4).Info(log("mounter.SetupAt WARNING: skipping fsGroup, access modes not provided")) return false } <del> if !hasReadWriteOnce(accessModes) { </del> <ins> if !hasReadWriteOnce(c.spec.PersistentVolume.Spec.AccessModes) { </ins> klog.V(4).Info(log("mounter.SetupAt WARNING: skipping fsGroup, only support ReadWriteOnce access mode")) return false }


authenticationv1 "k8s.io/api/authentication/v1" api "k8s.io/api/core/v1" <ins> corev1 "k8s.io/api/core/v1" </ins> storage "k8s.io/api/storage/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime"


}) } } <ins> func Test_csiMountMgr_supportsFSGroup(t *testing.T) { type fields struct { plugin              *csiPlugin driverName          csiDriverName volumeLifecycleMode storage.VolumeLifecycleMode fsGroupPolicy       storage.FSGroupPolicy volumeID            string specVolumeID        string readOnly            bool supportsSELinux     bool spec                *volume.Spec pod                 *api.Pod podUID              types.UID publishContext      map[string]string kubeVolHost         volume.KubeletVolumeHost MetricsProvider     volume.MetricsProvider } type args struct { fsType       string fsGroup      *int64 driverPolicy storage.FSGroupPolicy } tests := []struct { name   string fields fields args   args want   bool }{ { name: "empty all", args: args{}, want: false, }, { name: "driverPolicy is FileFSGroupPolicy", args: args{ fsGroup:      new(int64), driverPolicy: storage.FileFSGroupPolicy, }, want: true, }, { name: "driverPolicy is ReadWriteOnceWithFSTypeFSGroupPolicy", args: args{ fsGroup:      new(int64), driverPolicy: storage.ReadWriteOnceWithFSTypeFSGroupPolicy, }, want: false, }, { name: "driverPolicy is ReadWriteOnceWithFSTypeFSGroupPolicy with empty Spec", args: args{ fsGroup:      new(int64), fsType:       "ext4", driverPolicy: storage.ReadWriteOnceWithFSTypeFSGroupPolicy, }, fields: fields{ spec: &volume.Spec{}, }, want: false, }, { name: "driverPolicy is ReadWriteOnceWithFSTypeFSGroupPolicy with empty PersistentVolume", args: args{ fsGroup:      new(int64), fsType:       "ext4", driverPolicy: storage.ReadWriteOnceWithFSTypeFSGroupPolicy, }, fields: fields{ spec: volume.NewSpecFromPersistentVolume(&corev1.PersistentVolume{}, true), }, want: false, }, { name: "driverPolicy is ReadWriteOnceWithFSTypeFSGroupPolicy with empty AccessModes", args: args{ fsGroup:      new(int64), fsType:       "ext4", driverPolicy: storage.ReadWriteOnceWithFSTypeFSGroupPolicy, }, fields: fields{ spec: volume.NewSpecFromPersistentVolume(&api.PersistentVolume{ Spec: api.PersistentVolumeSpec{ AccessModes: []api.PersistentVolumeAccessMode{}, }, }, true), }, want: false, }, { name: "driverPolicy is ReadWriteOnceWithFSTypeFSGroupPolicy with ReadWriteOnce AccessModes", args: args{ fsGroup:      new(int64), fsType:       "ext4", driverPolicy: storage.ReadWriteOnceWithFSTypeFSGroupPolicy, }, fields: fields{ spec: volume.NewSpecFromPersistentVolume(&api.PersistentVolume{ Spec: api.PersistentVolumeSpec{ AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce}, }, }, true), }, want: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := &csiMountMgr{ plugin:              tt.fields.plugin, driverName:          tt.fields.driverName, volumeLifecycleMode: tt.fields.volumeLifecycleMode, fsGroupPolicy:       tt.fields.fsGroupPolicy, volumeID:            tt.fields.volumeID, specVolumeID:        tt.fields.specVolumeID, readOnly:            tt.fields.readOnly, supportsSELinux:     tt.fields.supportsSELinux, spec:                tt.fields.spec, pod:                 tt.fields.pod, podUID:              tt.fields.podUID, publishContext:      tt.fields.publishContext, kubeVolHost:         tt.fields.kubeVolHost, MetricsProvider:     tt.fields.MetricsProvider, } if got := c.supportsFSGroup(tt.args.fsType, tt.args.fsGroup, tt.args.driverPolicy); got != tt.want { t.Errorf("supportsFSGroup() = %v, want %v", got, tt.want) } }) } } </ins>


# $3 is the value to set the --pull flag for docker build; true by default # $4 is the set of --build-args for docker. function kube::build::docker_build() { <ins> kube::util::ensure-docker-buildx </ins> local -r image=$1 local -r context_dir=$2 local -r pull="${3:-true}"


"unicode" "unicode/utf8" <ins> "github.com/google/go-cmp/cmp" </ins> v1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/resource"


metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" unversionedvalidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" "k8s.io/apimachinery/pkg/labels" <del> "k8s.io/apimachinery/pkg/util/diff" </del> "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation"


// PersistentVolumeSource should be immutable after creation. if !apiequality.Semantic.DeepEqual(newPv.Spec.PersistentVolumeSource, oldPv.Spec.PersistentVolumeSource) { <del> pvcSourceDiff := diff.ObjectDiff(newPv.Spec.PersistentVolumeSource, oldPv.Spec.PersistentVolumeSource) </del> <ins> pvcSourceDiff := cmp.Diff(oldPv.Spec.PersistentVolumeSource, newPv.Spec.PersistentVolumeSource) </ins> allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "persistentvolumesource"), fmt.Sprintf("spec.persistentvolumesource is immutable after creationn%v", pvcSourceDiff))) } allErrs = append(allErrs, ValidateImmutableField(newPv.Spec.VolumeMode, oldPv.Spec.VolumeMode, field.NewPath("volumeMode"))...)


statusSize := oldPvc.Status.Capacity["storage"] if !apiequality.Semantic.DeepEqual(newPvcClone.Spec, oldPvcClone.Spec) { <del> specDiff := diff.ObjectDiff(newPvcClone.Spec, oldPvcClone.Spec) </del> <ins> specDiff := cmp.Diff(oldPvcClone.Spec, newPvcClone.Spec) </ins> allErrs = append(allErrs, field.Forbidden(field.NewPath("spec"), fmt.Sprintf("spec is immutable after creation except resources.requests for bound claimsn%v", specDiff))) } if newSize.Cmp(oldSize) < 0 {


// changes to Spec are not allowed, but updates to label/and some annotations are OK. // no-op updates pass validation. if !apiequality.Semantic.DeepEqual(newPvcClone.Spec, oldPvcClone.Spec) { <del> specDiff := diff.ObjectDiff(newPvcClone.Spec, oldPvcClone.Spec) </del> <ins> specDiff := cmp.Diff(oldPvcClone.Spec, newPvcClone.Spec) </ins> allErrs = append(allErrs, field.Forbidden(field.NewPath("spec"), fmt.Sprintf("field is immutable after creationn%v", specDiff))) } }


if !apiequality.Semantic.DeepEqual(mungedPodSpec, oldPod.Spec) { // This diff isn't perfect, but it's a helluva lot better an "I'm not going to tell you what the difference is". //TODO: Pinpoint the specific field that causes the invalid error after we have strategic merge diff <del> specDiff := diff.ObjectDiff(mungedPodSpec, oldPod.Spec) </del> <ins> specDiff := cmp.Diff(oldPod.Spec, mungedPodSpec) </ins> allErrs = append(allErrs, field.Forbidden(specPath, fmt.Sprintf("pod updates may not change fields other than `spec.containers[*].image`, `spec.initContainers[*].image`, `spec.activeDeadlineSeconds`, `spec.tolerations` (only additions to existing tolerations) or `spec.terminationGracePeriodSeconds` (allow it to be set to 1 if it was previously negative)n%v", specDiff))) }


if new, ok := newContainerIndex[old.Name]; !ok { allErrs = append(allErrs, field.Forbidden(specPath, fmt.Sprintf("existing ephemeral containers %q may not be removedn", old.Name))) } else if !apiequality.Semantic.DeepEqual(old, *new) { <del> specDiff := diff.ObjectDiff(old, *new) </del> <ins> specDiff := cmp.Diff(old, *new) </ins> allErrs = append(allErrs, field.Forbidden(specPath, fmt.Sprintf("existing ephemeral containers %q may not be changedn%v", old.Name, specDiff))) } }


import ( "context" "fmt" <ins> "os" "path/filepath" </ins> "strconv" "time"


}) ginkgo.It("after restart dbus, should be able to gracefully shutdown", func() { <ins> // allows manual restart of dbus to work in Ubuntu. err := overlayDbusConfig() framework.ExpectNoError(err) defer func() { err := restoreDbusConfig() framework.ExpectNoError(err) }() </ins> ginkgo.By("Restart Dbus") <del> err := restartDbus() </del> <ins> err = restartDbus() </ins> framework.ExpectNoError(err) ginkgo.By("Emitting Shutdown signal")


_, err := runCommand("sh", "-c", cmd) return err } <ins> func systemctlDaemonReload() error { cmd := "systemctl daemon-reload" _, err := runCommand("sh", "-c", cmd) return err } var ( dbusConfPath = "/etc/systemd/system/dbus.service.d/k8s-graceful-node-shutdown-e2e.conf" dbusConf     = ` [Unit] RefuseManualStart=no RefuseManualStop=no [Service] KillMode=control-group ExecStop= ` ) func overlayDbusConfig() error { err := os.MkdirAll(filepath.Dir(dbusConf), 0755) if err != nil { return err } err = os.WriteFile(dbusConfPath, []byte(dbusConf), 0644) if err != nil { return err } return systemctlDaemonReload() } func restoreDbusConfig() error { err := os.Remove(dbusConf) if err != nil { return err } return systemctlDaemonReload() } </ins>


less, err := isLess(iField, jField) if err != nil { <del> klog.Fatalf("Field %s in %T is an unsortable type: %s, err: %v", r.field, iObj, iField.Kind().String(), err) </del> <ins> klog.Exitf("Field %s in %T is an unsortable type: %s, err: %v", r.field, iObj, iField.Kind().String(), err) </ins> } return less }


less, err := isLess(iField, jField) if err != nil { <del> klog.Fatalf("Field %s in %T is an unsortable type: %s, err: %v", t.field, t.parsedRows, iField.Kind().String(), err) </del> <ins> klog.Exitf("Field %s in %T is an unsortable type: %s, err: %v", t.field, t.parsedRows, iField.Kind().String(), err) </ins> } return less }


return } <del> ch <- metrics.NewLazyConstMetric(containerStartTimeDesc, metrics.GaugeValue, float64(s.StartTime.UnixNano())/float64(time.Second), s.Name, pod.PodRef.Name, pod.PodRef.Namespace) </del> <ins> ch <- metrics.NewLazyMetricWithTimestamp(s.StartTime.Time, metrics.NewLazyConstMetric(containerStartTimeDesc, metrics.GaugeValue, float64(s.StartTime.UnixNano())/float64(time.Second), s.Name, pod.PodRef.Name, pod.PodRef.Namespace)) </ins> } func (rc *resourceMetricsCollector) collectContainerCPUMetrics(ch chan<- metrics.Metric, pod summary.PodStats, s summary.ContainerStats) {


container_memory_working_set_bytes{container="container_b",namespace="namespace_a",pod="pod_a"} 1000 1624396278302 # HELP container_start_time_seconds [ALPHA] Start time of the container since unix epoch in seconds # TYPE container_start_time_seconds gauge <del> container_start_time_seconds{container="container_a",namespace="namespace_a",pod="pod_a"} 1.6243962483020916e+09 container_start_time_seconds{container="container_a",namespace="namespace_b",pod="pod_b"} 1.6243956783020916e+09 container_start_time_seconds{container="container_b",namespace="namespace_a",pod="pod_a"} 1.6243961583020916e+09 </del> <ins> container_start_time_seconds{container="container_a",namespace="namespace_a",pod="pod_a"} 1.6243962483020916e+09 1624396248302 container_start_time_seconds{container="container_a",namespace="namespace_b",pod="pod_b"} 1.6243956783020916e+09 1624395678302 container_start_time_seconds{container="container_b",namespace="namespace_a",pod="pod_a"} 1.6243961583020916e+09 1624396158302 </ins> `, }, {


}), "container_start_time_seconds": gstruct.MatchElements(containerID, gstruct.IgnoreExtras, gstruct.Elements{ <del> fmt.Sprintf("%s::%s::%s", f.Namespace.Name, pod0, "busybox-container"): boundedSample(time.Now().Add(-maxStatsAge).UnixNano(), time.Now().Add(2*time.Minute).UnixNano()), fmt.Sprintf("%s::%s::%s", f.Namespace.Name, pod1, "busybox-container"): boundedSample(time.Now().Add(-maxStatsAge).UnixNano(), time.Now().Add(2*time.Minute).UnixNano()), </del> <ins> fmt.Sprintf("%s::%s::%s", f.Namespace.Name, pod0, "busybox-container"): boundedSample(time.Now().Add(-maxStatsAge).Unix(), time.Now().Add(2*time.Minute).Unix()), fmt.Sprintf("%s::%s::%s", f.Namespace.Name, pod1, "busybox-container"): boundedSample(time.Now().Add(-maxStatsAge).Unix(), time.Now().Add(2*time.Minute).Unix()), </ins> }), "pod_cpu_usage_seconds_total": gstruct.MatchElements(podID, gstruct.IgnoreExtras, gstruct.Elements{


Subsystem:         KubeletSubsystem, Name:              AssignedConfigKey, Help:              "The node's understanding of intended config. The count is always 1.", <del> DeprecatedVersion: "1.22", </del> <ins> DeprecatedVersion: "1.22.0", </ins> StabilityLevel:    metrics.ALPHA, }, []string{ConfigSourceLabelKey, ConfigUIDLabelKey, ConfigResourceVersionLabelKey, KubeletConfigKeyLabelKey},


Subsystem:         KubeletSubsystem, Name:              ActiveConfigKey, Help:              "The config source the node is actively using. The count is always 1.", <del> DeprecatedVersion: "1.22", </del> <ins> DeprecatedVersion: "1.22.0", </ins> StabilityLevel:    metrics.ALPHA, }, []string{ConfigSourceLabelKey, ConfigUIDLabelKey, ConfigResourceVersionLabelKey, KubeletConfigKeyLabelKey},


Subsystem:         KubeletSubsystem, Name:              LastKnownGoodConfigKey, Help:              "The config source the node will fall back to when it encounters certain errors. The count is always 1.", <del> DeprecatedVersion: "1.22", </del> <ins> DeprecatedVersion: "1.22.0", </ins> StabilityLevel:    metrics.ALPHA, }, []string{ConfigSourceLabelKey, ConfigUIDLabelKey, ConfigResourceVersionLabelKey, KubeletConfigKeyLabelKey},


Subsystem:         KubeletSubsystem, Name:              ConfigErrorKey, Help:              "This metric is true (1) if the node is experiencing a configuration-related error, false (0) otherwise.", <del> DeprecatedVersion: "1.22", </del> <ins> DeprecatedVersion: "1.22.0", </ins> StabilityLevel:    metrics.ALPHA, }, )


"custom": resource.MustParse("0"), }, Limits: v1.ResourceList{ <del> "memory": resource.MustParse("2G"), </del> <ins> "memory": resource.MustParse("2.5Gi"), </ins> "custom": resource.MustParse("6"), }, }},


expected := `# HELP kube_pod_resource_limit [ALPHA] Resources limit for workloads on the cluster, broken down by pod. This shows the resource usage the scheduler and kubelet expect per pod for resources along with the unit for the resource if any. # TYPE kube_pod_resource_limit gauge kube_pod_resource_limit{namespace="test",node="node-one",pod="foo",priority="",resource="custom",scheduler="",unit=""} 6 <del> kube_pod_resource_limit{namespace="test",node="node-one",pod="foo",priority="",resource="memory",scheduler="",unit="bytes"} 2e+09 </del> <ins> kube_pod_resource_limit{namespace="test",node="node-one",pod="foo",priority="",resource="memory",scheduler="",unit="bytes"} 2.68435456e+09 </ins> # HELP kube_pod_resource_request [ALPHA] Resources requested by workloads on the cluster, broken down by pod. This shows the resource usage the scheduler and kubelet expect per pod for resources along with the unit for the resource if any. # TYPE kube_pod_resource_request gauge kube_pod_resource_request{namespace="test",node="node-one",pod="foo",priority="",resource="cpu",scheduler="",unit="cores"} 2


return base } <del> // multiply by the appropriate exponential scale switch q.Format { case DecimalExponent, DecimalSI: return base * math.Pow10(exponent) default: // fast path for exponents that can fit in 64 bits if exponent > 0 && exponent < 7 { return base * float64(int64(1)<<(exponent*10)) } return base * math.Pow(2, float64(exponent*10)) } </del> <ins> return base * math.Pow10(exponent) </ins> } // AsInt64 returns a representation of the current value as an int64 if a fast conversion


{decQuantity(1024, 0, BinarySI), 1024}, {decQuantity(8*1024, 0, BinarySI), 8 * 1024}, {decQuantity(7*1024*1024, 0, BinarySI), 7 * 1024 * 1024}, <del> {decQuantity(7*1024*1024, 1, BinarySI), (7 * 1024 * 1024) * 1024}, {decQuantity(7*1024*1024, 4, BinarySI), (7 * 1024 * 1024) * (1024 * 1024 * 1024 * 1024)}, {decQuantity(7*1024*1024, 8, BinarySI), (7 * 1024 * 1024) * (1024 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024 * 1024)}, {decQuantity(7*1024*1024, -1, BinarySI), (7 * 1024 * 1024) / float64(1024)}, {decQuantity(7*1024*1024, -8, BinarySI), (7 * 1024 * 1024) / float64(1024*1024*1024*1024*1024*1024*1024*1024)}, </del> <ins> {decQuantity(7*1024*1024, 1, BinarySI), (7 * 1024 * 1024) * 10}, {decQuantity(7*1024*1024, 4, BinarySI), (7 * 1024 * 1024) * 10000}, {decQuantity(7*1024*1024, 8, BinarySI), (7 * 1024 * 1024) * 100000000}, {decQuantity(7*1024*1024, -1, BinarySI), (7 * 1024 * 1024) * math.Pow10(-1)}, // '* Pow10' and '/ float(10)' do not round the same way {decQuantity(7*1024*1024, -8, BinarySI), (7 * 1024 * 1024) / float64(100000000)}, </ins> {decQuantity(1024, 0, DecimalSI), 1024}, {decQuantity(8*1024, 0, DecimalSI), 8 * 1024},


} } <ins> func TestStringQuantityAsApproximateFloat64(t *testing.T) { table := []struct { in  string out float64 }{ {"2Ki", 2048}, {"1.1Ki", 1126.4e+0}, {"1Mi", 1.048576e+06}, {"2Gi", 2.147483648e+09}, } for _, item := range table { t.Run(item.in, func(t *testing.T) { in, err := ParseQuantity(item.in) if err != nil { t.Fatal(err) } out := in.AsApproximateFloat64() if out != item.out { t.Fatalf("expected %v, got %v", item.out, out) } if in.d.Dec != nil { if i, ok := in.AsInt64(); ok { q := intQuantity(i, 0, in.Format) out := q.AsApproximateFloat64() if out != item.out { t.Fatalf("as int quantity: expected %v, got %v", item.out, out) } } } }) } } </ins> func benchmarkQuantities() []Quantity { return []Quantity{ intQuantity(1024*1024*1024, 0, BinarySI),


fsckErrorsCorrected = 1 // 'fsck' found errors but exited without correcting them fsckErrorsUncorrected = 4 <ins> // Error thrown by exec cmd.Run() when process spawned by cmd.Start() completes before cmd.Wait() is called (see - k/k issue #103753) errNoChildProcesses = "wait: no child processes" </ins> ) // Mounter provides the default implementation of mount.Interface


command := exec.Command(mountCmd, mountArgs...) output, err := command.CombinedOutput() if err != nil { <ins> if err.Error() == errNoChildProcesses { if command.ProcessState.Success() { // We don't consider errNoChildProcesses an error if the process itself succeeded (see - k/k issue #103753). return nil } // Rewrite err with the actual exit error of the process. err = &exec.ExitError{ProcessState: command.ProcessState} } </ins> klog.Errorf("Mount failed: %vnMounting command: %snMounting arguments: %snOutput: %sn", err, mountCmd, mountArgsLogStr, string(output)) return fmt.Errorf("mount failed: %vnMounting command: %snMounting arguments: %snOutput: %s", err, mountCmd, mountArgsLogStr, string(output))