tpNeedsUpdate = true } if supportsNodesHealthCheck { <del> hcToCreate = makeHttpHealthCheck(makeNodesHealthCheckName(clusterID), GetNodesHealthCheckPath(), GetNodesHealthCheckPort()) </del> <ins> hcToCreate = makeHttpHealthCheck(MakeNodesHealthCheckName(clusterID), GetNodesHealthCheckPath(), GetNodesHealthCheckPort()) </ins> } } // Now we get to some slightly more interesting logic.


// using local traffic health check or nodes health check. Attempt to delete // both to prevent leaking. hcNames = append(hcNames, loadBalancerName) <del> hcNames = append(hcNames, makeNodesHealthCheckName(clusterID)) </del> <ins> hcNames = append(hcNames, MakeNodesHealthCheckName(clusterID)) </ins> } errs := utilerrors.AggregateGoroutines( func() error { <del> fwName := makeFirewallName(loadBalancerName) </del> <ins> fwName := MakeFirewallName(loadBalancerName) </ins> err := ignoreNotFound(gce.DeleteFirewall(fwName)) if isForbidden(err) && gce.OnXPN() { glog.V(4).Infof("ensureExternalLoadBalancerDeleted(%v): do not have permission to delete firewall rule (on XPN). Raising event.", loadBalancerName)


} func (gce *GCECloud) firewallNeedsUpdate(name, serviceName, region, ipAddress string, ports []v1.ServicePort, sourceRanges netsets.IPNet) (exists bool, needsUpdate bool, err error) { <del> fw, err := gce.service.Firewalls.Get(gce.NetworkProjectID(), makeFirewallName(name)).Do() </del> <ins> fw, err := gce.service.Firewalls.Get(gce.NetworkProjectID(), MakeFirewallName(name)).Do() </ins> if err != nil { if isHTTPErrorCode(err, http.StatusNotFound) { return false, true, nil


return fmt.Sprintf(`{"kubernetes.io/service-name":"%s"}`, serviceName) } <del> // makeNodesHealthCheckName returns name of the health check resource used by </del> <ins> // MakeNodesHealthCheckName returns name of the health check resource used by </ins> // the GCE load balancers (l4) for performing health checks on nodes. <del> func makeNodesHealthCheckName(clusterID string) string { </del> <ins> func MakeNodesHealthCheckName(clusterID string) string { </ins> return fmt.Sprintf("k8s-%v-node", clusterID) }


// balancers (l4) for performing health checks. func MakeHealthCheckFirewallName(clusterID, hcName string, isNodesHealthCheck bool) string { if isNodesHealthCheck { <del> return makeNodesHealthCheckName(clusterID) + "-http-hc" </del> <ins> return MakeNodesHealthCheckName(clusterID) + "-http-hc" </ins> } return "k8s-" + hcName + "-http-hc" } <del> func makeFirewallName(name string) string { </del> <ins> // MakeFirewallName returns the firewall name used by the GCE load // balancers (l4) for serving traffic. func MakeFirewallName(name string) string { </ins> return fmt.Sprintf("k8s-fw-%s", name) }


if err != nil { return fmt.Errorf("error parsing GCE/GKE region from zone %q: %v", zone, err) } <del> if err := gceCloud.DeleteFirewall(loadBalancerName); err != nil && </del> <ins> if err := gceCloud.DeleteFirewall(gcecloud.MakeFirewallName(loadBalancerName)); err != nil && </ins> !IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) { retErr = err }


!IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) { retErr = fmt.Errorf("%vn%v", retErr, err) } <del> var hcNames []string </del> <ins> clusterID, err := GetClusterID(c) if err != nil { retErr = fmt.Errorf("%vn%v", retErr, err) return } hcNames := []string{gcecloud.MakeNodesHealthCheckName(clusterID)} </ins> hc, getErr := gceCloud.GetHttpHealthCheck(loadBalancerName) if getErr != nil && !IsGoogleAPIHTTPErrorCode(getErr, http.StatusNotFound) { retErr = fmt.Errorf("%vn%v", retErr, getErr)


if hc != nil { hcNames = append(hcNames, hc.Name) } <del> clusterID, err := GetClusterID(c) if err != nil { retErr = fmt.Errorf("%vn%v", retErr, err) return } </del> if err := gceCloud.DeleteExternalTargetPoolAndChecks(nil, loadBalancerName, region, clusterID, hcNames...); err != nil && !IsGoogleAPIHTTPErrorCode(err, http.StatusNotFound) { retErr = fmt.Errorf("%vn%v", retErr, err)


// check the status of containers. for idx, container := range pod.Spec.Containers { containerStatus := podStatus.FindContainerStatusByName(container.Name) <ins> // Call internal container post-stop lifecycle hook for any non-running container so that any // allocated cpus are released immediately. If the container is restarted, cpus will be re-allocated // to it. if containerStatus != nil && containerStatus.State != kubecontainer.ContainerStateRunning { if err := m.internalLifecycle.PostStopContainer(containerStatus.ID.ID); err != nil { glog.Errorf("internal container post-stop lifecycle hook failed for container %v in pod %v with error %v", container.Name, pod.Name, err) } } </ins> // If container does not exist, or is not running, check whether we // need to restart it. if containerStatus == nil || containerStatus.State != kubecontainer.ContainerStateRunning {


kind: ConfigMap apiVersion: v1 metadata: <del> name: fluentd-es-config-v0.1.5 </del> <ins> name: fluentd-es-config-v0.1.6 </ins> namespace: kube-system labels: addonmanager.kubernetes.io/mode: Reconcile


@type kubernetes_metadata </filter> <ins> # Concatenate multi-line logs <filter **> @type concat key message multiline_end_regexp /n$/ separator "" </filter> </ins> <match **> @id elasticsearch @type elasticsearch


apiVersion: apps/v1 kind: DaemonSet metadata: <del> name: fluentd-es-v2.2.0 </del> <ins> name: fluentd-es-v2.2.1 </ins> namespace: kube-system labels: k8s-app: fluentd-es <del> version: v2.2.0 </del> <ins> version: v2.2.1 </ins> kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile spec: selector: matchLabels: k8s-app: fluentd-es <del> version: v2.2.0 </del> <ins> version: v2.2.1 </ins> template: metadata: labels: k8s-app: fluentd-es kubernetes.io/cluster-service: "true" <del> version: v2.2.0 </del> <ins> version: v2.2.1 </ins> # This annotation ensures that fluentd does not get evicted if the node # supports critical pod annotation based priority scheme. # Note that this does not guarantee admission on the nodes (#40573).


path: /var/lib/docker/containers - name: config-volume configMap: <del> name: fluentd-es-config-v0.1.5 </del> <ins> name: fluentd-es-config-v0.1.6 </ins>


gem 'fluentd', '<=1.2.4' gem 'activesupport', '~>5.2.1' <del> gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.0.0' gem 'fluent-plugin-elasticsearch', '~>2.11.5' gem 'fluent-plugin-systemd', '~>1.0.1' </del> <ins> gem 'fluent-plugin-concat', '~>2.3.0' </ins> gem 'fluent-plugin-detect-exceptions', '~>0.0.11' <del> gem 'fluent-plugin-prometheus', '~>1.0.1' </del> <ins> gem 'fluent-plugin-elasticsearch', '~>2.11.5' gem 'fluent-plugin-kubernetes_metadata_filter', '~>2.0.0' </ins> gem 'fluent-plugin-multi-format-parser', '~>1.0.0' <ins> gem 'fluent-plugin-prometheus', '~>1.0.1' gem 'fluent-plugin-systemd', '~>1.0.1' </ins> gem 'oj', '~>3.6.5'


local network=$(make-gcloud-network-argument  "${NETWORK_PROJECT}"  "${REGION}"  <del> "${NETWORK}" </del> <ins> "${NETWORK}"  </ins> "${SUBNETWORK:-}"  ""  "${ENABLE_IP_ALIASES:-}"


glog.V(2).Infof("Deleted obsolete listener: %s", listener.ID) } <del> status := &v1.LoadBalancerStatus{} status.Ingress = []v1.LoadBalancerIngress{{IP: loadbalancer.VipAddress}} </del> portID := loadbalancer.VipPortID floatIP, err := getFloatingIPByPortID(lbaas.network, portID) if err != nil && err != ErrNotFound {


return nil, fmt.Errorf("Error creating LB floatingip %+v: %v", floatIPOpts, err) } } <ins> status := &v1.LoadBalancerStatus{} </ins> if floatIP != nil { <del> status.Ingress = append(status.Ingress, v1.LoadBalancerIngress{IP: floatIP.FloatingIP}) </del> <ins> status.Ingress = []v1.LoadBalancerIngress{{IP: floatIP.FloatingIP}} } else { status.Ingress = []v1.LoadBalancerIngress{{IP: loadbalancer.VipAddress}} </ins> } if lbaas.opts.ManageSecurityGroups {


} status := &v1.LoadBalancerStatus{} <del> status.Ingress = []v1.LoadBalancerIngress{{IP: vip.Address}} </del> if floatingPool != "" && !internalAnnotation { floatIPOpts := floatingips.CreateOpts{ FloatingNetworkID: floatingPool,


return nil, fmt.Errorf("Error creating floatingip for openstack load balancer %s: %v", name, err) } <del> status.Ingress = append(status.Ingress, v1.LoadBalancerIngress{IP: floatIP.FloatingIP}) </del> <ins> status.Ingress = []v1.LoadBalancerIngress{{IP: floatIP.FloatingIP}} } else { status.Ingress = []v1.LoadBalancerIngress{{IP: vip.Address}} </ins> } return status, nil


return "", err } <del> if len(volumeSource.TargetWWNs) != 0 { </del> <ins> // API server validates these parameters beforehand but attach/detach // controller creates volumespec without validation. They may be nil // or zero length. We should check again to avoid unexpected conditions. if len(volumeSource.TargetWWNs) != 0 && volumeSource.Lun != nil { </ins> // TargetWWNs are the FibreChannel target worldwide names <del> return fmt.Sprintf("%v", volumeSource.TargetWWNs), nil </del> <ins> return fmt.Sprintf("%v:%v", volumeSource.TargetWWNs, *volumeSource.Lun), nil </ins> } else if len(volumeSource.WWIDs) != 0 { // WWIDs are the FibreChannel World Wide Identifiers return fmt.Sprintf("%v", volumeSource.WWIDs), nil


// attempted before returning. // // In case there is a pod with the same namespace+name already running, this <del> // function assumes it's an older instance of the recycler pod and watches // this old pod instead of starting a new one. </del> <ins> // function deletes it as it is not able to judge if it is an old recycler // or user has forged a fake recycler to block Kubernetes from recycling.// </ins> // //  pod - the pod designed by a volume plugin to recycle the volume. pod.Name //        will be overwritten with unique name based on PV.Name.


_, err = recyclerClient.CreatePod(pod) if err != nil { if errors.IsAlreadyExists(err) { <del> glog.V(5).Infof("old recycler pod %q found for volume", pod.Name) </del> <ins> deleteErr := recyclerClient.DeletePod(pod.Name, pod.Namespace) if deleteErr != nil { return fmt.Errorf("failed to delete old recycler pod %s/%s: %s", pod.Namespace, pod.Name, deleteErr) } // Recycler will try again and the old pod will be hopefuly deleted // at that time. return fmt.Errorf("old recycler pod found, will retry later") </ins> } else { return fmt.Errorf("unexpected error creating recycler pod:  %+vn", err) } } <del> defer func(pod *v1.Pod) { glog.V(2).Infof("deleting recycler pod %s/%s", pod.Namespace, pod.Name) if err := recyclerClient.DeletePod(pod.Name, pod.Namespace); err != nil { glog.Errorf("failed to delete recycler pod %s/%s: %v", pod.Namespace, pod.Name, err) } }(pod) </del> <ins> err = waitForPod(pod, recyclerClient, podCh) // In all cases delete the recycler pod and log its result. glog.V(2).Infof("deleting recycler pod %s/%s", pod.Namespace, pod.Name) deleteErr := recyclerClient.DeletePod(pod.Name, pod.Namespace) if deleteErr != nil { glog.Errorf("failed to delete recycler pod %s/%s: %v", pod.Namespace, pod.Name, err) } // Returning recycler error is preferred, the pod will be deleted again on // the next retry. if err != nil { return fmt.Errorf("failed to recycle volume: %s", err) } // Recycle succeeded but we failed to delete the recycler pod. Report it, // the controller will re-try recycling the PV again shortly. if deleteErr != nil { return fmt.Errorf("failed to delete recycler pod: %s", deleteErr) } return nil } </ins> <del> // Now only the old pod or the new pod run. Watch it until it finishes // and send all events on the pod to the PV </del> <ins> // waitForPod watches the pod it until it finishes and send all events on the // pod to the PV. func waitForPod(pod *v1.Pod, recyclerClient recyclerClient, podCh <-chan watch.Event) error { </ins> for { event, ok := <-podCh if !ok {


{v1.EventTypeWarning, "Unable to mount volumes for pod "recycler-for-podRecyclerFailure_default(3c9809e5-347c-11e6-a79b-3c970e965218)": timeout expired waiting for volumes to attach/mount"}, {v1.EventTypeWarning, "Error syncing pod, skipping: timeout expired waiting for volumes to attach/mount for pod "default"/"recycler-for-podRecyclerFailure". list of unattached/unmounted"}, }, <del> expectedError: "Pod was active on the node longer than specified deadline", </del> <ins> expectedError: "failed to recycle volume: Pod was active on the node longer than specified deadline", </ins> }, { // Recycler pod gets deleted


expectedEvents: []mockEvent{ {v1.EventTypeNormal, "Successfully assigned recycler-for-podRecyclerDeleted to 127.0.0.1"}, }, <del> expectedError: "recycler pod was deleted", </del> <ins> expectedError: "failed to recycle volume: recycler pod was deleted", </ins> }, { // Another recycler pod is already running <del> name:        "RecyclerRunning", existingPod: newPod("podOldRecycler", v1.PodRunning, ""), createPod:   newPod("podNewRecycler", v1.PodFailed, "mock message"), eventSequence: []watch.Event{ // Old pod succeeds newPodEvent(watch.Modified, "podOldRecycler", v1.PodSucceeded, ""), }, // No error = old pod succeeded. If the new pod was used, there // would be error with "mock message". expectedError: "", }, { // Another recycler pod is already running and fails name:        "FailedRecyclerRunning", existingPod: newPod("podOldRecycler", v1.PodRunning, ""), createPod:   newPod("podNewRecycler", v1.PodFailed, "mock message"), eventSequence: []watch.Event{ // Old pod failure newPodEvent(watch.Modified, "podOldRecycler", v1.PodFailed, "Pod was active on the node longer than specified deadline"), }, // If the new pod was used, there would be error with "mock message". expectedError: "Pod was active on the node longer than specified deadline", </del> <ins> name:          "RecyclerRunning", existingPod:   newPod("podOldRecycler", v1.PodRunning, ""), createPod:     newPod("podNewRecycler", v1.PodFailed, "mock message"), eventSequence: []watch.Event{}, expectedError: "old recycler pod found, will retry later", </ins> }, }


pkg_tar( name = "kubernetes-cni-data", <del> package_dir = "/opt/cni", </del> <ins> package_dir = "/opt/cni/bin", </ins> deps = ["@kubernetes_cni//file"], )


func ValidatePersistentVolumeUpdate(newPv, oldPv *api.PersistentVolume) field.ErrorList { allErrs := field.ErrorList{} allErrs = ValidatePersistentVolume(newPv) <ins> // PersistentVolumeSource should be immutable after creation. if !apiequality.Semantic.DeepEqual(newPv.Spec.PersistentVolumeSource, oldPv.Spec.PersistentVolumeSource) { allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "persistentvolumesource"), "is immutable after creation")) } </ins> newPv.Status = oldPv.Status return allErrs }


} <ins> func TestValidatePersistentVolumeSourceUpdate(t *testing.T) { validVolume := testVolume("foo", "", api.PersistentVolumeSpec{ Capacity: api.ResourceList{ api.ResourceName(api.ResourceStorage): resource.MustParse("1G"), }, AccessModes: []api.PersistentVolumeAccessMode{api.ReadWriteOnce}, PersistentVolumeSource: api.PersistentVolumeSource{ HostPath: &api.HostPathVolumeSource{ Path: "/foo", Type: newHostPathType(string(api.HostPathDirectory)), }, }, StorageClassName: "valid", }) validPvSourceNoUpdate := validVolume.DeepCopy() invalidPvSourceUpdateType := validVolume.DeepCopy() invalidPvSourceUpdateType.Spec.PersistentVolumeSource = api.PersistentVolumeSource{ FlexVolume: &api.FlexVolumeSource{ Driver: "kubernetes.io/blue", FSType: "ext4", }, } invalidPvSourceUpdateDeep := validVolume.DeepCopy() invalidPvSourceUpdateDeep.Spec.PersistentVolumeSource = api.PersistentVolumeSource{ HostPath: &api.HostPathVolumeSource{ Path: "/updated", Type: newHostPathType(string(api.HostPathDirectory)), }, } scenarios := map[string]struct { isExpectedFailure bool oldVolume         *api.PersistentVolume newVolume         *api.PersistentVolume }{ "condition-no-update": { isExpectedFailure: false, oldVolume:         validVolume, newVolume:         validPvSourceNoUpdate, }, "condition-update-source-type": { isExpectedFailure: true, oldVolume:         validVolume, newVolume:         invalidPvSourceUpdateType, }, "condition-update-source-deep": { isExpectedFailure: true, oldVolume:         validVolume, newVolume:         invalidPvSourceUpdateDeep, }, } for name, scenario := range scenarios { errs := ValidatePersistentVolumeUpdate(scenario.newVolume, scenario.oldVolume) if len(errs) == 0 && scenario.isExpectedFailure { t.Errorf("Unexpected success for scenario: %s", name) } if len(errs) > 0 && !scenario.isExpectedFailure { t.Errorf("Unexpected failure for scenario: %s - %+v", name, errs) } } } </ins> func TestValidateLocalVolumes(t *testing.T) { scenarios := map[string]struct { isExpectedFailure bool


- /heapster - --source=kubernetes.summary_api:'' - --sink=gcm <del> - image: gcr.io/google_containers/heapster-amd64:v1.5.0-beta.0 </del> <ins> - image: gcr.io/google_containers/heapster-amd64:v1.5.0-beta.1 </ins> name: eventer command: - /eventer


- --memory={{base_eventer_memory}} - --extra-memory={{eventer_memory_per_node}}Ki - --threshold=5 <del> - --deployment=heapster-v1.5.0-beta.0 </del> <ins> - --deployment=heapster-v1.5.0-beta.1 </ins> - --container=eventer - --poll-period=300000 - --estimator=exponential


- --source=kubernetes.summary_api:'' - --sink=influxdb:http://monitoring-influxdb:8086 - --sink=gcm:?metrics=autoscaling <del> - image: gcr.io/google_containers/heapster-amd64:v1.5.0-beta.0 </del> <ins> - image: gcr.io/google_containers/heapster-amd64:v1.5.0-beta.1 </ins> name: eventer command: - /eventer


- /heapster - --source=kubernetes.summary_api:'' - --sink=influxdb:http://monitoring-influxdb:8086 <del> - image: gcr.io/google_containers/heapster-amd64:v1.5.0-beta.0 </del> <ins> - image: gcr.io/google_containers/heapster-amd64:v1.5.0-beta.1 </ins> name: eventer command: - /eventer


- --memory={{ base_eventer_memory }} - --extra-memory={{ eventer_memory_per_node }}Ki - --threshold=5 <del> - --deployment=heapster-v1.5.0-beta.0 </del> <ins> - --deployment=heapster-v1.5.0-beta.1 </ins> - --container=eventer - --poll-period=300000 - --estimator=exponential


- --memory={{ base_metrics_memory }} - --extra-memory={{metrics_memory_per_node}}Mi - --threshold=5 <del> - --deployment=heapster-v1.5.0-beta.0 </del> <ins> - --deployment=heapster-v1.5.0-beta.1 </ins> - --container=heapster - --poll-period=300000 - --estimator=exponential


apiVersion: extensions/v1beta1 kind: Deployment metadata: <del> name: heapster-v1.5.0-beta.0 </del> <ins> name: heapster-v1.5.0-beta.1 </ins> namespace: kube-system labels: k8s-app: heapster kubernetes.io/cluster-service: "true" addonmanager.kubernetes.io/mode: Reconcile <del> version: v1.5.0-beta.0 </del> <ins> version: v1.5.0-beta.1 </ins> spec: replicas: 1 selector: matchLabels: k8s-app: heapster <del> version: v1.5.0-beta.0 </del> <ins> version: v1.5.0-beta.1 </ins> template: metadata: labels: k8s-app: heapster <del> version: v1.5.0-beta.0 </del> <ins> version: v1.5.0-beta.1 </ins> annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: containers: <del> - image: gcr.io/google_containers/heapster-amd64:v1.5.0-beta.0 </del> <ins> - image: gcr.io/google_containers/heapster-amd64:v1.5.0-beta.1 </ins> name: heapster livenessProbe: httpGet:


- --memory={{ base_metrics_memory }} - --extra-memory={{ metrics_memory_per_node }}Mi - --threshold=5 <del> - --deployment=heapster-v1.5.0-beta.0 </del> <ins> - --deployment=heapster-v1.5.0-beta.1 </ins> - --container=heapster - --poll-period=300000 - --estimator=exponential


func diskConsumingPod(name string, diskConsumedMB int, volumeSource *v1.VolumeSource, resources v1.ResourceRequirements) *v1.Pod { // Each iteration writes 1 Mb, so do diskConsumedMB iterations. <del> return podWithCommand(volumeSource, resources, name, fmt.Sprintf("i=0; while [ $i -lt %d ];", diskConsumedMB)+" do dd if=/dev/urandom of=%s${i} bs=1048576 count=1; i=$(($i+1)); done; while true; do sleep 5; done") </del> <ins> return podWithCommand(volumeSource, resources, name, fmt.Sprintf("i=0; while [ $i -lt %d ];", diskConsumedMB)+" do dd if=/dev/urandom of=%s${i} bs=1048576 count=1 2>/dev/null ; i=$(($i+1)); done; while true; do sleep 5; done") </ins> } // podWithCommand returns a pod with the provided volumeSource and resourceRequirements.


EmptyDir: &v1.EmptyDirVolumeSource{SizeLimit: &sizeLimit}, }, v1.ResourceRequirements{}), }, <ins> { evictionPriority: 0, // This pod should not be evicted because it uses less than its limit pod:              diskConsumingPod("emptydir-disk-below-sizelimit", useUnderLimit, nil, v1.ResourceRequirements{Limits: containerLimit}), }, </ins> }) }) })


}, { evictionPriority: 0, <del> pod: diskConsumingPod("guaranteed-disk", 299 /* Mb */, nil, v1.ResourceRequirements{ </del> <ins> // Only require 99% accuracy (297/300 Mb) because on some OS distributions, the file itself (excluding contents), consumes disk space. pod: diskConsumingPod("guaranteed-disk", 297 /* Mb */, nil, v1.ResourceRequirements{ </ins> Requests: v1.ResourceList{ v1.ResourceEphemeralStorage: resource.MustParse("300Mi"), },


"persistent_volumes-vsphere.go", "pv_reclaimpolicy.go", "pvc_label_selector.go", <ins> "volume_expand.go", </ins> "volume_io.go", "volume_metrics.go", "volume_provisioning.go",


<ins> /* Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package storage import ( "fmt" "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "k8s.io/api/core/v1" storage "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" ) const ( resizePollInterval = 2 * time.Second // total time to wait for cloudprovider or file system resize to finish totalResizeWaitPeriod = 20 * time.Minute ) var _ = SIGDescribe("Volume expand [Feature:ExpandPersistentVolumes] [Slow]", func() { var ( c           clientset.Interface ns          string err         error pvc         *v1.PersistentVolumeClaim resizableSc *storage.StorageClass ) f := framework.NewDefaultFramework("volume-expand") BeforeEach(func() { framework.SkipUnlessProviderIs("aws", "gce") c = f.ClientSet ns = f.Namespace.Name framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout)) test := storageClassTest{ name:      "default", claimSize: "2Gi", } resizableSc, err = createResizableStorageClass(test, ns, "resizing", c) Expect(err).NotTo(HaveOccurred(), "Error creating resizable storage class") Expect(*resizableSc.AllowVolumeExpansion).To(BeTrue()) pvc = newClaim(test, ns, "default") pvc.Spec.StorageClassName = &resizableSc.Name pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(pvc) Expect(err).NotTo(HaveOccurred(), "Error creating pvc") }) AfterEach(func() { framework.ExpectNoError(framework.DeletePersistentVolumeClaim(c, pvc.Name, pvc.Namespace)) framework.ExpectNoError(c.StorageV1().StorageClasses().Delete(resizableSc.Name, nil)) }) It("Verify if editing PVC allows resize", func() { By("Waiting for pvc to be in bound phase") pvcClaims := []*v1.PersistentVolumeClaim{pvc} pvs, err := framework.WaitForPVClaimBoundPhase(c, pvcClaims, framework.ClaimProvisionTimeout) Expect(err).NotTo(HaveOccurred(), "Failed waiting for PVC to be bound %v", err) Expect(len(pvs)).To(Equal(1)) By("Creating a pod with dynamically provisioned volume") pod, err := framework.CreatePod(c, ns, nil, pvcClaims, false, "") Expect(err).NotTo(HaveOccurred(), "While creating pods for resizing") defer func() { err = framework.DeletePodWithWait(f, c, pod) Expect(err).NotTo(HaveOccurred(), "while cleaning up pod already deleted in resize test") }() By("Expanding current pvc") newSize := resource.MustParse("6Gi") pvc, err = expandPVCSize(pvc, newSize, c) Expect(err).NotTo(HaveOccurred(), "While updating pvc for more size") Expect(pvc).NotTo(BeNil()) pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage] if pvcSize.Cmp(newSize) != 0 { framework.Failf("error updating pvc size %q", pvc.Name) } By("Waiting for cloudprovider resize to finish") err = waitForControllerVolumeResize(pvc, c) Expect(err).NotTo(HaveOccurred(), "While waiting for pvc resize to finish") By("Checking for conditions on pvc") pvc, err = c.CoreV1().PersistentVolumeClaims(ns).Get(pvc.Name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred(), "While fetching pvc after controller resize") inProgressConditions := pvc.Status.Conditions Expect(len(inProgressConditions)).To(Equal(1), "pvc must have resize condition") Expect(inProgressConditions[0].Type).To(Equal(v1.PersistentVolumeClaimResizing), "pvc must have resizing condition") By("Deleting the previously created pod") err = framework.DeletePodWithWait(f, c, pod) Expect(err).NotTo(HaveOccurred(), "while deleting pod for resizing") By("Creating a new pod with same volume") pod2, err := framework.CreatePod(c, ns, nil, pvcClaims, false, "") Expect(err).NotTo(HaveOccurred(), "while recreating pod for resizing") defer func() { err = framework.DeletePodWithWait(f, c, pod2) Expect(err).NotTo(HaveOccurred(), "while cleaning up pod before exiting resizing test") }() By("Waiting for file system resize to finish") pvc, err = waitForFSResize(pvc, c) Expect(err).NotTo(HaveOccurred(), "while waiting for fs resize to finish") pvcConditions := pvc.Status.Conditions Expect(len(pvcConditions)).To(Equal(0), "pvc should not have conditions") }) }) func createResizableStorageClass(t storageClassTest, ns string, suffix string, c clientset.Interface) (*storage.StorageClass, error) { stKlass := newStorageClass(t, ns, suffix) allowExpansion := true stKlass.AllowVolumeExpansion = &allowExpansion var err error stKlass, err = c.StorageV1().StorageClasses().Create(stKlass) return stKlass, err } func expandPVCSize(origPVC *v1.PersistentVolumeClaim, size resource.Quantity, c clientset.Interface) (*v1.PersistentVolumeClaim, error) { pvcName := origPVC.Name updatedPVC := origPVC.DeepCopy() waitErr := wait.PollImmediate(resizePollInterval, 30*time.Second, func() (bool, error) { var err error updatedPVC, err = c.CoreV1().PersistentVolumeClaims(origPVC.Namespace).Get(pvcName, metav1.GetOptions{}) if err != nil { return false, fmt.Errorf("error fetching pvc %q for resizing with %v", pvcName, err) } updatedPVC.Spec.Resources.Requests[v1.ResourceStorage] = size updatedPVC, err = c.CoreV1().PersistentVolumeClaims(origPVC.Namespace).Update(updatedPVC) if err == nil { return true, nil } framework.Logf("Error updating pvc %s with %v", pvcName, err) return false, nil }) return updatedPVC, waitErr } func waitForControllerVolumeResize(pvc *v1.PersistentVolumeClaim, c clientset.Interface) error { pvName := pvc.Spec.VolumeName return wait.PollImmediate(resizePollInterval, totalResizeWaitPeriod, func() (bool, error) { pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage] pv, err := c.CoreV1().PersistentVolumes().Get(pvName, metav1.GetOptions{}) if err != nil { return false, fmt.Errorf("error fetching pv %q for resizing %v", pvName, err) } pvSize := pv.Spec.Capacity[v1.ResourceStorage] // If pv size is greater or equal to requested size that means controller resize is finished. if pvSize.Cmp(pvcSize) >= 0 { return true, nil } return false, nil }) } func waitForFSResize(pvc *v1.PersistentVolumeClaim, c clientset.Interface) (*v1.PersistentVolumeClaim, error) { var updatedPVC *v1.PersistentVolumeClaim waitErr := wait.PollImmediate(resizePollInterval, totalResizeWaitPeriod, func() (bool, error) { var err error updatedPVC, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(pvc.Name, metav1.GetOptions{}) if err != nil { return false, fmt.Errorf("error fetching pvc %q for checking for resize status : %v", pvc.Name, err) } pvcSize := updatedPVC.Spec.Resources.Requests[v1.ResourceStorage] pvcStatusSize := updatedPVC.Status.Capacity[v1.ResourceStorage] //If pvc's status field size is greater than or equal to pvc's size then done if pvcStatusSize.Cmp(pvcSize) >= 0 { return true, nil } return false, nil }) return updatedPVC, waitErr } </ins>


// InstanceType returns the type of the specified instance. func (m *metadata) InstanceType(name types.NodeName) (string, error) { instanceType, err := m.get(metadataTypeInstanceType) <del> if err == nil { </del> <ins> if err != nil { </ins> return "", fmt.Errorf("could not get instance type: %v", err) }


Skip("Nvidia GPUs do not exist on the node. Skipping test.") } <ins> framework.WaitForAllNodesSchedulable(f.ClientSet, framework.TestContext.NodeSchedulableTimeout) </ins> By("Creating the Google Device Plugin pod for NVIDIA GPU in GKE") devicePluginPod = f.PodClient().CreateSync(framework.NVIDIADevicePlugin(f.Namespace.Name))


f.PodClient().Delete(devicePluginPod.Name, &metav1.DeleteOptions{}) By("Waiting for GPUs to become unavailable on the local node") Eventually(func() bool { <del> return framework.NumberOfNVIDIAGPUs(getLocalNode(f)) <= 0 </del> <ins> node, err := f.ClientSet.CoreV1().Nodes().Get(framework.TestContext.NodeName, metav1.GetOptions{}) framework.ExpectNoError(err) return framework.NumberOfNVIDIAGPUs(node) <= 0 </ins> }, 10*time.Minute, framework.Poll).Should(BeTrue()) By("Checking that scheduled pods can continue to run even after we delete device plugin.") count1, devIdRestart1 = getDeviceId(f, p1.Name, p1.Name, count1+1)


glog.V(6).Infof("Resizing is not enabled for this volume %s", volumeToMount.VolumeName) return nil } mounter := og.volumePluginMgr.Host.GetMounter(pluginName) // Get expander, if possible expandableVolumePlugin, _ :=


// File system resize was requested, proceed glog.V(4).Infof(volumeToMount.GenerateMsgDetailed("MountVolume.resizeFileSystem entering", fmt.Sprintf("DevicePath %q", volumeToMount.DevicePath))) <ins> if volumeToMount.VolumeSpec.ReadOnly { simpleMsg, detailedMsg := volumeToMount.GenerateMsg("MountVolume.resizeFileSystem failed", "requested read-only file system") glog.Warningf(detailedMsg) og.recorder.Eventf(volumeToMount.Pod, v1.EventTypeWarning, kevents.FileSystemResizeFailed, simpleMsg) return nil } </ins> diskFormatter := &mount.SafeFormatAndMount{ Interface: mounter, Exec:      og.volumePluginMgr.Host.GetExec(expandableVolumePlugin.GetPluginName()),


} pvcCopy.Status.Capacity = capacity <ins> pvcCopy.Status.Conditions = []v1.PersistentVolumeClaimCondition{} </ins> newData, err := json.Marshal(pvcCopy) if err != nil {


Accelerators utilfeature.Feature = "Accelerators" // owner: @jiayingz <del> // alpha: v1.8 </del> <ins> // beta: v1.10 </ins> // // Enables support for Device Plugins <del> // Only Nvidia GPUs are tested as of v1.8. </del> DevicePlugins utilfeature.Feature = "DevicePlugins" // owner: @gmarek


ExperimentalHostUserNamespaceDefaultingGate: {Default: false, PreRelease: utilfeature.Beta}, ExperimentalCriticalPodAnnotation:           {Default: false, PreRelease: utilfeature.Alpha}, Accelerators:                                {Default: false, PreRelease: utilfeature.Alpha}, <del> DevicePlugins:                               {Default: false, PreRelease: utilfeature.Alpha}, </del> <ins> DevicePlugins:                               {Default: true, PreRelease: utilfeature.Beta}, </ins> TaintBasedEvictions:                         {Default: false, PreRelease: utilfeature.Alpha}, RotateKubeletServerCertificate:              {Default: false, PreRelease: utilfeature.Alpha}, RotateKubeletClientCertificate:              {Default: true, PreRelease: utilfeature.Beta},


}, Spec: storage.VolumeAttachmentSpec{ NodeName: node, <del> Attacher: csiPluginName, </del> <ins> Attacher: csiSource.Driver, </ins> Source: storage.VolumeAttachmentSource{ PersistentVolumeName: &pvName, },


}, Spec: storage.VolumeAttachmentSpec{ NodeName: nodeName, <del> Attacher: csiPluginName, </del> <ins> Attacher: "mock", </ins> Source: storage.VolumeAttachmentSource{ PersistentVolumeName: &pvName, },


// This is a temporary workaround to get stats for cri-o from cadvisor // and should be removed. // Related to https://github.com/kubernetes/kubernetes/issues/51798 <del> if i.runtimeEndpoint == "/var/run/crio.sock" { </del> <ins> if i.runtimeEndpoint == CrioSocket { </ins> return cadvisorfs.LabelCrioImages, nil } }


kubetypes "k8s.io/kubernetes/pkg/kubelet/types" ) <ins> const ( // Please keep this in sync with the one in: // github.com/google/cadvisor/container/crio/client.go CrioSocket = "/var/run/crio/crio.sock" ) </ins> func CapacityFromMachineInfo(info *cadvisorapi.MachineInfo) v1.ResourceList { c := v1.ResourceList{ v1.ResourceCPU: *resource.NewMilliQuantity(


func UsingLegacyCadvisorStats(runtime, runtimeEndpoint string) bool { return runtime == kubetypes.RktContainerRuntime || (runtime == kubetypes.DockerContainerRuntime && goruntime.GOOS == "linux") || <del> runtimeEndpoint == "/var/run/crio.sock" </del> <ins> runtimeEndpoint == CrioSocket </ins> }


} if !s.LeaderElection.LeaderElect { <del> run(nil) </del> <ins> stopCh := make(chan struct{}) defer close(stopCh) run(stopCh) </ins> panic("unreachable") }


var _ Manager = &manager{} // NewManager creates new cpu manager based on provided policy <del> func NewManager( cpuPolicyName string, reconcilePeriod time.Duration, machineInfo *cadvisorapi.MachineInfo, nodeAllocatableReservation v1.ResourceList, stateFileDirecory string, ) (Manager, error) { </del> <ins> func NewManager(cpuPolicyName string, reconcilePeriod time.Duration, machineInfo *cadvisorapi.MachineInfo, nodeAllocatableReservation v1.ResourceList, stateFileDirecory string) (Manager, error) { </ins> var policy Policy switch policyName(cpuPolicyName) {


glog.Infof("[cpumanager] detected CPU topology: %v", topo) reservedCPUs, ok := nodeAllocatableReservation[v1.ResourceCPU] if !ok { <del> // The static policy cannot initialize without this information. Panic! panic("[cpumanager] unable to determine reserved CPU resources for static policy") </del> <ins> // The static policy cannot initialize without this information. return nil, fmt.Errorf("[cpumanager] unable to determine reserved CPU resources for static policy") </ins> } if reservedCPUs.IsZero() { <del> // Panic! // </del> // The static policy requires this to be nonzero. Zero CPU reservation // would allow the shared pool to be completely exhausted. At that point // either we would violate our guarantee of exclusivity or need to evict // any pod that has at least one container that requires zero CPUs. // See the comments in policy_static.go for more details. <del> panic("[cpumanager] the static policy requires systemreserved.cpu + kubereserved.cpu to be greater than zero") </del> <ins> return nil, fmt.Errorf("[cpumanager] the static policy requires systemreserved.cpu + kubereserved.cpu to be greater than zero") </ins> } // Take the ceiling of the reservation, since fractional CPUs cannot be


cpuPolicyName              string nodeAllocatableReservation v1.ResourceList isTopologyBroken           bool <del> panicMsg                   string </del> expectedPolicy             string expectedError              error skipIfPermissionsError     bool


description:                "static policy - broken reservation", cpuPolicyName:              "static", nodeAllocatableReservation: v1.ResourceList{}, <del> panicMsg:                   "unable to determine reserved CPU resources for static policy", </del> <ins> expectedError:              fmt.Errorf("unable to determine reserved CPU resources for static policy"), </ins> skipIfPermissionsError:     true, }, { description:                "static policy - no CPU resources", cpuPolicyName:              "static", nodeAllocatableReservation: v1.ResourceList{v1.ResourceCPU: *resource.NewQuantity(0, resource.DecimalSI)}, <del> panicMsg:                   "the static policy requires systemreserved.cpu + kubereserved.cpu to be greater than zero", </del> <ins> expectedError:              fmt.Errorf("the static policy requires systemreserved.cpu + kubereserved.cpu to be greater than zero"), </ins> skipIfPermissionsError:     true, }, }


t.Errorf("cannot create state file: %s", err.Error()) } defer os.RemoveAll(sDir) <del> defer func() { if err := recover(); err != nil { if testCase.panicMsg != "" { if !strings.Contains(err.(string), testCase.panicMsg) { t.Errorf("Unexpected panic message. Have: %q wants %q", err, testCase.panicMsg) } } else { t.Errorf("Unexpected panic: %q", err) } } else if testCase.panicMsg != "" { t.Error("Expected panic hasn't been raised") } }() </del> mgr, err := NewManager(testCase.cpuPolicyName, 5*time.Second, machineInfo, testCase.nodeAllocatableReservation, sDir) if testCase.expectedError != nil {


EnableLogsHandler:      true, EventTTL:               1 * time.Hour, MasterCount:            1, <del> EndpointReconcilerType: string(reconcilers.MasterCountReconcilerType), </del> <ins> EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType), </ins> KubeletConfig: kubeletclient.KubeletClientConfig{ Port:         ports.KubeletPort, ReadOnlyPort: ports.KubeletReadOnlyPort,


"Currently only applies to long-running requests.") fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount, <del> "The number of apiservers running in the cluster, must be a positive number.") </del> <ins> "The number of apiservers running in the cluster, must be a positive number. (In use when --endpoint-reconciler-type=master-count is enabled.)") </ins> fs.StringVar(&s.EndpointReconcilerType, "endpoint-reconciler-type", string(s.EndpointReconcilerType), "Use an endpoint reconciler ("+strings.Join(reconcilers.AllTypes.Names(), ", ")+")")


"--enable-aggregator-routing=true", "--enable-logs-handler=false", "--enable-swagger-ui=true", <del> "--endpoint-reconciler-type=" + string(reconcilers.MasterCountReconcilerType), </del> <ins> "--endpoint-reconciler-type=" + string(reconcilers.LeaseEndpointReconcilerType), </ins> "--etcd-quorum-read=false", "--etcd-keyfile=/var/run/kubernetes/etcd.key", "--etcd-certfile=/var/run/kubernetes/etcdce.crt",


ServiceNodePortRange:   kubeoptions.DefaultServiceNodePortRange, ServiceClusterIPRange:  kubeoptions.DefaultServiceIPCIDR, MasterCount:            5, <del> EndpointReconcilerType: string(reconcilers.MasterCountReconcilerType), </del> <ins> EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType), </ins> AllowPrivileged:        false, GenericServerRunOptions: &apiserveroptions.ServerRunOptions{ AdvertiseAddress:            net.ParseIP("192.168.10.10"),


local message=$1 local match=$2 <del> if echo "$message" | grep -q "$match"; then </del> <ins> if grep -q "${match}" <<< "${message}"; then </ins> echo "Successful" echo "message:$message" echo "has:$match"


local message=$1 local match=$2 <del> if echo "$message" | grep -q "$match"; then </del> <ins> if grep -q "${match}" <<< "${message}"; then </ins> echo "FAIL!" echo "message:$message" echo "has:$match"


print('File %s is missing the year' % filename, file=verbose_out) return False <del> # Replace all occurrences of the regex "2017|2016|2015|2014" with "YEAR" </del> <ins> # Replace all occurrences of the regex "2014|2015|2016|2017|2018" with "YEAR" </ins> p = regexs["date"] for i, d in enumerate(data): (data[i], found) = p.subn('YEAR', d)


regexs = {} # Search for "YEAR" which exists in the boilerplate, but shouldn't in the real thing regexs["year"] = re.compile( 'YEAR' ) <del> # dates can be 2014, 2015, 2016, or 2017; company holder names can be anything regexs["date"] = re.compile( '(2014|2015|2016|2017)' ) </del> <ins> # dates can be 2014, 2015, 2016, 2017, or 2018; company holder names can be anything regexs["date"] = re.compile( '(2014|2015|2016|2017|2018)' ) </ins> # strip // +build nn build constraints regexs["go_build_constraints"] = re.compile(r"^(// +build.*n)+n", re.MULTILINE) # strip #!.* from shell scripts


BASH_TARGETS=" update-generated-protobuf update-codegen <ins> update-generated-runtime update-generated-device-plugin </ins> update-generated-docs update-generated-swagger-docs update-swagger-spec


// +build !ignore_autogenerated /* <del> Copyright 2017 The Kubernetes Authors. </del> <ins> Copyright 2018 The Kubernetes Authors. </ins> Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.


/* <del> Copyright 2017 The Kubernetes Authors. </del> <ins> Copyright 2018 The Kubernetes Authors. </ins> Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.


} service, err := e.serviceLister.Services(namespace).Get(name) if err != nil { <del> // Delete the corresponding endpoint, as the service has been deleted. // TODO: Please note that this will delete an endpoint when a // service is deleted. However, if we're down at the time when // the service is deleted, we will miss that deletion, so this // doesn't completely solve the problem. See #6877. err = e.client.CoreV1().Endpoints(namespace).Delete(name, nil) if err != nil && !errors.IsNotFound(err) { return err } </del> <ins> // Service has been deleted. So no need to do any more operations. </ins> return nil }


// See http://docs.openstack.org/user-guide/cli_config_drive.html type Metadata struct { Uuid             string           `json:"uuid"` <del> Name             string           `json:"name"` </del> <ins> Hostname         string           `json:"hostname"` </ins> AvailabilityZone string           `json:"availability_zone"` Devices          []DeviceMetadata `json:"devices,omitempty"` // .. and other fields we don't care about.  Expand as necessary.


var FakeMetadata = Metadata{ Uuid:             "83679162-1378-4288-a2d4-70e13ec132aa", <del> Name:             "test", </del> <ins> Hostname:         "test", </ins> AvailabilityZone: "nova", }


t.Fatalf("Should succeed when provided with valid data: %s", err) } <del> if md.Name != "test" { t.Errorf("incorrect name: %s", md.Name) </del> <ins> if md.Hostname != "test.novalocal" { t.Errorf("incorrect hostname: %s", md.Hostname) </ins> } if md.Uuid != "83679162-1378-4288-a2d4-70e13ec132aa" {


if err != nil { return "", err } <del> return types.NodeName(md.Name), nil </del> <ins> return types.NodeName(md.Hostname), nil </ins> } func (i *Instances) AddSSHKeyToAllInstances(user string, keyData []byte) error {


"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" <ins> utilflag "k8s.io/apiserver/pkg/util/flag" </ins> clientset "k8s.io/client-go/kubernetes" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"


) flagSet.StringVar(featureGatesString, "feature-gates", *featureGatesString, "A set of key=value pairs that describe feature gates for various features. "+ "Options are:n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "n")) <ins> flagSet.Var(utilflag.NewMapStringString(&cfg.APIServerExtraArgs), "apiserver-extra-args", "A set of extra flags to pass to the API Server or override default ones in form of <flagname>=<value>") flagSet.Var(utilflag.NewMapStringString(&cfg.ControllerManagerExtraArgs), "controller-manager-extra-args", "A set of extra flags to pass to the Controller Manager or override default ones in form of <flagname>=<value>") flagSet.Var(utilflag.NewMapStringString(&cfg.SchedulerExtraArgs), "scheduler-extra-args", "A set of extra flags to pass to the Scheduler or override default ones in form of <flagname>=<value>") </ins> } // AddInitOtherFlags adds init flags that are not bound to a configuration file to the given flagset


"//vendor/github.com/spf13/pflag:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", <ins> "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library", </ins> "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/client-go/tools/bootstrap/token/api:go_default_library", "//vendor/k8s.io/client-go/tools/bootstrap/token/util:go_default_library",


"github.com/spf13/cobra" <ins> utilflag "k8s.io/apiserver/pkg/util/flag" </ins> kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1" "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"


cmd.Flags().StringVar(&cfg.Networking.ServiceSubnet, "service-cidr", cfg.Networking.ServiceSubnet, "The range of IP address used for service VIPs") cmd.Flags().StringVar(&featureGatesString, "feature-gates", featureGatesString, "A set of key=value pairs that describe feature gates for various features. "+ "Options are:n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "n")) <ins> cmd.Flags().Var(utilflag.NewMapStringString(&cfg.APIServerExtraArgs), "apiserver-extra-args", "A set of extra flags to pass to the API Server or override default ones in form of <flagname>=<value>") </ins> } if properties.use == "all" || properties.use == "controller-manager" { cmd.Flags().StringVar(&cfg.Networking.PodSubnet, "pod-network-cidr", cfg.Networking.PodSubnet, "The range of IP addresses used for the Pod network") <ins> cmd.Flags().Var(utilflag.NewMapStringString(&cfg.ControllerManagerExtraArgs), "controller-manager-extra-args", "A set of extra flags to pass to the Controller Manager or override default ones in form of <flagname>=<value>") } if properties.use == "all" || properties.use == "scheduler" { cmd.Flags().Var(utilflag.NewMapStringString(&cfg.SchedulerExtraArgs), "scheduler-extra-args", "A set of extra flags to pass to the Scheduler or override default ones in form of <flagname>=<value>") </ins> } cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to kubeadm config file (WARNING: Usage of a configuration file is experimental)")


} func (plugin *fcPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) { <del> fcVolume := &v1.Volume{ Name: volumeName, VolumeSource: v1.VolumeSource{ FC: &v1.FCVolumeSource{}, }, </del> <ins> // Find globalPDPath from pod volume directory(mountPath) // examples: //   mountPath:     pods/{podUid}/volumes/kubernetes.io~fc/{volumeName} //   globalPDPath : plugins/kubernetes.io/fc/50060e801049cfd1-lun-0 var globalPDPath string mounter := plugin.host.GetMounter(plugin.GetPluginName()) paths, err := mount.GetMountRefs(mounter, mountPath) if err != nil { return nil, err } for _, path := range paths { if strings.Contains(path, plugin.host.GetPluginDir(fcPluginName)) { globalPDPath = path break } } // Couldn't fetch globalPDPath if len(globalPDPath) == 0 { return nil, fmt.Errorf("couldn't fetch globalPDPath. failed to obtain volume spec") } arr := strings.Split(globalPDPath, "/") if len(arr) < 1 { return nil, fmt.Errorf("failed to retrieve volume plugin information from globalPDPath: %v", globalPDPath) } volumeInfo := arr[len(arr)-1] // Create volume from wwn+lun or wwid var fcVolume *v1.Volume if strings.Contains(volumeInfo, "-lun-") { wwnLun := strings.Split(volumeInfo, "-lun-") if len(wwnLun) < 2 { return nil, fmt.Errorf("failed to retrieve TargetWWN and Lun. volumeInfo is invalid: %v", volumeInfo) } lun, err := strconv.Atoi(wwnLun[1]) if err != nil { return nil, err } lun32 := int32(lun) fcVolume = &v1.Volume{ Name: volumeName, VolumeSource: v1.VolumeSource{ FC: &v1.FCVolumeSource{TargetWWNs: []string{wwnLun[0]}, Lun: &lun32}, }, } glog.V(5).Infof("ConstructVolumeSpec: TargetWWNs: %v, Lun: %v", fcVolume.VolumeSource.FC.TargetWWNs, *fcVolume.VolumeSource.FC.Lun) } else { fcVolume = &v1.Volume{ Name: volumeName, VolumeSource: v1.VolumeSource{ FC: &v1.FCVolumeSource{WWIDs: []string{volumeInfo}}, }, } glog.V(5).Infof("ConstructVolumeSpec: WWIDs: %v", fcVolume.VolumeSource.FC.WWIDs) </ins> } return volume.NewSpecFromVolume(fcVolume), nil }


//   - If a file is found, then retreives volumePluginDependentPath from globalMapPathUUID. //   - Once volumePluginDependentPath is obtained, store volume information to VolumeSource // examples: <del> //   mapPath: pods/{podUid}}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName} </del> <ins> //   mapPath: pods/{podUid}/{DefaultKubeletVolumeDevicesDirName}/{escapeQualifiedPluginName}/{volumeName} </ins> //   globalMapPathUUID : plugins/kubernetes.io/{PluginName}/{DefaultKubeletVolumeDevicesDirName}/{volumePluginDependentPath}/{pod uuid} func (plugin *fcPlugin) ConstructBlockVolumeSpec(podUID types.UID, volumeName, mapPath string) (*volume.Spec, error) { pluginDir := plugin.host.GetVolumeDevicePluginDir(fcPluginName)


v1.FCVolumeSource{TargetWWNs: []string{wwnLun[0]}, Lun: &lun32}) glog.V(5).Infof("ConstructBlockVolumeSpec: TargetWWNs: %v, Lun: %v", fcPV.Spec.PersistentVolumeSource.FC.TargetWWNs, <del> fcPV.Spec.PersistentVolumeSource.FC.Lun) </del> <ins> *fcPV.Spec.PersistentVolumeSource.FC.Lun) </ins> } else { fcPV = createPersistentVolumeFromFCVolumeSource(volumeName, v1.FCVolumeSource{WWIDs: []string{volumeInfo}})


import ( "fmt" "os" <ins> "strconv" "strings" </ins> "testing" "k8s.io/api/core/v1"


t.Errorf("unexpected fc disk found") } } <ins> func Test_ConstructVolumeSpec(t *testing.T) { fm := &mount.FakeMounter{ MountPoints: []mount.MountPoint{ {Device: "/dev/sdb", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~fc/fc-in-pod1"}, {Device: "/dev/sdb", Path: "/var/lib/kubelet/plugins/kubernetes.io/fc/50060e801049cfd1-lun-0"}, {Device: "/dev/sdc", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~fc/fc-in-pod2"}, {Device: "/dev/sdc", Path: "/var/lib/kubelet/plugins/kubernetes.io/fc/volumeDevices/3600508b400105e210000900000490000"}, }, } mountPaths := []string{ "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~fc/fc-in-pod1", "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~fc/fc-in-pod2", } for _, path := range mountPaths { refs, _ := mount.GetMountRefs(fm, path) var globalPDPath string for _, ref := range refs { if strings.Contains(ref, "kubernetes.io/fc") { globalPDPath = ref break } } if len(globalPDPath) == 0 { t.Errorf("couldn't fetch mountrefs") } arr := strings.Split(globalPDPath, "/") if len(arr) < 1 { t.Errorf("failed to retrieve volume plugin information from globalPDPath: %v", globalPDPath) } volumeInfo := arr[len(arr)-1] if strings.Contains(volumeInfo, "-lun-") { wwnLun := strings.Split(volumeInfo, "-lun-") if len(wwnLun) < 2 { t.Errorf("failed to retrieve TargetWWN and Lun. volumeInfo is invalid: %v", volumeInfo) } lun, _ := strconv.Atoi(wwnLun[1]) lun32 := int32(lun) if wwnLun[0] != "50060e801049cfd1" || lun32 != 0 { t.Errorf("failed to retrieve TargetWWN and Lun") } } else { if volumeInfo != "3600508b400105e210000900000490000" { t.Errorf("failed to retrieve WWIDs") } } } } func Test_ConstructVolumeSpecNoRefs(t *testing.T) { fm := &mount.FakeMounter{ MountPoints: []mount.MountPoint{ {Device: "/dev/sdd", Path: "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~fc/fc-in-pod1"}, }, } mountPaths := []string{ "/var/lib/kubelet/pods/some-pod/volumes/kubernetes.io~fc/fc-in-pod1", } for _, path := range mountPaths { refs, _ := mount.GetMountRefs(fm, path) var globalPDPath string for _, ref := range refs { if strings.Contains(ref, "kubernetes.io/fc") { globalPDPath = ref break } } if len(globalPDPath) != 0 { t.Errorf("invalid globalPDPath") } } } </ins>


GetStats(path string, stats *libcontainercgroups.Stats) error } <del> // getSupportedSubsystems returns list of subsystems supported func getSupportedSubsystems() []subsystem { supportedSubsystems := []subsystem{ &cgroupfs.MemoryGroup{}, &cgroupfs.CpuGroup{}, </del> <ins> // getSupportedSubsystems returns a map of subsystem and if it must be mounted for the kubelet to function. func getSupportedSubsystems() map[subsystem]bool { supportedSubsystems := map[subsystem]bool{ &cgroupfs.MemoryGroup{}: true, &cgroupfs.CpuGroup{}:    true, </ins> } <ins> // not all hosts support hugetlb cgroup, and in the absent of hugetlb, we will fail silently by reporting no capacity. </ins> if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.HugePages) { <del> supportedSubsystems = append(supportedSubsystems, &cgroupfs.HugetlbGroup{}) </del> <ins> supportedSubsystems[&cgroupfs.HugetlbGroup{}] = false </ins> } if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.SupportPodPidsLimit) { <del> supportedSubsystems = append(supportedSubsystems, &cgroupfs.PidsGroup{}) </del> <ins> supportedSubsystems[&cgroupfs.PidsGroup{}] = true </ins> } return supportedSubsystems }


// but this is not possible with libcontainers Set() method // See https://github.com/opencontainers/runc/issues/932 func setSupportedSubsystems(cgroupConfig *libcontainerconfigs.Cgroup) error { <del> for _, sys := range getSupportedSubsystems() { </del> <ins> for sys, required := range getSupportedSubsystems() { </ins> if _, ok := cgroupConfig.Paths[sys.Name()]; !ok { <del> return fmt.Errorf("Failed to find subsystem mount for subsystem: %v", sys.Name()) </del> <ins> if required { return fmt.Errorf("Failed to find subsystem mount for required subsystem: %v", sys.Name()) } // the cgroup is not mounted, but its not required so continue... glog.V(6).Infof("Unable to find subsystem mount for optional subsystem: %v", sys.Name()) continue </ins> } if err := sys.Set(cgroupConfig.Paths[sys.Name()], cgroupConfig); err != nil { return fmt.Errorf("Failed to set config for supported subsystems : %v", err)


func getStatsSupportedSubsystems(cgroupPaths map[string]string) (*libcontainercgroups.Stats, error) { stats := libcontainercgroups.NewStats() <del> for _, sys := range getSupportedSubsystems() { </del> <ins> for sys, required := range getSupportedSubsystems() { </ins> if _, ok := cgroupPaths[sys.Name()]; !ok { <del> return nil, fmt.Errorf("Failed to find subsystem mount for subsystem: %v", sys.Name()) </del> <ins> if required { return nil, fmt.Errorf("Failed to find subsystem mount for required subsystem: %v", sys.Name()) } // the cgroup is not mounted, but its not required so continue... glog.V(6).Infof("Unable to find subsystem mount for optional subsystem: %v", sys.Name()) continue </ins> } if err := sys.GetStats(cgroupPaths[sys.Name()], stats); err != nil { return nil, fmt.Errorf("Failed to get stats for supported subsystems : %v", err)


if !(runtime.IsMissingVersion(err) || runtime.IsMissingKind(err) || runtime.IsNotRegisteredError(err)) { return nil, err } <ins> // Only tolerate load errors if the file appears to be one of the two legacy plugin configs unstructuredData := map[string]interface{}{} if err2 := yaml.Unmarshal(data, &unstructuredData); err2 != nil { return nil, err } _, isLegacyImagePolicy := unstructuredData["imagePolicy"] _, isLegacyPodNodeSelector := unstructuredData["podNodeSelectorPluginConfig"] if !isLegacyImagePolicy && !isLegacyPodNodeSelector { return nil, err } </ins> // convert the legacy format to the new admission control format // in order to preserve backwards compatibility, we set plugins that // previously read input from a non-versioned file configuration to the


"k8s.io/client-go/rest" ) <del> var scheme = runtime.NewScheme() </del> <ins> var configScheme = runtime.NewScheme() </ins> func init() { <del> apiserverapi.AddToScheme(scheme) apiserverapiv1alpha1.AddToScheme(scheme) </del> <ins> apiserverapi.AddToScheme(configScheme) apiserverapiv1alpha1.AddToScheme(configScheme) </ins> } // AdmissionOptions holds the admission options


pluginNames = a.enabledPluginNames() } <del> pluginsConfigProvider, err := admission.ReadAdmissionConfiguration(pluginNames, a.ConfigFile, scheme) </del> <ins> pluginsConfigProvider, err := admission.ReadAdmissionConfiguration(pluginNames, a.ConfigFile, configScheme) </ins> if err != nil { return fmt.Errorf("failed to read plugin config: %v", err) }


metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" <ins> "k8s.io/apimachinery/pkg/runtime/serializer/json" </ins> utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait"


// ConfigFile is the location of the scheduler server's configuration file. ConfigFile string <ins> // WriteConfigTo is the path where the default configuration will be written. WriteConfigTo string </ins> // config is the scheduler server's configuration object. config *componentconfig.KubeSchedulerConfiguration


// AddFlags adds flags for a specific SchedulerServer to the specified FlagSet func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.StringVar(&o.ConfigFile, "config", o.ConfigFile, "The path to the configuration file.") <ins> fs.StringVar(&o.WriteConfigTo, "write-config-to", o.WriteConfigTo, "If set, write the configuration values to this file and exit.") </ins> // All flags below here are deprecated and will eventually be removed.


} func (o *Options) Complete() error { <del> if len(o.ConfigFile) == 0 { glog.Warning("WARNING: all flags other than --config are deprecated. Please begin using a config file ASAP.") </del> <ins> if len(o.ConfigFile) == 0 && len(o.WriteConfigTo) == 0 { glog.Warning("WARNING: all flags other than --config, --write-config-to, and --cleanup are deprecated. Please begin using a config file ASAP.") </ins> o.applyDeprecatedHealthzAddressToConfig() o.applyDeprecatedHealthzPortToConfig() o.applyDeprecatedAlgorithmSourceOptionsToConfig() } <ins> if len(o.ConfigFile) > 0 { if c, err := o.loadConfigFromFile(o.ConfigFile); err != nil { return err } else { o.config = c } } // Apply algorithms based on feature gates. // TODO: make configurable? algorithmprovider.ApplyFeatureGates() </ins> return nil }


} func (o *Options) Run() error { <del> config := o.config if len(o.ConfigFile) > 0 { if c, err := o.loadConfigFromFile(o.ConfigFile); err != nil { return err } else { config = c } </del> <ins> // If user only want to generate a configure file. if len(o.WriteConfigTo) > 0 { return o.writeConfigFile() </ins> } <del> // Apply algorithms based on feature gates. // TODO: make configurable? algorithmprovider.ApplyFeatureGates() server, err := NewSchedulerServer(config, o.master) </del> <ins> server, err := NewSchedulerServer(o.config, o.master) </ins> if err != nil { return err }


return server.Run(stop) } <ins> func (o *Options) writeConfigFile() error { var encoder runtime.Encoder mediaTypes := o.codecs.SupportedMediaTypes() for _, info := range mediaTypes { if info.MediaType == "application/yaml" { encoder = info.Serializer break } } if encoder == nil { return errors.New("unable to locate yaml encoder") } encoder = json.NewYAMLSerializer(json.DefaultMetaFactory, o.scheme, o.scheme) encoder = o.codecs.EncoderForVersion(encoder, componentconfigv1alpha1.SchemeGroupVersion) configFile, err := os.Create(o.WriteConfigTo) if err != nil { return err } defer configFile.Close() if err := encoder.Encode(o.config, configFile); err != nil { return err } glog.Infof("Wrote configuration to: %sn", o.WriteConfigTo) return nil } </ins> // NewSchedulerCommand creates a *cobra.Command object with default parameters func NewSchedulerCommand() *cobra.Command { opts, err := NewOptions()


return cmdutil.AddSourceToErr(fmt.Sprintf("retrieving modified configuration from:n%vnfor:", info), info.Source, err) } <ins> // Print object only if output format other than "name" is specified printObject := len(output) > 0 && !shortOutput </ins> if err := info.Get(); err != nil { if !errors.IsNotFound(err) { return cmdutil.AddSourceToErr(fmt.Sprintf("retrieving current configuration of:n%vnfrom server for:", info), info.Source, err)


} count++ <del> if len(output) > 0 && !shortOutput { </del> <ins> if printObject { </ins> return cmdutil.PrintObject(cmd, info.Object, out) } cmdutil.PrintSuccess(shortOutput, out, info.Object, dryRun, "created")


visitedUids.Insert(string(uid)) } <del> if string(patchBytes) == "{}" { </del> <ins> if string(patchBytes) == "{}" && !printObject { </ins> count++ cmdutil.PrintSuccess(shortOutput, out, info.Object, false, "unchanged") return nil } } count++ <del> if len(output) > 0 && !shortOutput { </del> <ins> if printObject { </ins> return cmdutil.PrintObject(cmd, info.Object, out) } cmdutil.PrintSuccess(shortOutput, out, info.Object, dryRun, "configured")


s.GenericServerRunOptions.AddUniversalFlags(fs) s.Etcd.AddFlags(fs) s.SecureServing.AddFlags(fs) <del> s.SecureServing.AddDeprecatedFlags(fs) </del> s.InsecureServing.AddFlags(fs) s.InsecureServing.AddDeprecatedFlags(fs) s.Audit.AddFlags(fs)


limitations under the License. */ <del> // Package options contains flags and options for initializing an apiserver </del> <ins> // Package options contains flags and options for initializing kube-apiserver </ins> package options import (


kubeserver "k8s.io/kubernetes/pkg/kubeapiserver/server" ) <del> // NewSecureServingOptions gives default values for the kube-apiserver and federation-apiserver which are not the options wanted by </del> <ins> // NewSecureServingOptions gives default values for the kube-apiserver which are not the options wanted by </ins> // "normal" API servers running on the platform func NewSecureServingOptions() *genericoptions.SecureServingOptions { return &genericoptions.SecureServingOptions{


if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() { hostIP, err := insecure.DefaultExternalAddress() if err != nil { <del> return fmt.Errorf("Unable to find suitable network address.error='%v'. "+ "Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err) </del> <ins> return fmt.Errorf("unable to find suitable network address.error='%v'. "+ "Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this", err) </ins> } s.AdvertiseAddress = hostIP }


errors := []error{} if s.BindPort < 0 || s.BindPort > 65535 { <del> errors = append(errors, fmt.Errorf("--insecure-port %v must be between 0 and 65535, inclusive. 0 for turning off insecure (HTTP) port.", s.BindPort)) </del> <ins> errors = append(errors, fmt.Errorf("--insecure-port %v must be between 0 and 65535, inclusive. 0 for turning off insecure (HTTP) port", s.BindPort)) </ins> } return errors


func (s *InsecureServingOptions) AddFlags(fs *pflag.FlagSet) { fs.IPVar(&s.BindAddress, "insecure-bind-address", s.BindAddress, ""+ "The IP address on which to serve the --insecure-port (set to 0.0.0.0 for all interfaces).") <ins> fs.MarkDeprecated("insecure-bind-address", "This flag will be removed in a future version.") </ins> fs.IntVar(&s.BindPort, "insecure-port", s.BindPort, ""+ "The port on which to serve unsecured, unauthenticated access. It is assumed "+ "that firewall rules are set up such that this port is not reachable from outside of "+ "the cluster and that port 443 on the cluster's public address is proxied to this "+ <del> "port. This is performed by nginx in the default setup. Set to zero to disable") </del> <ins> "port. This is performed by nginx in the default setup. Set to zero to disable.") fs.MarkDeprecated("insecure-port", "This flag will be removed in a future version.") </ins> } <ins> // TODO: remove it until kops stop using `--address` </ins> func (s *InsecureServingOptions) AddDeprecatedFlags(fs *pflag.FlagSet) { fs.IPVar(&s.BindAddress, "address", s.BindAddress, "DEPRECATED: see --insecure-bind-address instead.")


errors := []error{} if s.BindPort < 0 || s.BindPort > 65535 { <del> errors = append(errors, fmt.Errorf("--secure-port %v must be between 0 and 65535, inclusive. 0 for turning off secure port.", s.BindPort)) </del> <ins> errors = append(errors, fmt.Errorf("--secure-port %v must be between 0 and 65535, inclusive. 0 for turning off secure port", s.BindPort)) </ins> } return errors


} <del> func (s *SecureServingOptions) AddDeprecatedFlags(fs *pflag.FlagSet) { fs.IPVar(&s.BindAddress, "public-address-override", s.BindAddress, "DEPRECATED: see --bind-address instead.") fs.MarkDeprecated("public-address-override", "see --bind-address instead.") } </del> // ApplyTo fills up serving information in the server configuration. func (s *SecureServingOptions) ApplyTo(c *server.Config) error { if s == nil {


ec.InvalidateCachedPredicateItem(nodeName, invalidPredicates) } <del> // getHashEquivalencePod returns the hash of equivalence pod. // 1. equivalenceHash // 2. if equivalence hash is valid func (ec *EquivalenceCache) getHashEquivalencePod(pod *v1.Pod) (uint64, bool) { </del> <ins> // equivalenceClassInfo holds equivalence hash which is used for checking equivalence cache. // We will pass this to podFitsOnNode to ensure equivalence hash is only calculated per schedule. type equivalenceClassInfo struct { // Equivalence hash. hash uint64 } // getEquivalenceClassInfo returns the equivalence class of given pod. func (ec *EquivalenceCache) getEquivalenceClassInfo(pod *v1.Pod) *equivalenceClassInfo { </ins> equivalencePod := ec.getEquivalencePod(pod) if equivalencePod != nil { hash := fnv.New32a() hashutil.DeepHashObject(hash, equivalencePod) <del> return uint64(hash.Sum32()), true </del> <ins> return &equivalenceClassInfo{ hash: uint64(hash.Sum32()), } </ins> } <del> return 0, false </del> <ins> return nil </ins> }


for _, test := range tests { for i, podInfo := range test.podInfoList { testPod := podInfo.pod <del> hash, isValid := ecache.getHashEquivalencePod(testPod) if isValid != podInfo.hashIsValid { </del> <ins> eclassInfo := ecache.getEquivalenceClassInfo(testPod) if eclassInfo == nil && podInfo.hashIsValid { </ins> t.Errorf("Failed: pod %v is expected to have valid hash", testPod) } <del> // NOTE(harry): the first element will be used as target so // this logic can't verify more than two inequivalent pods if i == 0 { targetHash = hash targetPodInfo = podInfo } else { if targetHash != hash { if test.isEquivalent { t.Errorf("Failed: pod: %v is expected to be equivalent to: %v", testPod, targetPodInfo.pod) </del> <ins> if eclassInfo != nil { // NOTE(harry): the first element will be used as target so // this logic can't verify more than two inequivalent pods if i == 0 { targetHash = eclassInfo.hash targetPodInfo = podInfo } else { if targetHash != eclassInfo.hash { if test.isEquivalent { t.Errorf("Failed: pod: %v is expected to be equivalent to: %v", testPod, targetPodInfo.pod) } </ins> } } }


// We can use the same metadata producer for all nodes. meta := metadataProducer(pod, nodeNameToInfo) <ins> var equivCacheInfo *equivalenceClassInfo if ecache != nil { // getEquivalenceClassInfo will return immediately if no equivalence pod found equivCacheInfo = ecache.getEquivalenceClassInfo(pod) } </ins> checkNode := func(i int) { nodeName := nodes[i].Name <del> fits, failedPredicates, err := podFitsOnNode(pod, meta, nodeNameToInfo[nodeName], predicateFuncs, ecache, schedulingQueue, alwaysCheckAllPredicates) </del> <ins> fits, failedPredicates, err := podFitsOnNode( pod, meta, nodeNameToInfo[nodeName], predicateFuncs, ecache, schedulingQueue, alwaysCheckAllPredicates, equivCacheInfo, ) </ins> if err != nil { predicateResultLock.Lock() errs[err.Error()]++


} // podFitsOnNode checks whether a node given by NodeInfo satisfies the given predicate functions. <ins> // For given pod, podFitsOnNode will check if any equivalent pod exists and try to reuse its cached // predicate results as possible. </ins> // This function is called from two different places: Schedule and Preempt. // When it is called from Schedule, we want to test whether the pod is schedulable // on the node with all the existing pods on the node plus higher and equal priority


ecache *EquivalenceCache, queue SchedulingQueue, alwaysCheckAllPredicates bool, <ins> equivCacheInfo *equivalenceClassInfo, </ins> ) (bool, []algorithm.PredicateFailureReason, error) { var ( <del> equivalenceHash  uint64 failedPredicates []algorithm.PredicateFailureReason </del> eCacheAvailable  bool <ins> failedPredicates []algorithm.PredicateFailureReason </ins> invalid          bool fit              bool reasons          []algorithm.PredicateFailureReason


) predicateResults := make(map[string]HostPredicate) <del> if ecache != nil { // getHashEquivalencePod will return immediately if no equivalence pod found equivalenceHash, eCacheAvailable = ecache.getHashEquivalencePod(pod) } </del> podsAdded := false // We run predicates twice in some cases. If the node has greater or equal priority // nominated pods, we run them when those pods are added to meta and nodeInfo.


// Bypass eCache if node has any nominated pods. // TODO(bsalamat): consider using eCache and adding proper eCache invalidations // when pods are nominated or their nominations change. <del> eCacheAvailable = eCacheAvailable && !podsAdded </del> <ins> eCacheAvailable = equivCacheInfo != nil && !podsAdded </ins> for _, predicateKey := range predicates.PredicatesOrdering() { <del> //TODO (yastij) : compute average predicate restrictiveness to export it as promethus metric </del> <ins> //TODO (yastij) : compute average predicate restrictiveness to export it as Prometheus metric </ins> if predicate, exist := predicateFuncs[predicateKey]; exist { if eCacheAvailable { // PredicateWithECache will return its cached predicate results. <del> fit, reasons, invalid = ecache.PredicateWithECache(pod.GetName(), info.Node().GetName(), predicateKey, equivalenceHash) </del> <ins> fit, reasons, invalid = ecache.PredicateWithECache(pod.GetName(), info.Node().GetName(), predicateKey, equivCacheInfo.hash) </ins> } if !eCacheAvailable || invalid {


for predKey, result := range predicateResults { // update equivalence cache with newly computed fit & reasons // TODO(resouer) should we do this in another thread? any race? <del> ecache.UpdateCachedPredicateItem(pod.GetName(), nodeName, predKey, result.Fit, result.FailReasons, equivalenceHash) </del> <ins> ecache.UpdateCachedPredicateItem(pod.GetName(), nodeName, predKey, result.Fit, result.FailReasons, equivCacheInfo.hash) </ins> } } return len(failedPredicates) == 0, failedPredicates, nil


// that we should check is if the "pod" is failing to schedule due to pod affinity // failure. // TODO(bsalamat): Consider checking affinity to lower priority pods if feasible with reasonable performance. <del> if fits, _, err := podFitsOnNode(pod, meta, nodeInfoCopy, fitPredicates, nil, queue, false); !fits { </del> <ins> if fits, _, err := podFitsOnNode(pod, meta, nodeInfoCopy, fitPredicates, nil, queue, false, nil); !fits { </ins> if err != nil { glog.Warningf("Encountered error while selecting victims on node %v: %v", nodeInfo.Node().Name, err) }


violatingVictims, nonViolatingVictims := filterPodsWithPDBViolation(potentialVictims.Items, pdbs) reprievePod := func(p *v1.Pod) bool { addPod(p) <del> fits, _, _ := podFitsOnNode(pod, meta, nodeInfoCopy, fitPredicates, nil, queue, false) </del> <ins> fits, _, _ := podFitsOnNode(pod, meta, nodeInfoCopy, fitPredicates, nil, queue, false, nil) </ins> if !fits { removePod(p) victims = append(victims, p)


"//pkg/util/sysctl:go_default_library", "//pkg/util/version:go_default_library", "//vendor/github.com/golang/glog:go_default_library", <del> "//vendor/golang.org/x/sys/unix:go_default_library", </del> "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",


] + select({ "@io_bazel_rules_go//go/platform:linux": [ "//vendor/github.com/vishvananda/netlink:go_default_library", <ins> "//vendor/golang.org/x/sys/unix:go_default_library", </ins> ], "//conditions:default": [], }),


return false, nil } <del> // UnbindAddress unbind address from the interface </del> <ins> // UnbindAddress makes sure IP address is unbound from the network interface. </ins> func (h *netlinkHandle) UnbindAddress(address, devName string) error { dev, err := h.LinkByName(devName) if err != nil {


return fmt.Errorf("error parse ip address: %s", address) } if err := h.AddrDel(dev, &netlink.Addr{IPNet: netlink.NewIPNet(addr)}); err != nil { <del> return fmt.Errorf("error unbind address: %s from interface: %s, err: %v", address, devName, err) </del> <ins> if err != unix.ENXIO { return fmt.Errorf("error unbind address: %s from interface: %s, err: %v", address, devName, err) } </ins> } return nil }


utilipvs "k8s.io/kubernetes/pkg/util/ipvs" utilsysctl "k8s.io/kubernetes/pkg/util/sysctl" utilexec "k8s.io/utils/exec" <del> "golang.org/x/sys/unix" </del> ) const (


for _, addr := range unbindIPAddr.UnsortedList() { err := proxier.netlinkHandle.UnbindAddress(addr, DefaultDummyDevice) // Ignore no such address error when try to unbind address <del> if err != nil && err != unix.ENXIO { </del> <ins> if err != nil { </ins> glog.Errorf("Failed to unbind service addr %s from dummy interface %s: %v", addr, DefaultDummyDevice, err) } }


} if err := d.writeCachedFile(filename, liveResources); err != nil { <del> glog.V(3).Infof("failed to write cache to %v due to %v", filename, err) </del> <ins> glog.V(1).Infof("failed to write cache to %v due to %v", filename, err) </ins> } return liveResources, nil


} if err := d.writeCachedFile(filename, liveGroups); err != nil { <del> glog.V(3).Infof("failed to write cache to %v due to %v", filename, err) </del> <ins> glog.V(1).Infof("failed to write cache to %v due to %v", filename, err) </ins> } return liveGroups, nil


// 3. Ties are broken by sum of priorities of all victims. // 4. If there are still ties, node with the minimum number of victims is picked. // 5. If there are still ties, the first such node is picked (sort of randomly). <del> //TODO(bsalamat): Try to reuse the "min*Nodes" slices in order to save GC time. </del> <ins> // The 'minNodes1' and 'minNodes2' are being reused here to save the memory // allocation and garbage collection time. </ins> func pickOneNodeForPreemption(nodesToVictims map[*v1.Node]*Victims) *v1.Node { if len(nodesToVictims) == 0 { return nil } minNumPDBViolatingPods := math.MaxInt32 <del> var minPDBViolatingNodes []*v1.Node </del> <ins> var minNodes1 []*v1.Node lenNodes1 := 0 </ins> for node, victims := range nodesToVictims { if len(victims.pods) == 0 { // We found a node that doesn't need any preemption. Return it!


numPDBViolatingPods := victims.numPDBViolations if numPDBViolatingPods < minNumPDBViolatingPods { minNumPDBViolatingPods = numPDBViolatingPods <del> minPDBViolatingNodes = nil </del> <ins> minNodes1 = nil lenNodes1 = 0 </ins> } if numPDBViolatingPods == minNumPDBViolatingPods { <del> minPDBViolatingNodes = append(minPDBViolatingNodes, node) </del> <ins> minNodes1 = append(minNodes1, node) lenNodes1++ </ins> } } <del> if len(minPDBViolatingNodes) == 1 { return minPDBViolatingNodes[0] </del> <ins> if lenNodes1 == 1 { return minNodes1[0] </ins> } // There are more than one node with minimum number PDB violating pods. Find // the one with minimum highest priority victim. minHighestPriority := int32(math.MaxInt32) <del> var minPriorityNodes []*v1.Node for _, node := range minPDBViolatingNodes { </del> <ins> var minNodes2 = make([]*v1.Node, lenNodes1) lenNodes2 := 0 for i := 0; i < lenNodes1; i++ { node := minNodes1[i] </ins> victims := nodesToVictims[node] // highestPodPriority is the highest priority among the victims on this node. highestPodPriority := util.GetPodPriority(victims.pods[0]) if highestPodPriority < minHighestPriority { minHighestPriority = highestPodPriority <del> minPriorityNodes = nil </del> <ins> lenNodes2 = 0 </ins> } if highestPodPriority == minHighestPriority { <del> minPriorityNodes = append(minPriorityNodes, node) </del> <ins> minNodes2[lenNodes2] = node lenNodes2++ </ins> } } <del> if len(minPriorityNodes) == 1 { return minPriorityNodes[0] </del> <ins> if lenNodes2 == 1 { return minNodes2[0] </ins> } // There are a few nodes with minimum highest priority victim. Find the // smallest sum of priorities. minSumPriorities := int64(math.MaxInt64) <del> var minSumPriorityNodes []*v1.Node for _, node := range minPriorityNodes { </del> <ins> lenNodes1 = 0 for i := 0; i < lenNodes2; i++ { </ins> var sumPriorities int64 <ins> node := minNodes2[i] </ins> for _, pod := range nodesToVictims[node].pods { // We add MaxInt32+1 to all priorities to make all of them >= 0. This is // needed so that a node with a few pods with negative priority is not


} if sumPriorities < minSumPriorities { minSumPriorities = sumPriorities <del> minSumPriorityNodes = nil </del> <ins> lenNodes1 = 0 </ins> } if sumPriorities == minSumPriorities { <del> minSumPriorityNodes = append(minSumPriorityNodes, node) </del> <ins> minNodes1[lenNodes1] = node lenNodes1++ </ins> } } <del> if len(minSumPriorityNodes) == 1 { return minSumPriorityNodes[0] </del> <ins> if lenNodes1 == 1 { return minNodes1[0] </ins> } // There are a few nodes with minimum highest priority victim and sum of priorities. // Find one with the minimum number of pods. minNumPods := math.MaxInt32 <del> var minNumPodNodes []*v1.Node for _, node := range minSumPriorityNodes { </del> <ins> lenNodes2 = 0 for i := 0; i < lenNodes1; i++ { node := minNodes1[i] </ins> numPods := len(nodesToVictims[node].pods) if numPods < minNumPods { minNumPods = numPods <del> minNumPodNodes = nil </del> <ins> lenNodes2 = 0 </ins> } if numPods == minNumPods { <del> minNumPodNodes = append(minNumPodNodes, node) </del> <ins> minNodes2[lenNodes2] = node lenNodes2++ </ins> } } // At this point, even if there are more than one node with the same score, // return the first one. <del> if len(minNumPodNodes) > 0 { return minNumPodNodes[0] </del> <ins> if lenNodes2 > 0 { return minNodes2[0] </ins> } glog.Errorf("Error in logic of node scoring for preemption. We should never reach here!") return nil


Status: v1.ConditionTrue, }) <del> hostIP, err := kl.getHostIPAnyWay() if err != nil { glog.V(4).Infof("Cannot get host IP: %v", err) return *s } s.HostIP = hostIP.String() if kubecontainer.IsHostNetworkPod(pod) && s.PodIP == "" { s.PodIP = hostIP.String() </del> <ins> if kl.kubeClient != nil { hostIP, err := kl.getHostIPAnyWay() if err != nil { glog.V(4).Infof("Cannot get host IP: %v", err) } else { s.HostIP = hostIP.String() if kubecontainer.IsHostNetworkPod(pod) && s.PodIP == "" { s.PodIP = hostIP.String() } } </ins> } return *s }


glog.Infof(volumeToMount.GenerateMsgDetailed("MapVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath))) <del> // Update actual state of world to reflect volume is globally mounted markDeviceMappedErr := actualStateOfWorld.MarkDeviceAsMounted( volumeToMount.VolumeName, devicePath, globalMapPath) if markDeviceMappedErr != nil { // On failure, return error. Caller will log and retry. return volumeToMount.GenerateError("MapVolume.MarkDeviceAsMounted failed", markDeviceMappedErr) } </del> } // A plugin doesn't have attacher also needs to map device to global map path with SetUpDevice() pluginDevicePath, mapErr := blockVolumeMapper.SetUpDevice()


return volumeToMount.GenerateError("MapVolume failed", fmt.Errorf("Device path of the volume is empty")) } } <ins> // Update actual state of world to reflect volume is globally mounted markDeviceMappedErr := actualStateOfWorld.MarkDeviceAsMounted( volumeToMount.VolumeName, devicePath, globalMapPath) if markDeviceMappedErr != nil { // On failure, return error. Caller will log and retry. return volumeToMount.GenerateError("MapVolume.MarkDeviceAsMounted failed", markDeviceMappedErr) } </ins> mapErr = og.blkUtil.MapDevice(devicePath, globalMapPath, string(volumeToMount.Pod.UID)) if mapErr != nil {


// Try to unmap podUID symlink under global map path dir // plugins/kubernetes.io/{PluginName}/volumeDevices/{volumePluginDependentPath}/{podUID} globalUnmapPath := volumeToUnmount.DeviceMountPath <del> if err != nil { // On failure, return error. Caller will log and retry. return volumeToUnmount.GenerateError("UnmapVolume.GetGlobalUnmapPath failed", err) } </del> unmapDeviceErr = og.blkUtil.UnmapDevice(globalUnmapPath, string(volumeToUnmount.PodUID)) if unmapDeviceErr != nil { // On failure, return error. Caller will log and retry.


// Search under globalMapPath dir if all symbolic links from pods have been removed already. // If symbolick links are there, pods may still refer the volume. globalMapPath := deviceToDetach.DeviceMountPath <del> if err != nil { // On failure, return error. Caller will log and retry. return deviceToDetach.GenerateError("UnmapDevice.GetGlobalMapPath failed", err) } </del> refs, err := og.blkUtil.GetDeviceSymlinkRefs(deviceToDetach.DevicePath, globalMapPath) if err != nil { return deviceToDetach.GenerateError("UnmapDevice.GetDeviceSymlinkRefs check failed", err)


if newRS != nil && newRS.UID == rsUID { continue } <del> if len(podList.Items) > 0 { return true </del> <ins> for _, pod := range podList.Items { switch pod.Status.Phase { case v1.PodFailed, v1.PodSucceeded: // Don't count pods in terminal state. continue case v1.PodUnknown: // This happens in situation like when the node is temporarily disconnected from the cluster. // If we can't be sure that the pod is not running, we have to count it. return true default: // Pod is not in terminal phase. return true } </ins> } } return false


oldRSs []*extensions.ReplicaSet podMap map[types.UID]*v1.PodList <del> expected bool </del> <ins> hasOldPodsRunning bool </ins> }{ { <del> name:     "no old RSs", expected: false, </del> <ins> name:              "no old RSs", hasOldPodsRunning: false, </ins> }, { <del> name:     "old RSs with running pods", oldRSs:   []*extensions.ReplicaSet{rsWithUID("some-uid"), rsWithUID("other-uid")}, podMap:   podMapWithUIDs([]string{"some-uid", "other-uid"}), expected: true, </del> <ins> name:              "old RSs with running pods", oldRSs:            []*extensions.ReplicaSet{rsWithUID("some-uid"), rsWithUID("other-uid")}, podMap:            podMapWithUIDs([]string{"some-uid", "other-uid"}), hasOldPodsRunning: true, </ins> }, { <del> name:     "old RSs without pods but with non-zero status replicas", oldRSs:   []*extensions.ReplicaSet{newRSWithStatus("rs-blabla", 0, 1, nil)}, expected: true, </del> <ins> name:              "old RSs without pods but with non-zero status replicas", oldRSs:            []*extensions.ReplicaSet{newRSWithStatus("rs-1", 0, 1, nil)}, hasOldPodsRunning: true, </ins> }, { <del> name:     "old RSs without pods or non-zero status replicas", oldRSs:   []*extensions.ReplicaSet{newRSWithStatus("rs-blabla", 0, 0, nil)}, expected: false, </del> <ins> name:              "old RSs without pods or non-zero status replicas", oldRSs:            []*extensions.ReplicaSet{newRSWithStatus("rs-1", 0, 0, nil)}, hasOldPodsRunning: false, }, { name:   "old RSs with zero status replicas but pods in terminal state are present", oldRSs: []*extensions.ReplicaSet{newRSWithStatus("rs-1", 0, 0, nil)}, podMap: map[types.UID]*v1.PodList{ "uid-1": { Items: []v1.Pod{ { Status: v1.PodStatus{ Phase: v1.PodFailed, }, }, { Status: v1.PodStatus{ Phase: v1.PodSucceeded, }, }, }, }, }, hasOldPodsRunning: false, }, { name:   "old RSs with zero status replicas but pod in unknown phase present", oldRSs: []*extensions.ReplicaSet{newRSWithStatus("rs-1", 0, 0, nil)}, podMap: map[types.UID]*v1.PodList{ "uid-1": { Items: []v1.Pod{ { Status: v1.PodStatus{ Phase: v1.PodUnknown, }, }, }, }, }, hasOldPodsRunning: true, }, { name:   "old RSs with zero status replicas with pending pod present", oldRSs: []*extensions.ReplicaSet{newRSWithStatus("rs-1", 0, 0, nil)}, podMap: map[types.UID]*v1.PodList{ "uid-1": { Items: []v1.Pod{ { Status: v1.PodStatus{ Phase: v1.PodPending, }, }, }, }, }, hasOldPodsRunning: true, }, { name:   "old RSs with zero status replicas with running pod present", oldRSs: []*extensions.ReplicaSet{newRSWithStatus("rs-1", 0, 0, nil)}, podMap: map[types.UID]*v1.PodList{ "uid-1": { Items: []v1.Pod{ { Status: v1.PodStatus{ Phase: v1.PodRunning, }, }, }, }, }, hasOldPodsRunning: true, }, { name:   "old RSs with zero status replicas but pods in terminal state and pending are present", oldRSs: []*extensions.ReplicaSet{newRSWithStatus("rs-1", 0, 0, nil)}, podMap: map[types.UID]*v1.PodList{ "uid-1": { Items: []v1.Pod{ { Status: v1.PodStatus{ Phase: v1.PodFailed, }, }, { Status: v1.PodStatus{ Phase: v1.PodSucceeded, }, }, }, }, "uid-2": { Items: []v1.Pod{}, }, "uid-3": { Items: []v1.Pod{ { Status: v1.PodStatus{ Phase: v1.PodPending, }, }, }, }, }, hasOldPodsRunning: true, </ins> }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { <del> if expected, got := test.expected, oldPodsRunning(test.newRS, test.oldRSs, test.podMap); expected != got { </del> <ins> if expected, got := test.hasOldPodsRunning, oldPodsRunning(test.newRS, test.oldRSs, test.podMap); expected != got { </ins> t.Errorf("%s: expected %t, got %t", test.name, expected, got) } })


"//cmd/kube-proxy/app:go_default_library", "//pkg/client/metrics/prometheus:go_default_library", "//pkg/version/prometheus:go_default_library", <ins> "//vendor/github.com/spf13/pflag:go_default_library", </ins> "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/logs:go_default_library", ],


glog.Fatalf("unable to create flag defaults: %v", err) } <del> opts.AddFlags(pflag.CommandLine) </del> <ins> opts.AddFlags(cmd.Flags()) </ins> cmd.MarkFlagFilename("config", "yaml", "yml", "json")


package main import ( <ins> goflag "flag" </ins> "fmt" "math/rand" "os" "time" <ins> "github.com/spf13/pflag" </ins> utilflag "k8s.io/apiserver/pkg/util/flag" "k8s.io/apiserver/pkg/util/logs" "k8s.io/kubernetes/cmd/kube-proxy/app"


command := app.NewProxyCommand() <del> utilflag.InitFlags() </del> <ins> // TODO: once we switch everything over to Cobra commands, we can go back to calling // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the // normalize func and add the go flag set by hand. pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc) pflag.CommandLine.AddGoFlagSet(goflag.CommandLine) // utilflag.InitFlags() </ins> logs.InitLogs() defer logs.FlushLogs()


if err != nil { return nil, err } <ins> readOnly, err := getReadOnlyFromSpec(spec) if err != nil { return nil, err } </ins> // before it is used in any paths such as socket etc addr := fmt.Sprintf(csiAddrTemplate, pvSource.Driver)


volumeID:     pvSource.VolumeHandle, specVolumeID: spec.Name(), csiClient:    client, <ins> readOnly:     readOnly, </ins> } return mounter, nil }


return nil, fmt.Errorf("CSIPersistentVolumeSource not defined in spec") } <ins> func getReadOnlyFromSpec(spec *volume.Spec) (bool, error) { if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.CSI != nil { return spec.ReadOnly, nil } return false, fmt.Errorf("CSIPersistentVolumeSource not defined in spec") } </ins> // log prepends log string with `kubernetes.io/csi` func log(msg string, parts ...interface{}) string { return fmt.Sprintf(fmt.Sprintf("%s: %s", csiPluginName, msg), parts...)


set +o errexit } <ins> # Runs tests for kubectl alpha diff run_kubectl_diff_tests() { set -o nounset set -o errexit create_and_use_new_namespace kube::log::status "Testing kubectl alpha diff" kubectl apply -f hack/testdata/pod.yaml # Ensure that selfLink has been added, and shown in the diff output_message=$(kubectl alpha diff -f hack/testdata/pod.yaml) kube::test::if_has_string "${output_message}" 'selfLink' output_message=$(kubectl alpha diff LOCAL LIVE -f hack/testdata/pod.yaml) kube::test::if_has_string "${output_message}" 'selfLink' output_message=$(kubectl alpha diff LOCAL MERGED -f hack/testdata/pod.yaml) kube::test::if_has_string "${output_message}" 'selfLink' output_message=$(kubectl alpha diff MERGED MERGED -f hack/testdata/pod.yaml) kube::test::if_empty_string "${output_message}" output_message=$(kubectl alpha diff LIVE LIVE -f hack/testdata/pod.yaml) kube::test::if_empty_string "${output_message}" output_message=$(kubectl alpha diff LAST LAST -f hack/testdata/pod.yaml) kube::test::if_empty_string "${output_message}" output_message=$(kubectl alpha diff LOCAL LOCAL -f hack/testdata/pod.yaml) kube::test::if_empty_string "${output_message}" kubectl delete -f  hack/testdata/pod.yaml set +o nounset set +o errexit } </ins> # Runs tests for --save-config tests. run_save_config_tests() { set -o nounset


record_command run_kubectl_apply_deployments_tests fi <ins> ################ # Kubectl diff # ################ record_command run_kubectl_diff_tests </ins> ############### # Kubectl get # ###############


"github.com/ghodss/yaml" "github.com/spf13/cobra" <del> "k8s.io/apimachinery/pkg/api/errors" </del> "k8s.io/apimachinery/pkg/api/meta" <ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" </ins> "k8s.io/apimachinery/pkg/runtime" <ins> "k8s.io/client-go/dynamic" </ins> api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/kubectl/apply/parse" "k8s.io/kubernetes/pkg/kubectl/apply/strategy"


// InfoObject is an implementation of the Object interface. It gets all // the information from the Info object. type InfoObject struct { <ins> Remote  runtime.Unstructured </ins> Info    *resource.Info Encoder runtime.Encoder Parser  *parse.Factory


} func (obj InfoObject) Live() (map[string]interface{}, error) { <del> if obj.Info.Object == nil { </del> <ins> if obj.Remote == nil { </ins> return nil, nil // Object doesn't exist on cluster. } <del> data, err := runtime.Encode(obj.Encoder, obj.Info.Object) if err != nil { return nil, err } return obj.toMap(data) </del> <ins> return obj.Remote.UnstructuredContent(), nil </ins> } func (obj InfoObject) Merged() (map[string]interface{}, error) {


} func (obj InfoObject) Last() (map[string]interface{}, error) { <del> if obj.Info.Object == nil { </del> <ins> if obj.Remote == nil { </ins> return nil, nil // No object is live, return empty } <del> accessor, err := meta.Accessor(obj.Info.Object) </del> <ins> accessor, err := meta.Accessor(obj.Remote) </ins> if err != nil { return nil, err }


d.To.Dir.Delete()   // Ignore error } <ins> type Downloader struct { mapper  meta.RESTMapper dclient dynamic.Interface ns      string } func NewDownloader(f cmdutil.Factory) (*Downloader, error) { var err error var d Downloader d.mapper, err = f.RESTMapper() if err != nil { return nil, err } d.dclient, err = f.DynamicClient() if err != nil { return nil, err } d.ns, _, _ = f.DefaultNamespace() return &d, nil } func (d *Downloader) Download(info *resource.Info) (*unstructured.Unstructured, error) { gvk := info.Object.GetObjectKind().GroupVersionKind() mapping, err := d.mapper.RESTMapping(gvk.GroupKind(), gvk.Version) if err != nil { return nil, err } var resource dynamic.ResourceInterface switch mapping.Scope.Name() { case meta.RESTScopeNameNamespace: if info.Namespace == "" { info.Namespace = d.ns } resource = d.dclient.Resource(mapping.Resource).Namespace(info.Namespace) case meta.RESTScopeNameRoot: resource = d.dclient.Resource(mapping.Resource) } return resource.Get(info.Name, metav1.GetOptions{}) } </ins> // RunDiff uses the factory to parse file arguments, find the version to // diff, and find each Info object for each files, and runs against the // differ.


Unstructured(). NamespaceParam(cmdNamespace).DefaultNamespace(). FilenameParam(enforceNamespace, &options.FilenameOptions). <ins> Local(). </ins> Flatten(). Do() if err := r.Err(); err != nil { return err } <ins> dl, err := NewDownloader(f) if err != nil { return err } </ins> err = r.Visit(func(info *resource.Info, err error) error { if err != nil { return err } <del> if err := info.Get(); err != nil { if !errors.IsNotFound(err) { return cmdutil.AddSourceToErr(fmt.Sprintf("retrieving current configuration of:n%snfrom server for:", info.String()), info.Source, err) } info.Object = nil } </del> <ins> remote, _ := dl.Download(info) </ins> obj := InfoObject{ <ins> Remote:  remote, </ins> Info:    info, Parser:  parser, Encoder: cmdutil.InternalVersionJSONEncoder(),


// CommonAccessor returns a Common interface for the provided object or an error if the object does // not provide List. <del> // TODO: return bool instead of error </del> func CommonAccessor(obj interface{}) (metav1.Common, error) { switch t := obj.(type) { case List:


// not provide List. // IMPORTANT: Objects are NOT a superset of lists. Do not use this check to determine whether an // object *is* a List. <del> // TODO: return bool instead of error </del> func ListAccessor(obj interface{}) (List, error) { switch t := obj.(type) { case List:


// obj must be a pointer to an API type. An error is returned if the minimum // required fields are missing. Fields that are not required return the default // value and are a no-op if set. <del> // TODO: return bool instead of error </del> func Accessor(obj interface{}) (metav1.Object, error) { switch t := obj.(type) { case metav1.Object:


cmd.Flags().BoolVar(&options.DeleteAll, "all", options.DeleteAll, "Delete all resources, including uninitialized ones, in the namespace of the specified resource types.") cmd.Flags().BoolVar(&options.IgnoreNotFound, "ignore-not-found", options.IgnoreNotFound, "Treat "resource not found" as a successful delete. Defaults to "true" when --all is specified.") cmd.Flags().BoolVar(&options.Cascade, "cascade", options.Cascade, "If true, cascade the deletion of the resources managed by this resource (e.g. Pods created by a ReplicationController).  Default true.") <del> cmd.Flags().IntVar(&options.GracePeriod, "grace-period", options.GracePeriod, "Period of time in seconds given to the resource to terminate gracefully. Ignored if negative.") </del> <ins> cmd.Flags().IntVar(&options.GracePeriod, "grace-period", options.GracePeriod, "Period of time in seconds given to the resource to terminate gracefully. Ignored if negative. Set to 1 for immediate shutdown. Can only be set to 0 when --force is true (force deletion).") </ins> cmd.Flags().BoolVar(&options.DeleteNow, "now", options.DeleteNow, "If true, resources are signaled for immediate shutdown (same as --grace-period=1).") <del> cmd.Flags().BoolVar(&options.ForceDeletion, "force", options.ForceDeletion, "Immediate deletion of some resources may result in inconsistency or data loss and requires confirmation.") </del> <ins> cmd.Flags().BoolVar(&options.ForceDeletion, "force", options.ForceDeletion, "Only used when grace-period=0. If true, immediately remove resources from API and bypass graceful deletion. Note that immediate deletion of some resources may result in inconsistency or data loss and requires confirmation.") </ins> cmd.Flags().DurationVar(&options.Timeout, "timeout", options.Timeout, "The length of time to wait before giving up on a delete, zero means determine a timeout from the size of the object") cmdutil.AddOutputVarFlagsForMutation(cmd, &options.Output) cmdutil.AddInclude3rdPartyVarFlags(cmd, &options.Include3rdParty)


o.WaitForDeletion = true o.GracePeriod = 1 } <ins> } else if o.ForceDeletion { fmt.Fprintf(o.ErrOut, "warning: --force is ignored because --grace-period is not 0.n") </ins> } return nil }


// If there is no reaper for this resources and the user didn't explicitly ask for stop. if kubectl.IsNoSuchReaperError(err) && isDefaultDelete { // No client side reaper found. Let the server do cascading deletion. <del> return cascadingDeleteResource(info, out, shortOutput) </del> <ins> return cascadingDeleteResource(info, out, shortOutput, gracePeriod) </ins> } return cmdutil.AddSourceToErr("reaping", info.Source, err) }


} } if !quiet { <del> printDeletion(info, out, shortOutput) </del> <ins> printDeletion(info, out, shortOutput, gracePeriod) </ins> } return nil })


options = metav1.NewDeleteOptions(int64(gracePeriod)) } options.OrphanDependents = &orphan <del> return deleteResource(info, out, shortOutput, options) </del> <ins> return deleteResource(info, out, shortOutput, options, gracePeriod) </ins> }) if err != nil { return err


return nil } <del> func cascadingDeleteResource(info *resource.Info, out io.Writer, shortOutput bool) error { </del> <ins> func cascadingDeleteResource(info *resource.Info, out io.Writer, shortOutput bool, gracePeriod int) error { </ins> falseVar := false deleteOptions := &metav1.DeleteOptions{OrphanDependents: &falseVar} <del> return deleteResource(info, out, shortOutput, deleteOptions) </del> <ins> return deleteResource(info, out, shortOutput, deleteOptions, gracePeriod) </ins> } <del> func deleteResource(info *resource.Info, out io.Writer, shortOutput bool, deleteOptions *metav1.DeleteOptions) error { </del> <ins> func deleteResource(info *resource.Info, out io.Writer, shortOutput bool, deleteOptions *metav1.DeleteOptions, gracePeriod int) error { </ins> if err := resource.NewHelper(info.Client, info.Mapping).DeleteWithOptions(info.Namespace, info.Name, deleteOptions); err != nil { return cmdutil.AddSourceToErr("deleting", info.Source, err) } <del> printDeletion(info, out, shortOutput) </del> <ins> printDeletion(info, out, shortOutput, gracePeriod) </ins> return nil } // deletion printing is special because they don't have an object to print.  This logic mirrors PrintSuccess <del> func printDeletion(info *resource.Info, out io.Writer, shortOutput bool) { </del> <ins> func printDeletion(info *resource.Info, out io.Writer, shortOutput bool, gracePeriod int) { </ins> operation := "deleted" groupKind := info.Mapping.GroupVersionKind kindString := fmt.Sprintf("%s.%s", strings.ToLower(groupKind.Kind), groupKind.Group)


kindString = strings.ToLower(groupKind.Kind) } <ins> if gracePeriod == 0 { operation = "force deleted" } </ins> if shortOutput { // -o name: prints resource/name fmt.Fprintf(out, "%s/%sn", kindString, info.Name)


"format": "byte" }, "service": { <del> "description": "`service` is a reference to the service for this webhook. Either `service` or `url` must be specified.nnIf the webhook is running within the cluster, then you should use `service`.nnIf there is only one port open for the service, that port will be used. If there are multiple ports open, port 443 will be used if it is open, otherwise it is an error.", </del> <ins> "description": "`service` is a reference to the service for this webhook. Either `service` or `url` must be specified.nnIf the webhook is running within the cluster, then you should use `service`.nnPort 443 will be used if it is open, otherwise it is an error.", </ins> "$ref": "#/definitions/io.k8s.api.admissionregistration.v1beta1.ServiceReference" }, "url": {


}, "service": { "$ref": "v1beta1.ServiceReference", <del> "description": "`service` is a reference to the service for this webhook. Either `service` or `url` must be specified.nnIf the webhook is running within the cluster, then you should use `service`.nnIf there is only one port open for the service, that port will be used. If there are multiple ports open, port 443 will be used if it is open, otherwise it is an error." </del> <ins> "description": "`service` is a reference to the service for this webhook. Either `service` or `url` must be specified.nnIf the webhook is running within the cluster, then you should use `service`.nnPort 443 will be used if it is open, otherwise it is an error." </ins> }, "caBundle": { "type": "string",


 If the webhook is running within the cluster, then you should use service.

  If there is only one port open for the service, that port will be used. If there are multiple ports open, port 443 will be used if it is open, otherwise it is an error.
	   Port 443 will be used if it is open, otherwise it is an error.

true

v1beta1.ServiceReference


// // If the webhook is running within the cluster, then you should use `service`. // <del> // If there is only one port open for the service, that port will be // used. If there are multiple ports open, port 443 will be used if it // is open, otherwise it is an error. </del> <ins> // Port 443 will be used if it is open, otherwise it is an error. </ins> // // +optional Service *ServiceReference