// Tune conntrack, if requested if s.Conntracker != nil { if s.Config.ConntrackMax > 0 { <del> if err := s.Conntracker.SetMax(int(s.Config.ConntrackMax)); err != nil { return err </del> <ins> err := s.Conntracker.SetMax(int(s.Config.ConntrackMax)) if err != nil { if err != readOnlySysFSError { return err } // readOnlySysFSError is caused by a known docker issue (https://github.com/docker/docker/issues/24000), // the only remediation we know is to restart the docker daemon. // Here we'll send an node event with specific reason and message, the // administrator should decide whether and how to handle this issue, // whether to drain the node and restart docker. // TODO(random-liu): Remove this when the docker bug is fixed. const message = "DOCKER RESTART NEEDED (docker issue #24000): /sys is read-only: can't raise conntrack limits, problems may arise later." s.Recorder.Eventf(s.Config.NodeRef, api.EventTypeWarning, err.Error(), message) </ins> } } if s.Config.ConntrackTCPEstablishedTimeout.Duration > 0 {


return ConformanceContainer{containers[0], cc.Client, pod.Spec.RestartPolicy, pod.Spec.Volumes, pod.Spec.NodeName, cc.Namespace, cc.podName}, nil } <del> func (cc *ConformanceContainer) GetStatus() (api.ContainerStatus, api.PodPhase, error) { </del> <ins> func (cc *ConformanceContainer) IsReady() (bool, error) { </ins> pod, err := cc.Client.Pods(cc.Namespace).Get(cc.podName) if err != nil { <del> return api.ContainerStatus{}, api.PodUnknown, err </del> <ins> return false, err </ins> } <ins> return api.IsPodReady(pod), nil } func (cc *ConformanceContainer) GetPhase() (api.PodPhase, error) { pod, err := cc.Client.Pods(cc.Namespace).Get(cc.podName) if err != nil { return api.PodUnknown, err } return pod.Status.Phase, nil } func (cc *ConformanceContainer) GetStatus() (api.ContainerStatus, error) { pod, err := cc.Client.Pods(cc.Namespace).Get(cc.podName) if err != nil { return api.ContainerStatus{}, err } </ins> statuses := pod.Status.ContainerStatuses <del> if len(statuses) != 1 { return api.ContainerStatus{}, api.PodUnknown, errors.New("Failed to get container status") </del> <ins> if len(statuses) != 1 || statuses[0].Name != cc.Container.Name { return api.ContainerStatus{}, fmt.Errorf("unexpected container statuses %v", statuses) </ins> } <del> return statuses[0], pod.Status.Phase, nil </del> <ins> return statuses[0], nil </ins> } func (cc *ConformanceContainer) Present() (bool, error) {


return false, err } <del> type ContainerState uint32 </del> <ins> type ContainerState int </ins> const ( <del> ContainerStateWaiting ContainerState = 1 << iota </del> <ins> ContainerStateWaiting ContainerState = iota </ins> ContainerStateRunning ContainerStateTerminated ContainerStateUnknown


pollInterval           = time.Second * 5 ) <del> type testStatus struct { </del> <ins> type testCase struct { </ins> Name             string RestartPolicy    api.RestartPolicy Phase            api.PodPhase


Ready            bool } <del> var _ = Describe("[FLAKY] Container runtime Conformance Test", func() { </del> <ins> var _ = Describe("Container Runtime Conformance Test", func() { </ins> var cl *client.Client BeforeEach(func() {


}) Describe("container runtime conformance blackbox test", func() { <del> var testCContainers []ConformanceContainer </del> namespace := "runtime-conformance" <del> BeforeEach(func() { testCContainers = []ConformanceContainer{} }) Context("when start a container that exits successfully", func() { </del> <ins> Context("when starting a container that exits", func() { </ins> It("it should run with the expected status [Conformance]", func() { <ins> restartCountVolumeName := "restart-count" restartCountVolumePath := "/restart-count" </ins> testContainer := api.Container{ Image: ImageRegistry[busyBoxImage], VolumeMounts: []api.VolumeMount{ { <del> MountPath: "/restart-count", Name:      "restart-count", </del> <ins> MountPath: restartCountVolumePath, Name:      restartCountVolumeName, </ins> }, }, <del> ImagePullPolicy: api.PullIfNotPresent, </del> } testVolumes := []api.Volume{ { <del> Name: "restart-count", </del> <ins> Name: restartCountVolumeName, </ins> VolumeSource: api.VolumeSource{ HostPath: &api.HostPathVolumeSource{ Path: os.TempDir(),


}, }, } <del> testCount := int32(3) testStatuses := []testStatus{ {"terminate-cmd-rpa", api.RestartPolicyAlways, api.PodRunning, ContainerStateWaiting | ContainerStateRunning | ContainerStateTerminated, ">", testCount, false}, {"terminate-cmd-rpof", api.RestartPolicyOnFailure, api.PodSucceeded, ContainerStateTerminated, "==", testCount, false}, {"terminate-cmd-rpn", api.RestartPolicyNever, api.PodSucceeded, ContainerStateTerminated, "==", 0, false}, </del> <ins> testCases := []testCase{ {"terminate-cmd-rpa", api.RestartPolicyAlways, api.PodRunning, ContainerStateRunning, "==", 2, true}, {"terminate-cmd-rpof", api.RestartPolicyOnFailure, api.PodSucceeded, ContainerStateTerminated, "==", 1, false}, {"terminate-cmd-rpn", api.RestartPolicyNever, api.PodFailed, ContainerStateTerminated, "==", 0, false}, </ins> } <del> for _, testStatus := range testStatuses { </del> <ins> for _, testCase := range testCases { </ins> tmpFile, err := ioutil.TempFile("", "restartCount") Expect(err).NotTo(HaveOccurred()) defer os.Remove(tmpFile.Name()) <del> // It fails in the first three runs and succeeds after that. tmpCmd := fmt.Sprintf("echo 'hello' >> /restart-count/%s ; test $(wc -l /restart-count/%s| awk {'print $1'}) -ge %d", path.Base(tmpFile.Name()), path.Base(tmpFile.Name()), testCount+1) testContainer.Name = testStatus.Name </del> <ins> // It failed at the 1st run, then succeeded at 2nd run, then run forever cmdScripts := ` f=%s count=$(echo 'hello' >> $f ; wc -l $f | awk {'print $1'}) if [ $count -eq 1 ]; then exit 1 fi if [ $count -eq 2 ]; then exit 0 fi while true; do sleep 1; done ` tmpCmd := fmt.Sprintf(cmdScripts, path.Join(restartCountVolumePath, path.Base(tmpFile.Name()))) testContainer.Name = testCase.Name </ins> testContainer.Command = []string{"sh", "-c", tmpCmd} terminateContainer := ConformanceContainer{ Container:     testContainer, Client:        cl, <del> RestartPolicy: testStatus.RestartPolicy, </del> <ins> RestartPolicy: testCase.RestartPolicy, </ins> Volumes:       testVolumes, NodeName:      *nodeName, Namespace:     namespace, } <del> err = terminateContainer.Create() Expect(err).NotTo(HaveOccurred()) testCContainers = append(testCContainers, terminateContainer) </del> <ins> Expect(terminateContainer.Create()).To(Succeed()) defer terminateContainer.Delete() </ins> <del> Eventually(func() api.PodPhase { _, phase, _ := terminateContainer.GetStatus() return phase }, retryTimeout, pollInterval).ShouldNot(Equal(api.PodPending)) var status api.ContainerStatus </del> By("it should get the expected 'RestartCount'") <del> Eventually(func() int32 { status, _, _ = terminateContainer.GetStatus() return status.RestartCount }, retryTimeout, pollInterval).Should(BeNumerically(testStatus.RestartCountOper, testStatus.RestartCount)) </del> <ins> Eventually(func() (int32, error) { status, err := terminateContainer.GetStatus() return status.RestartCount, err }, retryTimeout, pollInterval).Should(BeNumerically(testCase.RestartCountOper, testCase.RestartCount)) </ins> <del> By("it should get the expected 'Ready' status") Expect(status.Ready).To(Equal(testStatus.Ready)) </del> <ins> By("it should get the expected 'Phase'") Eventually(terminateContainer.GetPhase, retryTimeout, pollInterval).Should(Equal(testCase.Phase)) </ins> <del> By("it should get the expected 'State'") Expect(GetContainerState(status.State) & testStatus.State).NotTo(Equal(0)) </del> <ins> By("it should get the expected 'Ready' condition") Expect(terminateContainer.IsReady()).Should(Equal(testCase.Ready)) </ins> <del> By("it should be possible to delete [Conformance]") err = terminateContainer.Delete() Expect(err).NotTo(HaveOccurred()) Eventually(func() bool { isPresent, err := terminateContainer.Present() return err == nil && !isPresent }, retryTimeout, pollInterval).Should(BeTrue()) } }) }) Context("when start a container that keeps running", func() { It("it should run with the expected status [Conformance]", func() { testContainer := api.Container{ Image:           ImageRegistry[busyBoxImage], Command:         []string{"sh", "-c", "while true; do echo hello; sleep 1; done"}, ImagePullPolicy: api.PullIfNotPresent, } testStatuses := []testStatus{ {"loop-cmd-rpa", api.RestartPolicyAlways, api.PodRunning, ContainerStateRunning, "==", 0, true}, {"loop-cmd-rpof", api.RestartPolicyOnFailure, api.PodRunning, ContainerStateRunning, "==", 0, true}, {"loop-cmd-rpn", api.RestartPolicyNever, api.PodRunning, ContainerStateRunning, "==", 0, true}, } for _, testStatus := range testStatuses { testContainer.Name = testStatus.Name runningContainer := ConformanceContainer{ Container:     testContainer, Client:        cl, RestartPolicy: testStatus.RestartPolicy, NodeName:      *nodeName, Namespace:     namespace, } err := runningContainer.Create() Expect(err).NotTo(HaveOccurred()) testCContainers = append(testCContainers, runningContainer) Eventually(func() api.PodPhase { _, phase, _ := runningContainer.GetStatus() return phase }, retryTimeout, pollInterval).Should(Equal(api.PodRunning)) var status api.ContainerStatus var phase api.PodPhase Consistently(func() api.PodPhase { status, phase, err = runningContainer.GetStatus() return phase }, consistentCheckTimeout, pollInterval).Should(Equal(testStatus.Phase)) Expect(err).NotTo(HaveOccurred()) By("it should get the expected 'RestartCount'") Expect(status.RestartCount).To(BeNumerically(testStatus.RestartCountOper, testStatus.RestartCount)) By("it should get the expected 'Ready' status") Expect(status.Ready).To(Equal(testStatus.Ready)) </del> <ins> status, err := terminateContainer.GetStatus() Expect(err).ShouldNot(HaveOccurred()) </ins> By("it should get the expected 'State'") <del> Expect(GetContainerState(status.State) & testStatus.State).NotTo(Equal(0)) </del> <ins> Expect(GetContainerState(status.State)).To(Equal(testCase.State)) </ins> By("it should be possible to delete [Conformance]") <del> err = runningContainer.Delete() Expect(err).NotTo(HaveOccurred()) Eventually(func() bool { isPresent, err := runningContainer.Present() return err == nil && !isPresent }, retryTimeout, pollInterval).Should(BeTrue()) } }) }) Context("when start a container that exits failure", func() { It("it should run with the expected status [Conformance]", func() { testContainer := api.Container{ Image:           ImageRegistry[busyBoxImage], Command:         []string{"false"}, ImagePullPolicy: api.PullIfNotPresent, } testStatuses := []testStatus{ {"fail-cmd-rpa", api.RestartPolicyAlways, api.PodRunning, ContainerStateWaiting | ContainerStateRunning | ContainerStateTerminated, ">", 0, false}, {"fail-cmd-rpof", api.RestartPolicyOnFailure, api.PodRunning, ContainerStateTerminated, ">", 0, false}, {"fail-cmd-rpn", api.RestartPolicyNever, api.PodFailed, ContainerStateTerminated, "==", 0, false}, } for _, testStatus := range testStatuses { testContainer.Name = testStatus.Name failureContainer := ConformanceContainer{ Container:     testContainer, Client:        cl, RestartPolicy: testStatus.RestartPolicy, NodeName:      *nodeName, Namespace:     namespace, } err := failureContainer.Create() Expect(err).NotTo(HaveOccurred()) testCContainers = append(testCContainers, failureContainer) Eventually(func() api.PodPhase { _, phase, _ := failureContainer.GetStatus() return phase }, retryTimeout, pollInterval).ShouldNot(Equal(api.PodPending)) var status api.ContainerStatus By("it should get the expected 'RestartCount'") Eventually(func() int32 { status, _, _ = failureContainer.GetStatus() return status.RestartCount }, retryTimeout, pollInterval).Should(BeNumerically(testStatus.RestartCountOper, testStatus.RestartCount)) By("it should get the expected 'Ready' status") Expect(status.Ready).To(Equal(testStatus.Ready)) By("it should get the expected 'State'") Expect(GetContainerState(status.State) & testStatus.State).NotTo(Equal(0)) By("it should be possible to delete [Conformance]") err = failureContainer.Delete() Expect(err).NotTo(HaveOccurred()) Eventually(func() bool { isPresent, err := failureContainer.Present() return err == nil && !isPresent }, retryTimeout, pollInterval).Should(BeTrue()) </del> <ins> Expect(terminateContainer.Delete()).To(Succeed()) Eventually(terminateContainer.Present, retryTimeout, pollInterval).Should(BeFalse()) </ins> } }) })


Context("when running a container with invalid image", func() { It("it should run with the expected status [Conformance]", func() { testContainer := api.Container{ <del> Image:           "foo.com/foo/foo", Command:         []string{"false"}, ImagePullPolicy: api.PullIfNotPresent, </del> <ins> Image:   "foo.com/foo/foo", Command: []string{"false"}, </ins> } <del> testStatus := testStatus{"invalid-image-rpa", api.RestartPolicyAlways, api.PodPending, ContainerStateWaiting, "==", 0, false} testContainer.Name = testStatus.Name </del> <ins> testCase := testCase{"invalid-image-rpa", api.RestartPolicyAlways, api.PodPending, ContainerStateWaiting, "==", 0, false} testContainer.Name = testCase.Name </ins> invalidImageContainer := ConformanceContainer{ Container:     testContainer, Client:        cl, <del> RestartPolicy: testStatus.RestartPolicy, </del> <ins> RestartPolicy: testCase.RestartPolicy, </ins> NodeName:      *nodeName, Namespace:     namespace, } <del> err := invalidImageContainer.Create() Expect(err).NotTo(HaveOccurred()) testCContainers = append(testCContainers, invalidImageContainer) </del> <ins> Expect(invalidImageContainer.Create()).To(Succeed()) defer invalidImageContainer.Delete() </ins> <del> var status api.ContainerStatus var phase api.PodPhase </del> <ins> Eventually(invalidImageContainer.GetPhase, retryTimeout, pollInterval).Should(Equal(testCase.Phase)) Consistently(invalidImageContainer.GetPhase, consistentCheckTimeout, pollInterval).Should(Equal(testCase.Phase)) </ins> <del> Consistently(func() api.PodPhase { if status, phase, err = invalidImageContainer.GetStatus(); err != nil { return api.PodPending } else { return phase } }, consistentCheckTimeout, pollInterval).Should(Equal(testStatus.Phase)) </del> <ins> status, err := invalidImageContainer.GetStatus() </ins> Expect(err).NotTo(HaveOccurred()) By("it should get the expected 'RestartCount'") <del> Expect(status.RestartCount).To(BeNumerically(testStatus.RestartCountOper, testStatus.RestartCount)) </del> <ins> Expect(status.RestartCount).To(BeNumerically(testCase.RestartCountOper, testCase.RestartCount)) </ins> By("it should get the expected 'Ready' status") <del> Expect(status.Ready).To(Equal(testStatus.Ready)) </del> <ins> Expect(status.Ready).To(Equal(testCase.Ready)) </ins> By("it should get the expected 'State'") <del> Expect(GetContainerState(status.State) & testStatus.State).NotTo(Equal(0)) </del> <ins> Expect(GetContainerState(status.State)).To(Equal(testCase.State)) </ins> By("it should be possible to delete [Conformance]") <del> err = invalidImageContainer.Delete() Expect(err).NotTo(HaveOccurred()) Eventually(func() bool { isPresent, err := invalidImageContainer.Present() return err == nil && !isPresent }, retryTimeout, pollInterval).Should(BeTrue()) </del> <ins> Expect(invalidImageContainer.Delete()).To(Succeed()) Eventually(invalidImageContainer.Present, retryTimeout, pollInterval).Should(BeFalse()) </ins> }) }) <del> AfterEach(func() { for _, cc := range testCContainers { cc.Delete() } }) </del> }) })


import ( "fmt" "io" <del> "sort" "strconv" </del> "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/meta"


clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/runtime" deploymentutil "k8s.io/kubernetes/pkg/util/deployment" <del> "k8s.io/kubernetes/pkg/util/errors" </del> <ins> sliceutil "k8s.io/kubernetes/pkg/util/slice" </ins> ) const (


return fmt.Sprintf("No rollout history found in %s %q", resource, name), nil } // Sort the revisionToChangeCause map by revision <del> var revisions []string for k := range historyInfo.RevisionToTemplate { revisions = append(revisions, strconv.FormatInt(k, 10)) </del> <ins> var revisions []int64 for r := range historyInfo.RevisionToTemplate { revisions = append(revisions, r) </ins> } <del> sort.Strings(revisions) </del> <ins> sliceutil.SortInts64(revisions) </ins> return tabbedString(func(out io.Writer) error { fmt.Fprintf(out, "%s %q:n", resource, name) fmt.Fprintf(out, "REVISIONtCHANGE-CAUSEn") <del> errs := []error{} </del> for _, r := range revisions { // Find the change-cause of revision r <del> r64, err := strconv.ParseInt(r, 10, 64) if err != nil { errs = append(errs, err) continue } changeCause := historyInfo.RevisionToTemplate[r64].Annotations[ChangeCauseAnnotation] </del> <ins> changeCause := historyInfo.RevisionToTemplate[r].Annotations[ChangeCauseAnnotation] </ins> if len(changeCause) == 0 { changeCause = "<none>" } <del> fmt.Fprintf(out, "%st%sn", r, changeCause) </del> <ins> fmt.Fprintf(out, "%dt%sn", r, changeCause) </ins> } <del> return errors.NewAggregate(errs) </del> <ins> return nil </ins> }) }


} return shuffled } <ins> // Int64Slice attaches the methods of Interface to []int64, // sorting in increasing order. type Int64Slice []int64 func (p Int64Slice) Len() int           { return len(p) } func (p Int64Slice) Less(i, j int) bool { return p[i] < p[j] } func (p Int64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] } // Sorts []int64 in increasing order func SortInts64(a []int64) { sort.Sort(Int64Slice(a)) } </ins>


} } } <ins> func TestSortInts64(t *testing.T) { src := []int64{10, 1, 2, 3, 4, 5, 6} expected := []int64{1, 2, 3, 4, 5, 6, 10} SortInts64(src) if !reflect.DeepEqual(src, expected) { t.Errorf("func Ints64 didnt sort correctly, %v !- %v", src, expected) } } </ins>


numVolumes++ allErrs = append(allErrs, validateAzureFile(source.AzureFile, fldPath.Child("azureFile"))...) } <ins> if source.VsphereVolume != nil { if numVolumes > 0 { allErrs = append(allErrs, field.Forbidden(fldPath.Child("vsphereVolume"), "may not specify more than 1 volume type")) } else { numVolumes++ allErrs = append(allErrs, validateVsphereVolumeSource(source.VsphereVolume, fldPath.Child("vsphereVolume"))...) } } </ins> if numVolumes == 0 { allErrs = append(allErrs, field.Required(fldPath, "must specify a volume type")) }


return allErrs } <ins> func validateVsphereVolumeSource(cd *api.VsphereVirtualDiskVolumeSource, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} if len(cd.VolumePath) == 0 { allErrs = append(allErrs, field.Required(fldPath.Child("volumePath"), "")) } return allErrs } </ins> // ValidatePersistentVolumeName checks that a name is appropriate for a // PersistentVolumeName object. var ValidatePersistentVolumeName = NameIsDNSSubdomain


numVolumes++ allErrs = append(allErrs, validateAzureFile(pv.Spec.AzureFile, specPath.Child("azureFile"))...) } <ins> if pv.Spec.VsphereVolume != nil { if numVolumes > 0 { allErrs = append(allErrs, field.Forbidden(specPath.Child("vsphereVolume"), "may not specify more than 1 volume type")) } else { numVolumes++ allErrs = append(allErrs, validateVsphereVolumeSource(pv.Spec.VsphereVolume, specPath.Child("vsphereVolume"))...) } } </ins> if numVolumes == 0 { allErrs = append(allErrs, field.Required(specPath, "must specify a volume type")) }


pvc := createPVC("fake-pvc", "5G", []api.PersistentVolumeAccessMode{api.ReadWriteOnce}) <del> w, _ := testClient.PersistentVolumes().Watch(api.ListOptions{}) </del> <ins> w, err := testClient.PersistentVolumes().Watch(api.ListOptions{}) if err != nil { t.Errorf("Failed to watch PersistentVolumes: %v", err) } </ins> defer w.Stop() <del> _, _ = testClient.PersistentVolumes().Create(pv) _, _ = testClient.PersistentVolumeClaims(api.NamespaceDefault).Create(pvc) </del> <ins> _, err = testClient.PersistentVolumes().Create(pv) if err != nil { t.Errorf("Failed to create PersistentVolume: %v", err) } _, err = testClient.PersistentVolumeClaims(api.NamespaceDefault).Create(pvc) if err != nil { t.Errorf("Failed to create PersistentVolumeClaim: %v", err) } </ins> // wait until the controller pairs the volume and claim waitForPersistentVolumePhase(w, api.VolumeBound)


// change the reclamation policy of the PV for the next test pv.Spec.PersistentVolumeReclaimPolicy = api.PersistentVolumeReclaimDelete <del> w, _ = testClient.PersistentVolumes().Watch(api.ListOptions{}) </del> <ins> w, err = testClient.PersistentVolumes().Watch(api.ListOptions{}) if err != nil { t.Errorf("Failed to watch PersistentVolumes: %v", err) } </ins> defer w.Stop() <del> _, _ = testClient.PersistentVolumes().Create(pv) _, _ = testClient.PersistentVolumeClaims(api.NamespaceDefault).Create(pvc) </del> <ins> _, err = testClient.PersistentVolumes().Create(pv) if err != nil { t.Errorf("Failed to create PersistentVolume: %v", err) } _, err = testClient.PersistentVolumeClaims(api.NamespaceDefault).Create(pvc) if err != nil { t.Errorf("Failed to create PersistentVolumeClaim: %v", err) } </ins> waitForPersistentVolumePhase(w, api.VolumeBound)


pvc := createPVC("pvc-2", strconv.Itoa(maxPVs/2)+"G", []api.PersistentVolumeAccessMode{api.ReadWriteOnce}) <del> w, _ := testClient.PersistentVolumes().Watch(api.ListOptions{}) </del> <ins> w, err := testClient.PersistentVolumes().Watch(api.ListOptions{}) if err != nil { t.Errorf("Failed to watch PersistentVolumes: %v", err) } </ins> defer w.Stop() for i := 0; i < maxPVs; i++ { <del> _, _ = testClient.PersistentVolumes().Create(pvs[i]) </del> <ins> _, err = testClient.PersistentVolumes().Create(pvs[i]) if err != nil { t.Errorf("Failed to create PersistentVolume %d: %v", i, err) } </ins> } <del> _, _ = testClient.PersistentVolumeClaims(api.NamespaceDefault).Create(pvc) </del> <ins> _, err = testClient.PersistentVolumeClaims(api.NamespaceDefault).Create(pvc) if err != nil { t.Errorf("Failed to create PersistentVolumeClaim: %v", err) } </ins> // wait until the controller pairs the volume and claim waitForPersistentVolumePhase(w, api.VolumeBound)


pvc := createPVC("pvc-rwm", "5G", []api.PersistentVolumeAccessMode{api.ReadWriteMany}) <del> w, _ := testClient.PersistentVolumes().Watch(api.ListOptions{}) </del> <ins> w, err := testClient.PersistentVolumes().Watch(api.ListOptions{}) if err != nil { t.Errorf("Failed to watch PersistentVolumes: %v", err) } </ins> defer w.Stop() <del> _, _ = testClient.PersistentVolumes().Create(pv_rwm) _, _ = testClient.PersistentVolumes().Create(pv_rwo) </del> <ins> _, err = testClient.PersistentVolumes().Create(pv_rwm) if err != nil { t.Errorf("Failed to create PersistentVolume: %v", err) } _, err = testClient.PersistentVolumes().Create(pv_rwo) if err != nil { t.Errorf("Failed to create PersistentVolume: %v", err) } </ins> <del> _, _ = testClient.PersistentVolumeClaims(api.NamespaceDefault).Create(pvc) </del> <ins> _, err = testClient.PersistentVolumeClaims(api.NamespaceDefault).Create(pvc) if err != nil { t.Errorf("Failed to create PersistentVolumeClaim: %v", err) } </ins> // wait until the controller pairs the volume and claim waitForPersistentVolumePhase(w, api.VolumeBound)


ETCD_HOST=${ETCD_HOST:-127.0.0.1} ETCD_PORT=${ETCD_PORT:-4001} <ins> ETCD_PREFIX=${ETCD_PREFIX:-randomPrefix} </ins> API_PORT=${API_PORT:-8080} API_HOST=${API_HOST:-127.0.0.1} KUBE_API_VERSIONS=""


--bind-address="${API_HOST}"  --insecure-port="${API_PORT}"  --etcd-servers="http://${ETCD_HOST}:${ETCD_PORT}"  <ins> --etcd-prefix="${ETCD_PREFIX}"  </ins> --runtime-config="${RUNTIME_CONFIG}"  --cert-dir="${TMPDIR:-/tmp/}"  --service-cluster-ip-range="10.0.0.0/24"


### END TEST DEFINITION CUSTOMIZATION ### ####################################################### # Step 1: Start a server which supports both the old and new api versions, # but KUBE_OLD_API_VERSION is the latest (storage) version.


old_storage_version=${test_data[4]} kube::log::status "Verifying ${resource}/${namespace}/${name} has storage version ${old_storage_version} in etcd" <del> curl -s http://${ETCD_HOST}:${ETCD_PORT}/v2/keys/registry/${resource}/${namespace}/${name} | grep ${old_storage_version} </del> <ins> curl -s http://${ETCD_HOST}:${ETCD_PORT}/v2/keys/${ETCD_PREFIX}/${resource}/${namespace}/${name} | grep ${old_storage_version} </ins> done killApiServer


new_storage_version=${test_data[5]} kube::log::status "Verifying ${resource}/${namespace}/${name} has updated storage version ${new_storage_version} in etcd" <del> curl -s http://${ETCD_HOST}:${ETCD_PORT}/v2/keys/registry/${resource}/${namespace}/${name} | grep ${new_storage_version} </del> <ins> curl -s http://${ETCD_HOST}:${ETCD_PORT}/v2/keys/${ETCD_PREFIX}/${resource}/${namespace}/${name} | grep ${new_storage_version} </ins> done killApiServer


const ( SchedulerAnnotationKey = "scheduler.alpha.kubernetes.io/name" <ins> initialGetBackoff      = 100 * time.Millisecond maximalGetBackoff      = time.Minute </ins> ) // ConfigFactory knows how to fill out a scheduler config with its support functions.


} // Get the pod again; it may have changed/been scheduled already. pod = &api.Pod{} <del> err := factory.Client.Get().Namespace(podID.Namespace).Resource("pods").Name(podID.Name).Do().Into(pod) if err != nil { if !errors.IsNotFound(err) { glog.Errorf("Error getting pod %v for retry: %v; abandoning", podID, err) </del> <ins> getBackoff := initialGetBackoff for { if err := factory.Client.Get().Namespace(podID.Namespace).Resource("pods").Name(podID.Name).Do().Into(pod); err == nil { break </ins> } <del> return </del> <ins> if errors.IsNotFound(err) { glog.Warning("A pod %v no longer exists", podID) return } glog.Errorf("Error getting pod %v for retry: %v; retrying...", podID, err) if getBackoff = getBackoff * 2; getBackoff > maximalGetBackoff { getBackoff = maximalGetBackoff } time.Sleep(getBackoff) </ins> } if pod.Spec.NodeName == "" { podQueue.AddIfNotPresent(pod)


if len(list.Items) > 1 { return cmdutil.UsageError(cmd, "%s specifies multiple items", filename) } <ins> if len(list.Items) == 0 { return cmdutil.UsageError(cmd, "please make sure %s exists and is not empty", filename) } </ins> obj = list.Items[0] } newRc, ok = obj.(*api.ReplicationController)


apiVersion: v1 kind: ReplicationController metadata: <del> name: kube-dns-v18 </del> <ins> name: kube-dns-v19 </ins> namespace: kube-system labels: k8s-app: kube-dns <del> version: v18 </del> <ins> version: v19 </ins> kubernetes.io/cluster-service: "true" spec: replicas: __PILLAR__DNS__REPLICAS__ selector: k8s-app: kube-dns <del> version: v18 </del> <ins> version: v19 </ins> template: metadata: labels: k8s-app: kube-dns <del> version: v18 </del> <ins> version: v19 </ins> kubernetes.io/cluster-service: "true" spec: containers:


# keep request = limit to keep this container in guaranteed class limits: cpu: 10m <del> memory: 20Mi </del> <ins> memory: 50Mi </ins> requests: cpu: 10m <del> memory: 20Mi </del> <ins> # Note that this container shouldn't really need 50Mi of memory. The # limits are set higher than expected pending investigation on #29688. # The extra memory was stolen from the kubedns container to keep the # net memory requested by the pod constant. memory: 50Mi </ins> args: - -cmd=nslookup kubernetes.default.svc.__PILLAR__DNS__DOMAIN__ 127.0.0.1 >/dev/null && nslookup kubernetes.default.svc.__PILLAR__DNS__DOMAIN__ 127.0.0.1:10053 >/dev/null - -port=8080


apiVersion: v1 kind: ReplicationController metadata: <del> name: kube-dns-v18 </del> <ins> name: kube-dns-v19 </ins> namespace: kube-system labels: k8s-app: kube-dns <del> version: v18 </del> <ins> version: v19 </ins> kubernetes.io/cluster-service: "true" spec: replicas: {{ pillar['dns_replicas'] }} selector: k8s-app: kube-dns <del> version: v18 </del> <ins> version: v19 </ins> template: metadata: labels: k8s-app: kube-dns <del> version: v18 </del> <ins> version: v19 </ins> kubernetes.io/cluster-service: "true" spec: containers:


# keep request = limit to keep this container in guaranteed class limits: cpu: 10m <del> memory: 20Mi </del> <ins> memory: 50Mi </ins> requests: cpu: 10m <del> memory: 20Mi </del> <ins> # Note that this container shouldn't really need 50Mi of memory. The # limits are set higher than expected pending investigation on #29688. # The extra memory was stolen from the kubedns container to keep the # net memory requested by the pod constant. memory: 50Mi </ins> args: - -cmd=nslookup kubernetes.default.svc.{{ pillar['dns_domain'] }} 127.0.0.1 >/dev/null && nslookup kubernetes.default.svc.{{ pillar['dns_domain'] }} 127.0.0.1:10053 >/dev/null - -port=8080


apiVersion: v1 kind: ReplicationController metadata: <del> name: kube-dns-v18 </del> <ins> name: kube-dns-v19 </ins> namespace: kube-system labels: k8s-app: kube-dns <del> version: v18 </del> <ins> version: v19 </ins> kubernetes.io/cluster-service: "true" spec: replicas: $DNS_REPLICAS selector: k8s-app: kube-dns <del> version: v18 </del> <ins> version: v19 </ins> template: metadata: labels: k8s-app: kube-dns <del> version: v18 </del> <ins> version: v19 </ins> kubernetes.io/cluster-service: "true" spec: containers:


# "burstable" category so the kubelet doesn't backoff from restarting it. limits: cpu: 100m <del> memory: 200Mi </del> <ins> memory: 170Mi </ins> requests: cpu: 100m <del> memory: 100Mi </del> <ins> memory: 70Mi </ins> livenessProbe: httpGet: path: /healthz


# keep request = limit to keep this container in guaranteed class limits: cpu: 10m <del> memory: 20Mi </del> <ins> memory: 50Mi </ins> requests: cpu: 10m <del> memory: 20Mi </del> <ins> # Note that this container shouldn't really need 50Mi of memory. The # limits are set higher than expected pending investigation on #29688. # The extra memory was stolen from the kubedns container to keep the # net memory requested by the pod constant. memory: 50Mi </ins> args: - -cmd=nslookup kubernetes.default.svc.$DNS_DOMAIN 127.0.0.1 >/dev/null && nslookup kubernetes.default.svc.$DNS_DOMAIN 127.0.0.1:10053 >/dev/null - -port=8080


apiVersion: v1 kind: ReplicationController metadata: <del> name: kube-dns-v17 </del> <ins> name: kube-dns-v17.1 </ins> namespace: kube-system labels: k8s-app: kube-dns <del> version: v17 </del> <ins> version: v17.1 </ins> kubernetes.io/cluster-service: "true" spec: replicas: __PILLAR__DNS__REPLICAS__ selector: k8s-app: kube-dns <del> version: v17 </del> <ins> version: v17.1 </ins> template: metadata: labels: k8s-app: kube-dns <del> version: v17 </del> <ins> version: v17.1 </ins> kubernetes.io/cluster-service: "true" spec: containers:


# keep request = limit to keep this container in guaranteed class limits: cpu: 10m <del> memory: 20Mi </del> <ins> memory: 50Mi </ins> requests: cpu: 10m <del> memory: 20Mi </del> <ins> # Note that this container shouldn't really need 50Mi of memory. The # limits are set higher than expected pending investigation on #29688. # The extra memory was stolen from the kubedns container to keep the # net memory requested by the pod constant. memory: 50Mi </ins> args: - -cmd=nslookup kubernetes.default.svc.__PILLAR__DNS__DOMAIN__ 127.0.0.1 >/dev/null - -port=8080


apiVersion: v1 kind: ReplicationController metadata: <del> name: kube-dns-v17 </del> <ins> name: kube-dns-v17.1 </ins> namespace: kube-system labels: k8s-app: kube-dns <del> version: v17 </del> <ins> version: v17.1 </ins> kubernetes.io/cluster-service: "true" spec: replicas: {{ pillar['dns_replicas'] }} selector: k8s-app: kube-dns <del> version: v17 </del> <ins> version: v17.1 </ins> template: metadata: labels: k8s-app: kube-dns <del> version: v17 </del> <ins> version: v17.1 </ins> kubernetes.io/cluster-service: "true" spec: containers:


# keep request = limit to keep this container in guaranteed class limits: cpu: 10m <del> memory: 20Mi </del> <ins> memory: 50Mi </ins> requests: cpu: 10m <del> memory: 20Mi </del> <ins> # Note that this container shouldn't really need 50Mi of memory. The # limits are set higher than expected pending investigation on #29688. # The extra memory was stolen from the kubedns container to keep the # net memory requested by the pod constant. memory: 50Mi </ins> args: - -cmd=nslookup kubernetes.default.svc.{{ pillar['dns_domain'] }} 127.0.0.1 >/dev/null - -port=8080


apiVersion: v1 kind: ReplicationController metadata: <del> name: kube-dns-v17 </del> <ins> name: kube-dns-v17.1 </ins> namespace: kube-system labels: k8s-app: kube-dns <del> version: v17 </del> <ins> version: v17.1 </ins> kubernetes.io/cluster-service: "true" spec: replicas: $DNS_REPLICAS selector: k8s-app: kube-dns <del> version: v17 </del> <ins> version: v17.1 </ins> template: metadata: labels: k8s-app: kube-dns <del> version: v17 </del> <ins> version: v17.1 </ins> kubernetes.io/cluster-service: "true" spec: containers:


# keep request = limit to keep this container in guaranteed class limits: cpu: 10m <del> memory: 20Mi </del> <ins> memory: 50Mi </ins> requests: cpu: 10m <del> memory: 20Mi </del> <ins> # Note that this container shouldn't really need 50Mi of memory. The # limits are set higher than expected pending investigation on #29688. # The extra memory was stolen from the kubedns container to keep the # net memory requested by the pod constant. memory: 50Mi </ins> args: - -cmd=nslookup kubernetes.default.svc.$DNS_DOMAIN 127.0.0.1 >/dev/null - -port=8080


// test when the pod anti affinity rule is not satisfied, the pod would stay pending. It("validates that InterPodAntiAffinity is respected if matching 2", func() { <del> // launch a pod to find a node which can launch a pod. We intentionally do // not just take the node list and choose the first of them. Depending on the // cluster and the scheduler it might be that a "normal" pod cannot be // scheduled onto it. By("Trying to launch a pod with a label to get a node which can launch it.") </del> <ins> // launch pods to find nodes which can launch a pod. We intentionally do // not just take the node list and choose the first and the second of them. // Depending on the cluster and the scheduler it might be that a "normal" pod // cannot be scheduled onto it. By("Launching two pods on two distinct nodes to get two node names") CreateHostPortPods(f, "host-port", 2, true) defer framework.DeleteRC(f.Client, f.Namespace.Name, "host-port") podList, err := c.Pods(ns).List(api.ListOptions{}) ExpectNoError(err) Expect(len(podList.Items)).To(Equal(2)) nodeNames := []string{podList.Items[0].Spec.NodeName, podList.Items[1].Spec.NodeName} Expect(nodeNames[0]).ToNot(Equal(nodeNames[1])) By("Applying a random label to both nodes.") k := "e2e.inter-pod-affinity.kubernetes.io/zone" v := "china-e2etest" for _, nodeName := range nodeNames { framework.AddOrUpdateLabelOnNode(c, nodeName, k, v) framework.ExpectNodeHasLabel(c, nodeName, k, v) defer framework.RemoveLabelOffNode(c, nodeName, k) } By("Trying to launch another pod on the first node with the service label.") </ins> podName := "with-label-" + string(uuid.NewUUID()) pod, err := c.Pods(ns).Create(&api.Pod{ TypeMeta: unversioned.TypeMeta{


Image: framework.GetPauseImageName(f.Client), }, }, <ins> NodeSelector: map[string]string{k: v}, // only launch on our two nodes </ins> }, }) framework.ExpectNoError(err)


pod, err = c.Pods(ns).Get(podName) framework.ExpectNoError(err) <del> nodeName := pod.Spec.NodeName By("Trying to apply a random label on the found node.") k := "e2e.inter-pod-affinity.kubernetes.io/zone" v := "china-e2etest" framework.AddOrUpdateLabelOnNode(c, nodeName, k, v) framework.ExpectNodeHasLabel(c, nodeName, k, v) defer framework.RemoveLabelOffNode(c, nodeName, k) By("Trying to launch the pod, now with podAffinity with same Labels.") labelPodName := "with-podaffinity-" + string(uuid.NewUUID()) </del> <ins> By("Trying to launch another pod, now with podAntiAffinity with same Labels.") labelPodName := "with-podantiaffinity-" + string(uuid.NewUUID()) </ins> _, err = c.Pods(ns).Create(&api.Pod{ TypeMeta: unversioned.TypeMeta{ Kind: "Pod",


Image: framework.GetPauseImageName(f.Client), }, }, <ins> NodeSelector: map[string]string{k: v}, // only launch on our two nodes, contradicting the podAntiAffinity </ins> }, }) framework.ExpectNoError(err)


framework.Logf("Sleeping 10 seconds and crossing our fingers that scheduler will run in that time.") time.Sleep(10 * time.Second) <del> verifyResult(c, labelPodName, 2, 0, ns) </del> <ins> verifyResult(c, labelPodName, 3, 1, ns) </ins> }) // test the pod affinity successful matching scenario with multiple Label Operators.


Expect(err).NotTo(HaveOccurred()) }() <del> // Waiting for service to expose endpoint </del> <ins> // Waiting for service to expose endpoint. </ins> validateEndpointsOrFail(c, ns, serviceName, PortsByPodName{serverPodName: {servicePort}}) By("Retrieve sourceip from a pod on the same node")


timeout := 2 * time.Minute framework.Logf("Waiting up to %v for sourceIp test to be executed", timeout) cmd := fmt.Sprintf(`wget -T 30 -qO- %s:%d | grep client_address`, serviceIp, servicePort) <del> // need timeout mechanism because it may takes more times for iptables to be populated </del> <ins> // Need timeout mechanism because it may takes more times for iptables to be populated. </ins> for start := time.Now(); time.Since(start) < timeout; time.Sleep(2) { stdout, err = framework.RunHostCmd(execPod.Namespace, execPod.Name, cmd) if err != nil { framework.Logf("got err: %v, retry until timeout", err) continue } <ins> // Need to check output because wget -q might omit the error. if strings.TrimSpace(stdout) == "" { framework.Logf("got empty stdout, retry until timeout") continue } </ins> break } ExpectNoError(err) <del> // the stdout return from RunHostCmd seems to come with "n", so TrimSpace is needed // desired stdout in this format: client_address=x.x.x.x </del> <ins> // The stdout return from RunHostCmd seems to come with "n", so TrimSpace is needed. // Desired stdout in this format: client_address=x.x.x.x </ins> outputs := strings.Split(strings.TrimSpace(stdout), "=") <ins> sourceIp := "" </ins> if len(outputs) != 2 { <del> // fail the test if output format is unexpected </del> <ins> // Fail the test if output format is unexpected. </ins> framework.Failf("exec pod returned unexpected stdout format: [%v]n", stdout) <del> return execPodIp, "" </del> <ins> } else { sourceIp = outputs[1] </ins> } <del> return execPodIp, outputs[1] </del> <ins> return execPodIp, sourceIp </ins> }


docker_opts+=" --log-level=warn" fi local use_net_plugin="true" <del> if [[ "${NETWORK_PROVIDER:-}" != "kubenet" && "${NETWORK_PROVIDER:-}" != "cni" ]]; then </del> <ins> if [[ "${NETWORK_PROVIDER:-}" == "kubenet" || "${NETWORK_PROVIDER:-}" == "cni" ]]; then # set docker0 cidr to private ip address range to avoid conflict with cbr0 cidr range docker_opts+=" --bip=169.254.123.1/24" else </ins> use_net_plugin="false" docker_opts+=" --bridge=cbr0" fi


WantedBy=multi-user.target EOF <del> # Delete docker0 to avoid interference </del> <ins> # Flush iptables nat table </ins> iptables -t nat -F || true <del> ip link set docker0 down || true brctl delbr docker0 || true </del> systemctl start kubelet.service }


"os" "k8s.io/kubernetes/pkg/healthz" <del> "k8s.io/kubernetes/pkg/util" </del> "k8s.io/kubernetes/pkg/util/flag" "k8s.io/kubernetes/pkg/util/logs" "k8s.io/kubernetes/pkg/version/verflag"


"github.com/spf13/pflag" "k8s.io/kubernetes/contrib/mesos/pkg/executor/service" "k8s.io/kubernetes/contrib/mesos/pkg/hyperkube" <del> "k8s.io/kubernetes/pkg/util" </del> "k8s.io/kubernetes/pkg/util/flag" "k8s.io/kubernetes/pkg/util/logs" "k8s.io/kubernetes/pkg/version/verflag"


"github.com/spf13/pflag" "k8s.io/kubernetes/contrib/mesos/pkg/hyperkube" "k8s.io/kubernetes/contrib/mesos/pkg/scheduler/service" <del> "k8s.io/kubernetes/pkg/util" </del> "k8s.io/kubernetes/pkg/util/flag" "k8s.io/kubernetes/pkg/util/logs" "k8s.io/kubernetes/pkg/version/verflag"


livenessProbe: httpGet: path: /healthz <del> port: 8080 </del> <ins> port: 8082 </ins> scheme: HTTP initialDelaySeconds: 180 timeoutSeconds: 5


if len(address.Hostname) > 0 { allErrs = append(allErrs, ValidateDNS1123Label(address.Hostname, fldPath.Child("hostname"))...) } <del> // During endpoint update, validate NodeName is DNS1123 compliant and transition rules allow the update </del> <ins> // During endpoint update, verify that NodeName is a DNS subdomain and transition rules allow the update </ins> if address.NodeName != nil { <del> allErrs = append(allErrs, ValidateDNS1123Label(*address.NodeName, fldPath.Child("nodeName"))...) </del> <ins> for _, msg := range ValidateNodeName(*address.NodeName, false) { allErrs = append(allErrs, field.Invalid(fldPath.Child("nodeName"), *address.NodeName, msg)) } </ins> } allErrs = append(allErrs, validateEpAddrNodeNameTransition(address, ipToNodeName, fldPath.Child("nodeName"))...) if len(allErrs) > 0 {


} } <del> func TestEndpointAddressNodeNameInvalidDNS1123(t *testing.T) { </del> <ins> func TestEndpointAddressNodeNameInvalidDNSSubdomain(t *testing.T) { </ins> // Check NodeName DNS validation <del> endpoint := newNodeNameEndpoint("illegal.nodename") </del> <ins> endpoint := newNodeNameEndpoint("illegal*.nodename") </ins> errList := ValidateEndpoints(endpoint) if len(errList) == 0 { t.Error("Endpoint should reject invalid NodeName") } } <ins> func TestEndpointAddressNodeNameCanBeAnIPAddress(t *testing.T) { endpoint := newNodeNameEndpoint("10.10.1.1") errList := ValidateEndpoints(endpoint) if len(errList) != 0 { t.Error("Endpoint should accept a NodeName that is an IP address") } } </ins>


function usage() { echo "!!! EXPERIMENTAL !!!" echo "" <del> echo "${0} [-M|-N|-P] -l | <version number or publication>" </del> <ins> echo "${0} [-M|-N|-P] -l -o | <version number or publication>" </ins> echo "  Upgrades master and nodes by default" echo "  -M:  Upgrade master only" echo "  -N:  Upgrade nodes only" echo "  -P:  Node upgrade prerequisites only (create a new instance template)" <ins> echo "  -o:  Use os distro sepcified in KUBE_NODE_OS_DISTRIBUTION for new nodes" </ins> echo "  -l:  Use local(dev) binaries" echo "" echo '  Version number or publication is either a proper version number'


'http://metadata/computeMetadata/v1/instance/attributes/kube-env'" 2>/dev/null } <ins> # Read os distro information from /os/release on node. # $1: The name of node # # Assumed vars: #   PROJECT #   ZONE function get-node-os() { gcloud compute ssh "$1"  --project "${PROJECT}"  --zone "${ZONE}"  --command  "cat /etc/os-release | grep "^ID=.*" | cut -c 4-" } </ins> # Assumed vars: #   KUBE_VERSION #   NODE_SCOPES


#                 compatible way? write-node-env <ins> if [[ "${env_os_distro}" == "false" ]]; then NODE_OS_DISTRIBUTION=$(get-node-os "${NODE_NAMES[0]}") source "${KUBE_ROOT}/cluster/gce/${NODE_OS_DISTRIBUTION}/node-helper.sh" # Reset the node image based on current os distro set-node-image fi </ins> # TODO(zmerlynn): Get configure-vm script from ${version}. (Must plumb this #                 through all create-node-instance-template implementations). local template_name=$(get-template-name-from-version ${SANITIZED_VERSION})


node_upgrade=true node_prereqs=false local_binaries=false <ins> env_os_distro=false </ins> while getopts ":MNPlh" opt; do case ${opt} in


l) local_binaries=true ;; <ins> o) env_os_distro=true ;; </ins> h) usage exit 0


MASTER_IMAGE_PROJECT=${KUBE_GCE_MASTER_PROJECT:-google-containers} fi <del> if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then # If the node image is not set, we use the latest GCI image. # Otherwise, we respect whatever is set by the user. NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${GCI_VERSION}} NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-google-containers} elif [[ "${NODE_OS_DISTRIBUTION}" == "debian" ]]; then NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${CVM_VERSION}} NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-google-containers} fi </del> <ins> # Sets node image based on the specified os distro. Currently this function only # supports gci and debian. function set-node-image() { if [[ "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then # If the node image is not set, we use the latest GCI image. # Otherwise, we respect whatever is set by the user. NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${GCI_VERSION}} NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-google-containers} elif [[ "${NODE_OS_DISTRIBUTION}" == "debian" ]]; then NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-${CVM_VERSION}} NODE_IMAGE_PROJECT=${KUBE_GCE_NODE_PROJECT:-google-containers} fi } set-node-image </ins> # Verfiy cluster autoscaler configuration. if [[ "${ENABLE_CLUSTER_AUTOSCALER}" == "true" ]]; then


# Delete the master replica pd (possibly leaked by kube-up if master create failed). # TODO(jszczepkowski): remove also possibly leaked replicas' pds <del> local -r replica-pd="${REPLICA_NAME:-${MASTER_NAME}}-pd" if gcloud compute disks describe "${replica-pd}" --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then </del> <ins> local -r replica_pd="${REPLICA_NAME:-${MASTER_NAME}}-pd" if gcloud compute disks describe "${replica_pd}" --zone "${ZONE}" --project "${PROJECT}" &>/dev/null; then </ins> gcloud compute disks delete  --project "${PROJECT}"  --quiet  --zone "${ZONE}"  <del> "${replica-pd}" </del> <ins> "${replica_pd}" </ins> fi # Delete disk for cluster registry if enabled


<ins> # This Dockerfile will build an image that is configured # to run Fluentd with an Elasticsearch plug-in and the # provided configuration file. # TODO(satnam6502): Use a lighter base image, e.g. some form of busybox. # The image acts as an executable for the binary /usr/sbin/td-agent. # Note that fluentd is run with root permssion to allow access to # log files with root only access under /var/lib/docker/containers/* # Please see http://docs.fluentd.org/articles/install-by-deb for more # information about installing fluentd using deb package. FROM ubuntu:14.04 MAINTAINER Satnam Singh "satnam@google.com" # Ensure there are enough file descriptors for running Fluentd. RUN ulimit -n 65536 # Install prerequisites. RUN apt-get update &&  apt-get install -y curl &&  apt-get install -y -q libcurl4-openssl-dev make &&  apt-get clean # Install Fluentd. RUN /usr/bin/curl -L http://toolbelt.treasuredata.com/sh/install-ubuntu-trusty-td-agent2.sh | sh # Change the default user and group to root. # Needed to allow access to /var/log/docker/... files. RUN sed -i -e "s/USER=td-agent/USER=root/" -e "s/GROUP=td-agent/GROUP=root/" /etc/init.d/td-agent # Install the Elasticsearch Fluentd plug-in. RUN /usr/sbin/td-agent-gem install fluent-plugin-elasticsearch # Copy the Fluentd configuration file. COPY td-agent.conf /etc/td-agent/td-agent.conf # Copy a script that determines the name of the host machine # and then patch the Fluentd configuration files and then # run Fluentd in the foreground. ADD run.sh /run.sh # Always run the this setup script. ENTRYPOINT ["/run.sh"] </ins>


<ins> #!/bin/bash # Copyright 2014 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Build the fluentd-elasticsearch image and push # to google/fluentd-elasticsearch. sudo docker build -t kubernetes/fluentd-elasticsearch . sudo docker push kubernetes/fluentd-elasticsearch </ins>


<ins> #!/bin/bash # Copyright 2014 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # WARNING! HORRIBLE HACK! We expect /outerhost to be mapped to # the enclosing /etc/host file so we can determine the name of # the host machine (super fragile). This is a temporary hack until # service IPs are done. OUTER_HOST=`tail -n 1 /outerhost | awk '{print $3}'` # Copy the Fluentd config file and patch it to refer to the # name of the host machine for ES_HOST. HACK! cp td-agent.conf /etc/td-agent sed -i -e "s/ES_HOST/${OUTER_HOST}/" /etc/td-agent/td-agent.conf /usr/sbin/td-agent </ins>


<ins> # This configuration file for Fluentd / td-agent is used # to watch changes to Docker log files that live in the # directory /var/lib/docker/containers/ which are then submitted to # Elasticsearch (running on the machine ES_HOST:9200) which # assumes the installation of the fluentd-elasticsearch plug-in. # See https://github.com/uken/fluent-plugin-elasticsearch for # more information about the plug-in. This file needs to be # patched to replace ES_HOST with the name of the actual # machine running Elasticsearch. # Maintainer: Satnam Singh (satnam@google.com) # # Exampe # ====== # A line in the Docker log file might like like this JSON: # # {"log":"2014/09/25 21:15:03 Got request with path wombatn", #  "stream":"stderr", #   "time":"2014-09-25T21:15:03.499185026Z"} # # The time_format specification below makes sure we properly # parse the time format produced by Docker. This will be # submitted to Elasticsearch and should appear like: # $ curl 'http://elasticsearch:9200/_search?pretty' # ... # { #      "_index" : "logstash-2014.09.25", #      "_type" : "fluentd", #      "_id" : "VBrbor2QTuGpsQyTCdfzqA", #      "_score" : 1.0, #      "_source":{"log":"2014/09/25 22:45:50 Got request with path wombatn", #                 "stream":"stderr","tag":"docker.container.all", #                 "@timestamp":"2014-09-25T22:45:50+00:00"} #    }, # ... <source> type tail format json time_key time path /var/lib/docker/containers/*/*-json.log time_format %Y-%m-%dT%H:%M:%S tag docker.container.all </source> <match docker.container.*> type elasticsearch log_level info include_tag_key true host ES_HOST port 9200 logstash_format true flush_interval 5s </match> </ins>


source /run/flannel/subnet.env source /etc/default/docker <del> echo DOCKER_OPTS="${DOCKER_OPTS} -H tcp://127.0.0.1:4243 -H unix:///var/run/docker.sock  </del> <ins> echo DOCKER_OPTS=" -H tcp://127.0.0.1:4243 -H unix:///var/run/docker.sock  </ins> --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU}" > /etc/default/docker sudo service docker restart }


if len(tc.plugin) != 0 { c.AuthProvider = &clientcmdapi.AuthProviderConfig{Name: tc.plugin} } <del> tConfig, err := c.transportConfig() </del> <ins> tConfig, err := c.TransportConfig() </ins> if err != nil { // Unknown/bad plugins are expected to fail here. if !tc.expectErr {


// TLSConfigFor returns a tls.Config that will provide the transport level security defined // by the provided Config. Will return nil if no transport level security is requested. func TLSConfigFor(config *Config) (*tls.Config, error) { <del> cfg, err := config.transportConfig() </del> <ins> cfg, err := config.TransportConfig() </ins> if err != nil { return nil, err }


// or transport level security defined by the provided Config. Will return the // default http.DefaultTransport if no special case behavior is needed. func TransportFor(config *Config) (http.RoundTripper, error) { <del> cfg, err := config.transportConfig() </del> <ins> cfg, err := config.TransportConfig() </ins> if err != nil { return nil, err }


// the underlying connection (like WebSocket or HTTP2 clients). Pure HTTP clients should use // the higher level TransportFor or RESTClientFor methods. func HTTPWrappersForConfig(config *Config, rt http.RoundTripper) (http.RoundTripper, error) { <del> cfg, err := config.transportConfig() </del> <ins> cfg, err := config.TransportConfig() </ins> if err != nil { return nil, err } return transport.HTTPWrappersForConfig(cfg, rt) } <del> // transportConfig converts a client config to an appropriate transport config. func (c *Config) transportConfig() (*transport.Config, error) { </del> <ins> // TransportConfig converts a client config to an appropriate transport config. func (c *Config) TransportConfig() (*transport.Config, error) { </ins> wt := c.WrapTransport if c.AuthProvider != nil { provider, err := GetAuthProvider(c.Host, c.AuthProvider, c.AuthConfigPersister)


"fmt" "math" "math/rand" <ins> "net" "net/http" </ins> "os" "strconv" "sync"


"k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" unversionedcore "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/typed/core/unversioned" <ins> "k8s.io/kubernetes/pkg/client/restclient" "k8s.io/kubernetes/pkg/client/transport" </ins> client "k8s.io/kubernetes/pkg/client/unversioned" "k8s.io/kubernetes/pkg/labels" "k8s.io/kubernetes/pkg/util/intstr" <ins> utilnet "k8s.io/kubernetes/pkg/util/net" </ins> "k8s.io/kubernetes/test/e2e/framework" . "github.com/onsi/ginkgo"


namespaces = createNamespaces(f, nodeCount, itArg.podsPerNode) totalPods := itArg.podsPerNode * nodeCount <del> configs = generateRCConfigs(totalPods, itArg.image, itArg.command, c, namespaces) </del> <ins> configs = generateRCConfigs(totalPods, itArg.image, itArg.command, namespaces) </ins> var services []*api.Service // Read the environment variable to see if we want to create services createServices := os.Getenv("CREATE_SERVICES")


return namespaces } <ins> func createClients(numberOfClients int) ([]*client.Client, error) { clients := make([]*client.Client, numberOfClients) for i := 0; i < numberOfClients; i++ { config, err := framework.LoadConfig() Expect(err).NotTo(HaveOccurred()) config.QPS = 100 config.Burst = 200 if framework.TestContext.KubeAPIContentType != "" { config.ContentType = framework.TestContext.KubeAPIContentType } // For the purpose of this test, we want to force that clients // do not share underlying transport (which is a default behavior // in Kubernetes). Thus, we are explicitly creating transport for // each client here. transportConfig, err := config.TransportConfig() if err != nil { return nil, err } tlsConfig, err := transport.TLSConfigFor(transportConfig) if err != nil { return nil, err } config.Transport = utilnet.SetTransportDefaults(&http.Transport{ Proxy:               http.ProxyFromEnvironment, TLSHandshakeTimeout: 10 * time.Second, TLSClientConfig:     tlsConfig, MaxIdleConnsPerHost: 100, Dial: (&net.Dialer{ Timeout:   30 * time.Second, KeepAlive: 30 * time.Second, }).Dial, }) // Overwrite TLS-related fields from config to avoid collision with // Transport field. config.TLSClientConfig = restclient.TLSClientConfig{} c, err := client.New(config) if err != nil { return nil, err } clients[i] = c } return clients, nil } </ins> func computeRCCounts(total int) (int, int, int) { // Small RCs owns ~0.5 of total number of pods, medium and big RCs ~0.25 each. // For example for 3000 pods (100 nodes, 30 pods per node) there are:


return smallRCCount, mediumRCCount, bigRCCount } <del> func generateRCConfigs(totalPods int, image string, command []string, c *client.Client, nss []*api.Namespace) []*framework.RCConfig { </del> <ins> func generateRCConfigs(totalPods int, image string, command []string, nss []*api.Namespace) []*framework.RCConfig { </ins> configs := make([]*framework.RCConfig, 0) smallRCCount, mediumRCCount, bigRCCount := computeRCCounts(totalPods) <del> configs = append(configs, generateRCConfigsForGroup(c, nss, smallRCGroupName, smallRCSize, smallRCCount, image, command)...) configs = append(configs, generateRCConfigsForGroup(c, nss, mediumRCGroupName, mediumRCSize, mediumRCCount, image, command)...) configs = append(configs, generateRCConfigsForGroup(c, nss, bigRCGroupName, bigRCSize, bigRCCount, image, command)...) </del> <ins> configs = append(configs, generateRCConfigsForGroup(nss, smallRCGroupName, smallRCSize, smallRCCount, image, command)...) configs = append(configs, generateRCConfigsForGroup(nss, mediumRCGroupName, mediumRCSize, mediumRCCount, image, command)...) configs = append(configs, generateRCConfigsForGroup(nss, bigRCGroupName, bigRCSize, bigRCCount, image, command)...) // Create a number of clients to better simulate real usecase // where not everyone is using exactly the same client. rcsPerClient := 20 clients, err := createClients((len(configs) + rcsPerClient - 1) / rcsPerClient) framework.ExpectNoError(err) for i := 0; i < len(configs); i++ { configs[i].Client = clients[i%len(clients)] } </ins> return configs } <del> func generateRCConfigsForGroup(c *client.Client, nss []*api.Namespace, groupName string, size, count int, image string, command []string) []*framework.RCConfig { </del> <ins> func generateRCConfigsForGroup( nss []*api.Namespace, groupName string, size, count int, image string, command []string) []*framework.RCConfig { </ins> configs := make([]*framework.RCConfig, 0, count) for i := 1; i <= count; i++ { config := &framework.RCConfig{ <del> Client:     c, </del> <ins> Client:     nil, // this will be overwritten later </ins> Name:       groupName + "-" + strconv.Itoa(i), Namespace:  nss[i%len(nss)].Name, Timeout:    10 * time.Minute,


MASTER_CPU=1 NODE_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_NODES}})) <del> NODE_IP_RANGES="10.244.0.0/16" </del> <ins> NODE_IP_RANGES="10.244.0.0/16" # Min Prefix supported is 16 </ins> MASTER_IP_RANGE="${MASTER_IP_RANGE:-10.246.0.0/24}" NODE_MEMORY_MB=2048 NODE_CPU=1


# identify the subnet assigned to the node by the kubernetes controller manager. KUBE_NODE_BRIDGE_NETWORK=() for (( i=0; i<${#NODE_NAMES[@]}; i++)); do <del> printf " finding network of cbr0 bridge on node  ${NODE_NAMES[$i]}n" network=$(kube-ssh ${KUBE_NODE_IP_ADDRESSES[$i]} 'sudo ip route show | grep -E "dev cbr0" | cut -d     " " -f1') KUBE_NODE_BRIDGE_NETWORK+=("${network}") done </del> <ins> printf " finding network of cbr0 bridge on node  ${NODE_NAMES[$i]}n" network="" top2_octets_final=$(echo $NODE_IP_RANGES | awk -F "." '{ print $1 "." $2 }') # Assume that a 24 bit mask per node attempt=0 max_attempt=60 while true ; do attempt=$(($attempt+1)) network=$(kube-ssh ${KUBE_NODE_IP_ADDRESSES[$i]} 'sudo ip route show | grep -E "dev cbr0" | cut -d     " " -f1') top2_octets_read=$(echo $network | awk -F "." '{ print $1 "." $2 }') if [[ "$top2_octets_read" == "$top2_octets_final" ]]; then break fi if (( $attempt == $max_attempt )); then echo echo "(Failed) Waiting for cbr0 bridge to come up @ ${NODE_NAMES[$i]}" echo exit 1 fi printf "." sleep 5 done printf "n" KUBE_NODE_BRIDGE_NETWORK+=("${network}") done </ins> # Make the pods visible to each other and to the master. # The master needs have routes to the pods for the UI to work. local j for (( i=0; i<${#NODE_NAMES[@]}; i++)); do <del> printf "setting up routes for ${NODE_NAMES[$i]}" kube-ssh "${KUBE_MASTER_IP}" "sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[${i}]} gw ${KUBE_NODE_IP_ADDRESSES[${i}]}" for (( j=0; j<${#NODE_NAMES[@]}; j++)); do if [[ $i != $j ]]; then kube-ssh ${KUBE_NODE_IP_ADDRESSES[$i]} "sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[$j]} gw ${KUBE_NODE_IP_ADDRESSES[$j]}" fi done </del> <ins> printf "setting up routes for ${NODE_NAMES[$i]}n" printf " adding route to ${MASTER_NAME} for network ${KUBE_NODE_BRIDGE_NETWORK[${i}]} via ${KUBE_NODE_IP_ADDRESSES[${i}]}n" kube-ssh "${KUBE_MASTER_IP}" "sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[${i}]} gw ${KUBE_NODE_IP_ADDRESSES[${i}]}" for (( j=0; j<${#NODE_NAMES[@]}; j++)); do if [[ $i != $j ]]; then printf " adding route to ${NODE_NAMES[$j]} for network ${KUBE_NODE_BRIDGE_NETWORK[${i}]} via ${KUBE_NODE_IP_ADDRESSES[${i}]}n" kube-ssh ${KUBE_NODE_IP_ADDRESSES[$i]} "sudo route add -net ${KUBE_NODE_BRIDGE_NETWORK[$j]} gw ${KUBE_NODE_IP_ADDRESSES[$j]}" fi done printf "n" </ins> done }


printf "Waiting for salt-master to be up on ${KUBE_MASTER} ...n" remote-pgrep ${KUBE_MASTER_IP} "salt-master" <ins> printf "Waiting for all packages to be installed on ${KUBE_MASTER} ...n" kube-check  ${KUBE_MASTER_IP} 'sudo salt "kubernetes-master" state.highstate -t 30 | grep -E "Failed:[[:space:]]+0"' </ins> local i for (( i=0; i<${#NODE_NAMES[@]}; i++)); do printf "Waiting for salt-minion to be up on ${NODE_NAMES[$i]} ....n" remote-pgrep ${KUBE_NODE_IP_ADDRESSES[$i]} "salt-minion" <ins> printf "Waiting for all salt packages to be installed on ${NODE_NAMES[$i]} .... n" kube-check  ${KUBE_MASTER_IP} 'sudo salt '"${NODE_NAMES[$i]}"' state.highstate -t 30 | grep -E "Failed:[[:space:]]+0"' </ins> printf " OKn" done <del> printf "Waiting for init highstate to be done on all nodes (this can take a few minutes) ...n" kube-check  ${KUBE_MASTER_IP} 'sudo salt '''*''' state.show_highstate -t 50' printf "Waiting for all packages to be installed on all nodes (this can take a few minutes) ...n" kube-check  ${KUBE_MASTER_IP} 'sudo salt '''*''' state.highstate -t 50 | grep -E "Failed:[[:space:]]+0"' </del> echo echo "Waiting for master and node initialization."


"//pkg/runtime:go_default_library", "//pkg/types:go_default_library", "//pkg/util/errors:go_default_library", <del> "//pkg/util/hash:go_default_library", </del> "//pkg/util/metrics:go_default_library", "//pkg/util/runtime:go_default_library", "//pkg/util/wait:go_default_library",


import ( "encoding/json" "fmt" <del> "hash/adler32" </del> "time" "github.com/golang/glog"


"k8s.io/kubernetes/pkg/apis/batch" "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/types" <del> hashutil "k8s.io/kubernetes/pkg/util/hash" </del> ) // Utilities for dealing with Jobs and CronJobs and time.


return job, nil } <del> func getTimeHash(scheduledTime time.Time) uint32 { timeHasher := adler32.New() hashutil.DeepHashObject(timeHasher, scheduledTime) return timeHasher.Sum32() </del> <ins> // Return Unix Epoch Time func getTimeHash(scheduledTime time.Time) int64 { return scheduledTime.Unix() </ins> } // makeCreatedByRefJson makes a json string with an object reference for use in "created-by" annotation value


kube::build::run_build_command make test-integration fi <ins> kube::build::copy_output </ins> if [[ "${FEDERATION:-}" == "true" ]];then ( source "${KUBE_ROOT}/build/util.sh"


) fi <del> kube::build::copy_output </del> kube::release::package_tarballs kube::release::package_hyperkube


<ins> /* Copyright 2016 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package images import ( "fmt" "runtime" "testing" kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" ) type getCoreImageTest struct { i string c *kubeadmapi.MasterConfiguration o string } const testversion = "1" func TestGetCoreImage(t *testing.T) { var tokenTest = []struct { t        getCoreImageTest expected string }{ {getCoreImageTest{o: "override"}, "override"}, {getCoreImageTest{ i: KubeEtcdImage, c: &kubeadmapi.MasterConfiguration{}}, fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "etcd", runtime.GOARCH, etcdVersion), }, {getCoreImageTest{ i: KubeAPIServerImage, c: &kubeadmapi.MasterConfiguration{KubernetesVersion: testversion}}, fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-apiserver", runtime.GOARCH, testversion), }, {getCoreImageTest{ i: KubeControllerManagerImage, c: &kubeadmapi.MasterConfiguration{KubernetesVersion: testversion}}, fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-controller-manager", runtime.GOARCH, testversion), }, {getCoreImageTest{ i: KubeSchedulerImage, c: &kubeadmapi.MasterConfiguration{KubernetesVersion: testversion}}, fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-scheduler", runtime.GOARCH, testversion), }, {getCoreImageTest{ i: KubeProxyImage, c: &kubeadmapi.MasterConfiguration{KubernetesVersion: testversion}}, fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-proxy", runtime.GOARCH, testversion), }, } for _, rt := range tokenTest { actual := GetCoreImage(rt.t.i, rt.t.c, rt.t.o) if actual != rt.expected { t.Errorf( "failed GetCoreImage:ntexpected: %snt  actual: %s", rt.expected, actual, ) } } } func TestGetAddonImage(t *testing.T) { var tokenTest = []struct { t        string expected string }{ {"matches nothing", ""}, { KubeDNSImage, fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kubedns", runtime.GOARCH, kubeDNSVersion), }, { KubeDNSmasqImage, fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "kube-dnsmasq", runtime.GOARCH, dnsmasqVersion), }, { KubeExechealthzImage, fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "exechealthz", runtime.GOARCH, exechealthzVersion), }, { Pause, fmt.Sprintf("%s/%s-%s:%s", gcrPrefix, "pause", runtime.GOARCH, pauseVersion), }, } for _, rt := range tokenTest { actual := GetAddonImage(rt.t) if actual != rt.expected { t.Errorf( "failed GetCoreImage:ntexpected: %snt  actual: %s", rt.expected, actual, ) } } } </ins>


package mount <del> type Mounter struct{} </del> <ins> type Mounter struct { mounterPath string } </ins> func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { return nil


tags = ["automanaged"], deps = [ "//pkg/api:go_default_library", <ins> "//pkg/api/helper:go_default_library", </ins> "//pkg/api/service:go_default_library", "//pkg/api/testing:go_default_library", "//pkg/features:go_default_library",


} }() <del> if helper.IsServiceIPRequested(service) { // Allocate next available. ip, err := rs.serviceIPs.AllocateNext() if err != nil { // TODO: what error should be returned here?  It's not a // field-level validation failure (the field is valid), and it's // not really an internal error. return nil, errors.NewInternalError(fmt.Errorf("failed to allocate a serviceIP: %v", err)) } service.Spec.ClusterIP = ip.String() releaseServiceIP = true } else if helper.IsServiceIPSet(service) { // Try to respect the requested IP. if err := rs.serviceIPs.Allocate(net.ParseIP(service.Spec.ClusterIP)); err != nil { // TODO: when validation becomes versioned, this gets more complicated. el := field.ErrorList{field.Invalid(field.NewPath("spec", "clusterIP"), service.Spec.ClusterIP, err.Error())} return nil, errors.NewInvalid(api.Kind("Service"), service.Name, el) </del> <ins> var err error if service.Spec.Type != api.ServiceTypeExternalName { if releaseServiceIP, err = rs.initClusterIP(service); err != nil { return nil, err </ins> } <del> releaseServiceIP = true </del> } nodePortOp := portallocator.StartOperation(rs.serviceNodePorts) defer nodePortOp.Finish() <del> assignNodePorts := shouldAssignNodePorts(service) svcPortToNodePort := map[int]int{} for i := range service.Spec.Ports { servicePort := &service.Spec.Ports[i] allocatedNodePort := svcPortToNodePort[int(servicePort.Port)] if allocatedNodePort == 0 { // This will only scan forward in the service.Spec.Ports list because any matches // before the current port would have been found in svcPortToNodePort. This is really // looking for any user provided values. np := findRequestedNodePort(int(servicePort.Port), service.Spec.Ports) if np != 0 { err := nodePortOp.Allocate(np) if err != nil { // TODO: when validation becomes versioned, this gets more complicated. el := field.ErrorList{field.Invalid(field.NewPath("spec", "ports").Index(i).Child("nodePort"), np, err.Error())} return nil, errors.NewInvalid(api.Kind("Service"), service.Name, el) } servicePort.NodePort = int32(np) svcPortToNodePort[int(servicePort.Port)] = np } else if assignNodePorts { nodePort, err := nodePortOp.AllocateNext() if err != nil { // TODO: what error should be returned here?  It's not a // field-level validation failure (the field is valid), and it's // not really an internal error. return nil, errors.NewInternalError(fmt.Errorf("failed to allocate a nodePort: %v", err)) } servicePort.NodePort = int32(nodePort) svcPortToNodePort[int(servicePort.Port)] = nodePort } } else if int(servicePort.NodePort) != allocatedNodePort { if servicePort.NodePort == 0 { servicePort.NodePort = int32(allocatedNodePort) } else { err := nodePortOp.Allocate(int(servicePort.NodePort)) if err != nil { // TODO: when validation becomes versioned, this gets more complicated. el := field.ErrorList{field.Invalid(field.NewPath("spec", "ports").Index(i).Child("nodePort"), servicePort.NodePort, err.Error())} return nil, errors.NewInvalid(api.Kind("Service"), service.Name, el) } } </del> <ins> if service.Spec.Type == api.ServiceTypeNodePort || service.Spec.Type == api.ServiceTypeLoadBalancer { if err := rs.initNodePorts(service, nodePortOp); err != nil { return nil, err </ins> } }


// Update service from NodePort or LoadBalancer to ExternalName or ClusterIP, should release NodePort if exists. if (oldService.Spec.Type == api.ServiceTypeNodePort || oldService.Spec.Type == api.ServiceTypeLoadBalancer) && (service.Spec.Type == api.ServiceTypeExternalName || service.Spec.Type == api.ServiceTypeClusterIP) { <del> rs.releaseNodePort(oldService, nodePortOp) </del> <ins> rs.releaseNodePorts(oldService, nodePortOp) </ins> } // Update service from any type to NodePort or LoadBalancer, should update NodePort. if service.Spec.Type == api.ServiceTypeNodePort || service.Spec.Type == api.ServiceTypeLoadBalancer { <del> if err := rs.updateNodePort(oldService, service, nodePortOp); err != nil { </del> <ins> if err := rs.updateNodePorts(oldService, service, nodePortOp); err != nil { </ins> return nil, false, err } }


return servicePorts } <del> func shouldAssignNodePorts(service *api.Service) bool { switch service.Spec.Type { case api.ServiceTypeLoadBalancer: return true case api.ServiceTypeNodePort: return true case api.ServiceTypeClusterIP: return false case api.ServiceTypeExternalName: return false default: glog.Errorf("Unknown service type: %v", service.Spec.Type) return false } } </del> // Loop through the service ports list, find one with the same port number and // NodePort specified, return this NodePort otherwise return 0. func findRequestedNodePort(port int, servicePorts []api.ServicePort) int {


return false, nil } <del> func (rs *REST) updateNodePort(oldService, newService *api.Service, nodePortOp *portallocator.PortAllocationOperation) error { </del> <ins> func (rs *REST) initNodePorts(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) error { svcPortToNodePort := map[int]int{} for i := range service.Spec.Ports { servicePort := &service.Spec.Ports[i] allocatedNodePort := svcPortToNodePort[int(servicePort.Port)] if allocatedNodePort == 0 { // This will only scan forward in the service.Spec.Ports list because any matches // before the current port would have been found in svcPortToNodePort. This is really // looking for any user provided values. np := findRequestedNodePort(int(servicePort.Port), service.Spec.Ports) if np != 0 { err := nodePortOp.Allocate(np) if err != nil { // TODO: when validation becomes versioned, this gets more complicated. el := field.ErrorList{field.Invalid(field.NewPath("spec", "ports").Index(i).Child("nodePort"), np, err.Error())} return errors.NewInvalid(api.Kind("Service"), service.Name, el) } servicePort.NodePort = int32(np) svcPortToNodePort[int(servicePort.Port)] = np } else { nodePort, err := nodePortOp.AllocateNext() if err != nil { // TODO: what error should be returned here?  It's not a // field-level validation failure (the field is valid), and it's // not really an internal error. return errors.NewInternalError(fmt.Errorf("failed to allocate a nodePort: %v", err)) } servicePort.NodePort = int32(nodePort) svcPortToNodePort[int(servicePort.Port)] = nodePort } } else if int(servicePort.NodePort) != allocatedNodePort { // TODO(xiangpengzhao): do we need to allocate a new NodePort in this case? // Note: the current implementation is better, because it saves a NodePort. if servicePort.NodePort == 0 { servicePort.NodePort = int32(allocatedNodePort) } else { err := nodePortOp.Allocate(int(servicePort.NodePort)) if err != nil { // TODO: when validation becomes versioned, this gets more complicated. el := field.ErrorList{field.Invalid(field.NewPath("spec", "ports").Index(i).Child("nodePort"), servicePort.NodePort, err.Error())} return errors.NewInvalid(api.Kind("Service"), service.Name, el) } } } } return nil } func (rs *REST) updateNodePorts(oldService, newService *api.Service, nodePortOp *portallocator.PortAllocationOperation) error { </ins> oldNodePorts := CollectServiceNodePorts(oldService) newNodePorts := []int{}


return nil } <del> func (rs *REST) releaseNodePort(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) { </del> <ins> func (rs *REST) releaseNodePorts(service *api.Service, nodePortOp *portallocator.PortAllocationOperation) { </ins> nodePorts := CollectServiceNodePorts(service) for _, nodePort := range nodePorts {


"k8s.io/apiserver/pkg/registry/rest" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api" <ins> "k8s.io/kubernetes/pkg/api/helper" </ins> "k8s.io/kubernetes/pkg/api/service" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/registry/core/service/ipallocator"


if test.name == "Allocate specified ClusterIP" && test.svc.Spec.ClusterIP != "1.2.3.4" { t.Errorf("%q: expected ClusterIP %q, but got %q", test.name, "1.2.3.4", test.svc.Spec.ClusterIP) } <ins> if hasAllocatedIP { if helper.IsServiceIPSet(test.svc) { storage.serviceIPs.Release(net.ParseIP(test.svc.Spec.ClusterIP)) } } } } func TestInitNodePorts(t *testing.T) { storage, _ := NewTestREST(t, nil) nodePortOp := portallocator.StartOperation(storage.serviceNodePorts) defer nodePortOp.Finish() testCases := []struct { name                     string service                  *api.Service expectSpecifiedNodePorts []int }{ { name: "Service doesn't have specified NodePort", service: &api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, Type:     api.ServiceTypeNodePort, Ports: []api.ServicePort{ { Name:       "port-tcp", Port:       53, TargetPort: intstr.FromInt(6502), Protocol:   api.ProtocolTCP, }, }, }, }, expectSpecifiedNodePorts: []int{}, }, { name: "Service has one specified NodePort", service: &api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, Type:     api.ServiceTypeNodePort, Ports: []api.ServicePort{{ Name:       "port-tcp", Port:       53, TargetPort: intstr.FromInt(6502), Protocol:   api.ProtocolTCP, NodePort:   30053, }}, }, }, expectSpecifiedNodePorts: []int{30053}, }, { name: "Service has two same ports with different protocols and specifies same NodePorts", service: &api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, Type:     api.ServiceTypeNodePort, Ports: []api.ServicePort{ { Name:       "port-tcp", Port:       53, TargetPort: intstr.FromInt(6502), Protocol:   api.ProtocolTCP, NodePort:   30054, }, { Name:       "port-udp", Port:       53, TargetPort: intstr.FromInt(6502), Protocol:   api.ProtocolUDP, NodePort:   30054, }, }, }, }, expectSpecifiedNodePorts: []int{30054, 30054}, }, { name: "Service has two same ports with different protocols and specifies different NodePorts", service: &api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, Type:     api.ServiceTypeNodePort, Ports: []api.ServicePort{ { Name:       "port-tcp", Port:       53, TargetPort: intstr.FromInt(6502), Protocol:   api.ProtocolTCP, NodePort:   30055, }, { Name:       "port-udp", Port:       53, TargetPort: intstr.FromInt(6502), Protocol:   api.ProtocolUDP, NodePort:   30056, }, }, }, }, expectSpecifiedNodePorts: []int{30055, 30056}, }, { name: "Service has two different ports with different protocols and specifies different NodePorts", service: &api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, Type:     api.ServiceTypeNodePort, Ports: []api.ServicePort{ { Name:       "port-tcp", Port:       53, TargetPort: intstr.FromInt(6502), Protocol:   api.ProtocolTCP, NodePort:   30057, }, { Name:       "port-udp", Port:       54, TargetPort: intstr.FromInt(6502), Protocol:   api.ProtocolUDP, NodePort:   30058, }, }, }, }, expectSpecifiedNodePorts: []int{30057, 30058}, }, { name: "Service has two same ports with different protocols but only specifies one NodePort", service: &api.Service{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: api.ServiceSpec{ Selector: map[string]string{"bar": "baz"}, Type:     api.ServiceTypeNodePort, Ports: []api.ServicePort{ { Name:       "port-tcp", Port:       53, TargetPort: intstr.FromInt(6502), Protocol:   api.ProtocolTCP, NodePort:   30059, }, { Name:       "port-udp", Port:       53, TargetPort: intstr.FromInt(6502), Protocol:   api.ProtocolUDP, }, }, }, }, expectSpecifiedNodePorts: []int{30059, 30059}, }, } for _, test := range testCases { err := storage.initNodePorts(test.service, nodePortOp) if err != nil { t.Errorf("%q: unexpected error: %v", test.name, err) continue } serviceNodePorts := CollectServiceNodePorts(test.service) if len(test.expectSpecifiedNodePorts) == 0 { for _, nodePort := range serviceNodePorts { if !storage.serviceNodePorts.Has(nodePort) { t.Errorf("%q: unexpected NodePort %d, out of range", test.name, nodePort) } } } else if !reflect.DeepEqual(serviceNodePorts, test.expectSpecifiedNodePorts) { t.Errorf("%q: expected NodePorts %v, but got %v", test.name, test.expectSpecifiedNodePorts, serviceNodePorts) } </ins> } } <del> func TestUpdateNodePort(t *testing.T) { </del> <ins> func TestUpdateNodePorts(t *testing.T) { </ins> storage, _ := NewTestREST(t, nil) nodePortOp := portallocator.StartOperation(storage.serviceNodePorts) defer nodePortOp.Finish()


} for _, test := range testCases { <del> err := storage.updateNodePort(test.oldService, test.newService, nodePortOp) </del> <ins> err := storage.updateNodePorts(test.oldService, test.newService, nodePortOp) </ins> if err != nil { t.Errorf("%q: unexpected error: %v", test.name, err) continue } <del> _ = nodePortOp.Commit() </del> serviceNodePorts := CollectServiceNodePorts(test.newService)


KUBELET_CERT_BASE64=$(get-env-val "${master_env}" "KUBELET_CERT") KUBELET_KEY_BASE64=$(get-env-val "${master_env}" "KUBELET_KEY") } <ins> # Update or verify required gcloud components are installed # at minimum required version. # Assumed vars #   KUBE_PROMPT_FOR_UPDATE function update-or-verify-gcloud() { local sudo_prefix="" if [ ! -w $(dirname `which gcloud`) ]; then sudo_prefix="sudo" fi # update and install components as needed if [[ "${KUBE_PROMPT_FOR_UPDATE}" == "y" ]]; then ${sudo_prefix} gcloud ${gcloud_prompt:-} components install alpha ${sudo_prefix} gcloud ${gcloud_prompt:-} components install beta ${sudo_prefix} gcloud ${gcloud_prompt:-} components update else local version=$(${sudo_prefix} gcloud version --format=json) python -c' import json,sys from distutils import version minVersion = version.LooseVersion("1.3.0") required = [ "alpha", "beta", "core" ] data = json.loads(sys.argv[1]) rel = data.get("Google Cloud SDK") if rel != "HEAD" and version.LooseVersion(rel) < minVersion: print "gcloud version out of date ( < %s )" % minVersion exit(1) missing = [] for c in required: if not data.get(c): missing += [c] if missing: for c in missing: print ("missing required gcloud component "{0}"".format(c)) exit(1) ' """${version}""" fi } </ins>


ALLOCATE_NODE_CIDRS=true <del> KUBE_PROMPT_FOR_UPDATE=y KUBE_SKIP_UPDATE=${KUBE_SKIP_UPDATE-"n"} </del> <ins> KUBE_PROMPT_FOR_UPDATE=${KUBE_PROMPT_FOR_UPDATE:-"n"} </ins> # How long (in seconds) to wait for cluster initialization. KUBE_CLUSTER_INITIALIZATION_TIMEOUT=${KUBE_CLUSTER_INITIALIZATION_TIMEOUT:-300}


local cmd for cmd in gcloud gsutil; do if ! which "${cmd}" >/dev/null; then <del> local resp </del> <ins> local resp="n" </ins> if [[ "${KUBE_PROMPT_FOR_UPDATE}" == "y" ]]; then echo "Can't find ${cmd} in PATH.  Do you wish to install the Google Cloud SDK? [Y/n]" read resp <del> else resp="y" </del> fi if [[ "${resp}" != "n" && "${resp}" != "N" ]]; then curl https://sdk.cloud.google.com | bash


fi fi done <del> if [[ "${KUBE_SKIP_UPDATE}" == "y" ]]; then return fi # update and install components as needed if [[ "${KUBE_PROMPT_FOR_UPDATE}" != "y" ]]; then gcloud_prompt="-q" fi local sudo_prefix="" if [ ! -w $(dirname `which gcloud`) ]; then sudo_prefix="sudo" fi ${sudo_prefix} gcloud ${gcloud_prompt:-} components install alpha || true ${sudo_prefix} gcloud ${gcloud_prompt:-} components install beta || true ${sudo_prefix} gcloud ${gcloud_prompt:-} components update || true </del> <ins> update-or-verify-gcloud </ins> } # Create a temp dir that'll be deleted at the end of this bash session.


# Uses the config file specified in $KUBE_CONFIG_FILE, or defaults to config-default.sh <del> KUBE_PROMPT_FOR_UPDATE=y KUBE_SKIP_UPDATE=${KUBE_SKIP_UPDATE-"n"} </del> <ins> KUBE_PROMPT_FOR_UPDATE=${KUBE_PROMPT_FOR_UPDATE:-"n"} </ins> KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. source "${KUBE_ROOT}/cluster/gke/${KUBE_CONFIG_FILE:-config-default.sh}" source "${KUBE_ROOT}/cluster/common.sh"


if [[ "${KUBE_PROMPT_FOR_UPDATE}" == "y" ]]; then echo "Can't find gcloud in PATH.  Do you wish to install the Google Cloud SDK? [Y/n]" read resp <del> else resp="y" </del> fi if [[ "${resp}" != "n" && "${resp}" != "N" ]]; then curl https://sdk.cloud.google.com | bash


exit 1 fi fi <del> if [[ "${KUBE_SKIP_UPDATE}" == "y" ]]; then return fi # update and install components as needed if [[ "${KUBE_PROMPT_FOR_UPDATE}" != "y" ]]; then gcloud_prompt="-q" fi local sudo_prefix="" if [ ! -w $(dirname `which gcloud`) ]; then sudo_prefix="sudo" fi ${sudo_prefix} gcloud ${gcloud_prompt:-} components install alpha || true ${sudo_prefix} gcloud ${gcloud_prompt:-} components install beta || true ${sudo_prefix} gcloud ${gcloud_prompt:-} components install kubectl|| true ${sudo_prefix} gcloud ${gcloud_prompt:-} components update || true </del> <ins> update-or-verify-gcloud </ins> } # Validate a kubernetes cluster


"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", <ins> "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", </ins> "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", "//vendor/k8s.io/client-go/pkg/api/v1:go_default_library",


metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" pkgruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/runtime" <ins> "k8s.io/apimachinery/pkg/util/sets" </ins> "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/tools/cache"


non-nil error is also returned (possibly along with a partially complete list of resolved endpoints. */ func getResolvedEndpoints(endpoints []string) ([]string, error) { <del> resolvedEndpoints := make([]string, 0, len(endpoints)) </del> <ins> resolvedEndpoints := sets.String{} </ins> for _, endpoint := range endpoints { if net.ParseIP(endpoint) == nil { // It's not a valid IP address, so assume it's a DNS name, and try to resolve it, // replacing its DNS name with its IP addresses in expandedEndpoints ipAddrs, err := net.LookupHost(endpoint) if err != nil { <del> return resolvedEndpoints, err </del> <ins> return resolvedEndpoints.List(), err } for _, ip := range ipAddrs { resolvedEndpoints = resolvedEndpoints.Union(sets.NewString(ip)) </ins> } <del> resolvedEndpoints = append(resolvedEndpoints, ipAddrs...) </del> } else { <del> resolvedEndpoints = append(resolvedEndpoints, endpoint) </del> <ins> resolvedEndpoints = resolvedEndpoints.Union(sets.NewString(endpoint)) </ins> } } <del> return resolvedEndpoints, nil </del> <ins> return resolvedEndpoints.List(), nil </ins> } /* ensureDNSRrsets ensures (idempotently, and with minimum mutations) that all of the DNS resource record sets for dnsName are consistent with endpoints.


if observedReadyCondition.Status != v1.ConditionUnknown { currentReadyCondition.Status = v1.ConditionUnknown currentReadyCondition.Reason = "NodeStatusUnknown" <del> currentReadyCondition.Message = fmt.Sprintf("Kubelet stopped posting node status.") </del> <ins> currentReadyCondition.Message = "Kubelet stopped posting node status." </ins> // LastProbeTime is the last time we heard from kubelet. currentReadyCondition.LastHeartbeatTime = observedReadyCondition.LastHeartbeatTime currentReadyCondition.LastTransitionTime = nc.now() } } <del> // Like NodeReady condition, NodeOutOfDisk was last set longer ago than gracePeriod, so update // it to Unknown (regardless of its current value) in the master. // TODO(madhusudancs): Refactor this with readyCondition to remove duplicated code. _, oodCondition := v1.GetNodeCondition(&node.Status, v1.NodeOutOfDisk) if oodCondition == nil { glog.V(2).Infof("Out of disk condition of node %v is never updated by kubelet", node.Name) node.Status.Conditions = append(node.Status.Conditions, v1.NodeCondition{ Type:               v1.NodeOutOfDisk, Status:             v1.ConditionUnknown, Reason:             "NodeStatusNeverUpdated", Message:            fmt.Sprintf("Kubelet never posted node status."), LastHeartbeatTime:  node.CreationTimestamp, LastTransitionTime: nc.now(), }) } else { glog.V(4).Infof("node %v hasn't been updated for %+v. Last out of disk condition is: %+v", node.Name, nc.now().Time.Sub(savedNodeStatus.probeTimestamp.Time), oodCondition) if oodCondition.Status != v1.ConditionUnknown { oodCondition.Status = v1.ConditionUnknown oodCondition.Reason = "NodeStatusUnknown" oodCondition.Message = fmt.Sprintf("Kubelet stopped posting node status.") oodCondition.LastTransitionTime = nc.now() </del> <ins> // remaining node conditions should also be set to Unknown remainingNodeConditionTypes := []v1.NodeConditionType{v1.NodeOutOfDisk, v1.NodeMemoryPressure, v1.NodeDiskPressure} nowTimestamp := nc.now() for _, nodeConditionType := range remainingNodeConditionTypes { _, currentCondition := v1.GetNodeCondition(&node.Status, nodeConditionType) if currentCondition == nil { glog.V(2).Infof("Condition %v of node %v was never updated by kubelet", nodeConditionType, node.Name) node.Status.Conditions = append(node.Status.Conditions, v1.NodeCondition{ Type:               nodeConditionType, Status:             v1.ConditionUnknown, Reason:             "NodeStatusNeverUpdated", Message:            "Kubelet never posted node status.", LastHeartbeatTime:  node.CreationTimestamp, LastTransitionTime: nowTimestamp, }) } else { glog.V(4).Infof("node %v hasn't been updated for %+v. Last %v is: %+v", node.Name, nc.now().Time.Sub(savedNodeStatus.probeTimestamp.Time), nodeConditionType, currentCondition) if currentCondition.Status != v1.ConditionUnknown { currentCondition.Status = v1.ConditionUnknown currentCondition.Reason = "NodeStatusUnknown" currentCondition.Message = "Kubelet stopped posting node status." currentCondition.LastTransitionTime = nowTimestamp } </ins> } }


LastHeartbeatTime:  metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), LastTransitionTime: fakeNow, }, <ins> { Type:               v1.NodeMemoryPressure, Status:             v1.ConditionUnknown, Reason:             "NodeStatusNeverUpdated", Message:            "Kubelet never posted node status.", LastHeartbeatTime:  metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), LastTransitionTime: fakeNow, }, { Type:               v1.NodeDiskPressure, Status:             v1.ConditionUnknown, Reason:             "NodeStatusNeverUpdated", Message:            "Kubelet never posted node status.", LastHeartbeatTime:  metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), LastTransitionTime: fakeNow, }, </ins> }, }, },


LastHeartbeatTime:  metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: metav1.Time{Time: metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC).Add(time.Hour)}, }, <ins> { Type:               v1.NodeMemoryPressure, Status:             v1.ConditionUnknown, Reason:             "NodeStatusNeverUpdated", Message:            "Kubelet never posted node status.", LastHeartbeatTime:  metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), // should default to node creation time if condition was never updated LastTransitionTime: metav1.Time{Time: metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC).Add(time.Hour)}, }, { Type:               v1.NodeDiskPressure, Status:             v1.ConditionUnknown, Reason:             "NodeStatusNeverUpdated", Message:            "Kubelet never posted node status.", LastHeartbeatTime:  metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), // should default to node creation time if condition was never updated LastTransitionTime: metav1.Time{Time: metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC).Add(time.Hour)}, }, </ins> }, Capacity: v1.ResourceList{ v1.ResourceName(v1.ResourceCPU):    resource.MustParse("10"),


"strings" "time" <ins> "k8s.io/kubernetes/pkg/api/resource" </ins> "k8s.io/kubernetes/pkg/api/v1" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5" "k8s.io/kubernetes/pkg/labels"


const ( DNSdefaultTimeout      = 5 * time.Minute <del> DNSscaleUpTimeout      = 5 * time.Minute DNSscaleDownTimeout    = 10 * time.Minute </del> DNSNamespace           = "kube-system" ClusterAddonLabelKey   = "k8s-app" KubeDNSLabelName       = "kube-dns"


var _ = framework.KubeDescribe("DNS horizontal autoscaling", func() { f := framework.NewDefaultFramework("dns-autoscaling") var c clientset.Interface <del> var nodeCount int </del> var previousParams map[string]string <del> DNSParams_1 := map[string]string{"linear": "{"nodesPerReplica": 1}"} DNSParams_2 := map[string]string{"linear": "{"nodesPerReplica": 2}"} DNSParams_3 := map[string]string{"linear": "{"nodesPerReplica": 3}"} </del> <ins> DNSParams_1 := DNSParamsLinear{map[string]string{"linear": "{"nodesPerReplica": 1}"}, 1.0, 0.0} DNSParams_2 := DNSParamsLinear{map[string]string{"linear": "{"nodesPerReplica": 2}"}, 2.0, 0.0} DNSParams_3 := DNSParamsLinear{map[string]string{"linear": "{"nodesPerReplica": 3, "coresPerReplica": 3}"}, 3.0, 3.0} </ins> BeforeEach(func() { framework.SkipUnlessProviderIs("gce") c = f.ClientSet <del> nodes := framework.GetReadySchedulableNodesOrDie(c) nodeCount = len(nodes.Items) Expect(nodeCount).NotTo(BeZero()) </del> <ins> Expect(len(framework.GetReadySchedulableNodesOrDie(c).Items)).NotTo(BeZero()) </ins> pcm, err := fetchDNSScalingConfigMap(c) <del> ExpectNoError(err) </del> <ins> Expect(err).NotTo(HaveOccurred()) </ins> previousParams = pcm.Data By("Replace the dns autoscaling parameters with testing parameters") <del> ExpectNoError(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_1))) </del> <ins> Expect(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_1.data))).NotTo(HaveOccurred()) </ins> By("Wait for kube-dns scaled to expected number") <del> ExpectNoError(waitForDNSReplicasSatisfied(c, nodeCount, DNSdefaultTimeout)) </del> <ins> getExpectReplicasLinear := getExpectReplicasFuncLinear(c, &DNSParams_1) Expect(waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout)).NotTo(HaveOccurred()) </ins> }) AfterEach(func() { By("Restoring intial dns autoscaling parameters") <del> ExpectNoError(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(previousParams))) </del> <ins> Expect(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(previousParams))).NotTo(HaveOccurred()) </ins> }) // This test is separated because it is slow and need to run serially


sum := 0 for _, mig := range strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",") { size, err := GroupSize(mig) <del> ExpectNoError(err) </del> <ins> Expect(err).NotTo(HaveOccurred()) </ins> By(fmt.Sprintf("Initial size of %s: %d", mig, size)) originalSizes[mig] = size sum += size } <del> Expect(nodeCount).Should(Equal(sum)) </del> By("Manually increase cluster size") increasedSize := 0


increasedSize += increasedSizes[key] } setMigSizes(increasedSizes) <del> ExpectNoError(WaitForClusterSizeFunc(c, func(size int) bool { return size == increasedSize }, DNSscaleUpTimeout)) </del> <ins> Expect(WaitForClusterSizeFunc(c, func(size int) bool { return size == increasedSize }, scaleUpTimeout)).NotTo(HaveOccurred()) By("Wait for kube-dns scaled to expected number") getExpectReplicasLinear := getExpectReplicasFuncLinear(c, &DNSParams_1) Expect(waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout)).NotTo(HaveOccurred()) By("Replace the dns autoscaling parameters with another testing parameters") Expect(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_3.data))).NotTo(HaveOccurred()) </ins> By("Wait for kube-dns scaled to expected number") <del> ExpectNoError(waitForDNSReplicasSatisfied(c, increasedSize, DNSdefaultTimeout)) </del> <ins> getExpectReplicasLinear = getExpectReplicasFuncLinear(c, &DNSParams_3) Expect(waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout)).NotTo(HaveOccurred()) </ins> By("Restoring cluster size") setMigSizes(originalSizes) <del> framework.ExpectNoError(framework.WaitForClusterSize(c, nodeCount, DNSscaleDownTimeout)) </del> <ins> Expect(framework.WaitForClusterSize(c, sum, scaleDownTimeout)).NotTo(HaveOccurred()) </ins> By("Wait for kube-dns scaled to expected number") <del> ExpectNoError(waitForDNSReplicasSatisfied(c, nodeCount, DNSdefaultTimeout)) </del> <ins> Expect(waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout)).NotTo(HaveOccurred()) </ins> }) It("kube-dns-autoscaler should scale kube-dns pods in both nonfaulty and faulty scenarios", func() { By("--- Scenario: should scale kube-dns based on changed parameters ---") <del> By("Replace the dns autoscaling parameters with the second testing parameters") ExpectNoError(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_2))) </del> <ins> By("Replace the dns autoscaling parameters with another testing parameters") Expect(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_3.data))).NotTo(HaveOccurred()) </ins> By("Wait for kube-dns scaled to expected number") <del> ExpectNoError(waitForDNSReplicasSatisfied(c, int(math.Ceil(float64(nodeCount)/2.0)), DNSdefaultTimeout)) </del> <ins> getExpectReplicasLinear := getExpectReplicasFuncLinear(c, &DNSParams_3) Expect(waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout)).NotTo(HaveOccurred()) </ins> By("--- Scenario: should re-create scaling parameters with default value when parameters got deleted ---") By("Delete the ConfigMap for autoscaler") err := deleteDNSScalingConfigMap(c) <del> ExpectNoError(err) </del> <ins> Expect(err).NotTo(HaveOccurred()) </ins> By("Wait for the ConfigMap got re-created") configMap, err := waitForDNSConfigMapCreated(c, DNSdefaultTimeout) <del> ExpectNoError(err) </del> <ins> Expect(err).NotTo(HaveOccurred()) </ins> By("Check the new created ConfigMap got the same data as we have") Expect(reflect.DeepEqual(previousParams, configMap.Data)).To(Equal(true)) <del> By("Replace the dns autoscaling parameters with the second testing parameters") ExpectNoError(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_2))) </del> <ins> By("Replace the dns autoscaling parameters with another testing parameters") Expect(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_2.data))).NotTo(HaveOccurred()) </ins> By("Wait for kube-dns scaled to expected number") <del> ExpectNoError(waitForDNSReplicasSatisfied(c, int(math.Ceil(float64(nodeCount)/2.0)), DNSdefaultTimeout)) </del> <ins> getExpectReplicasLinear = getExpectReplicasFuncLinear(c, &DNSParams_2) Expect(waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout)).NotTo(HaveOccurred()) </ins> By("--- Scenario: should recover after autoscaler pod got deleted ---") By("Delete the autoscaler pod for kube-dns") <del> ExpectNoError(deleteDNSAutoscalerPod(c)) </del> <ins> Expect(deleteDNSAutoscalerPod(c)).NotTo(HaveOccurred()) </ins> <del> By("Replace the dns autoscaling parameters with the third testing parameters") ExpectNoError(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_3))) </del> <ins> By("Replace the dns autoscaling parameters with another testing parameters") Expect(updateDNSScalingConfigMap(c, packDNSScalingConfigMap(DNSParams_1.data))).NotTo(HaveOccurred()) </ins> By("Wait for kube-dns scaled to expected number") <del> ExpectNoError(waitForDNSReplicasSatisfied(c, int(math.Ceil(float64(nodeCount)/3.0)), DNSdefaultTimeout)) </del> <ins> getExpectReplicasLinear = getExpectReplicasFuncLinear(c, &DNSParams_1) Expect(waitForDNSReplicasSatisfied(c, getExpectReplicasLinear, DNSdefaultTimeout)).NotTo(HaveOccurred()) </ins> }) }) <ins> type DNSParamsLinear struct { data            map[string]string nodesPerReplica float64 coresPerReplica float64 } type getExpectReplicasFunc func(c clientset.Interface) int func getExpectReplicasFuncLinear(c clientset.Interface, params *DNSParamsLinear) getExpectReplicasFunc { return func(c clientset.Interface) int { var replicasFromNodes float64 var replicasFromCores float64 nodes := framework.GetReadySchedulableNodesOrDie(c).Items if params.nodesPerReplica > 0 { replicasFromNodes = math.Ceil(float64(len(nodes)) / params.nodesPerReplica) } if params.coresPerReplica > 0 { replicasFromCores = math.Ceil(float64(getScheduableCores(nodes)) / params.coresPerReplica) } return int(math.Max(1.0, math.Max(replicasFromNodes, replicasFromCores))) } } func getScheduableCores(nodes []v1.Node) int64 { var sc resource.Quantity for _, node := range nodes { if !node.Spec.Unschedulable { sc.Add(node.Status.Capacity[v1.ResourceCPU]) } } scInt64, scOk := sc.AsInt64() if !scOk { framework.Logf("unable to compute integer values of schedulable cores in the cluster") return 0 } return scInt64 } </ins> func fetchDNSScalingConfigMap(c clientset.Interface) (*v1.ConfigMap, error) { cm, err := c.Core().ConfigMaps(DNSNamespace).Get(DNSAutoscalerLabelName) if err != nil {


return nil } <del> func waitForDNSReplicasSatisfied(c clientset.Interface, expected int, timeout time.Duration) (err error) { </del> <ins> func waitForDNSReplicasSatisfied(c clientset.Interface, getExpected getExpectReplicasFunc, timeout time.Duration) (err error) { </ins> var current int <del> framework.Logf("Waiting up to %v for kube-dns reach %v replicas", timeout, expected) </del> <ins> var expected int framework.Logf("Waiting up to %v for kube-dns to reach expected replicas", timeout) </ins> condition := func() (bool, error) { current, err = getDNSReplicas(c) if err != nil { return false, err } <ins> expected = getExpected(c) </ins> if current != expected { <ins> framework.Logf("replicas not as expected: got %v, expected %v", current, expected) </ins> return false, nil } return true, nil } <del> if err = wait.Poll(time.Second, timeout, condition); err != nil { </del> <ins> if err = wait.Poll(2*time.Second, timeout, condition); err != nil { </ins> return fmt.Errorf("err waiting for DNS replicas to satisfy %v, got %v: %v", expected, current, err) } <ins> framework.Logf("kube-dns reaches expected replicas: %v", expected) </ins> return nil }


VolumeMounts:  volumeMounts, LivenessProbe: componentProbe(8080, "/healthz"), Resources:     componentResources("250m"), <ins> Env:           getProxyEnvVars(), </ins> }, volumes...), kubeControllerManager: componentPod(api.Container{ Name:          kubeControllerManager,