<ins> /* Copyright 2014 Google Inc. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package config import ( "reflect" "testing" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/api/testapi" "github.com/ghodss/yaml" ) func noDefault(*api.Pod) error { return nil } func TestDecodeSinglePod(t *testing.T) { pod := &api.Pod{ TypeMeta: api.TypeMeta{ APIVersion: "", }, ObjectMeta: api.ObjectMeta{ Name:      "test", UID:       "12345", Namespace: "mynamespace", }, Spec: api.PodSpec{ RestartPolicy: api.RestartPolicyAlways, DNSPolicy:     api.DNSClusterFirst, Containers: []api.Container{{ Name:                   "image", Image:                  "test/image", ImagePullPolicy:        "IfNotPresent", TerminationMessagePath: "/dev/termination-log", }}, }, } json, err := testapi.Codec().Encode(pod) if err != nil { t.Errorf("unexpected error: %v", err) } parsed, podOut, err := tryDecodeSinglePod(json, noDefault) if testapi.Version() == "v1beta1" { // v1beta1 conversion leaves empty lists that should be nil podOut.Spec.Containers[0].Resources.Limits = nil podOut.Spec.Containers[0].Resources.Requests = nil } if !parsed { t.Errorf("expected to have parsed file: (%s)", string(json)) } if err != nil { t.Errorf("unexpected error: %v (%s)", err, string(json)) } if !reflect.DeepEqual(pod, podOut) { t.Errorf("expected:n%#vngot:n%#vn%s", pod, podOut, string(json)) } externalPod, err := testapi.Converter().ConvertToVersion(pod, "v1beta3") if err != nil { t.Errorf("unexpected error: %v", err) } yaml, err := yaml.Marshal(externalPod) if err != nil { t.Errorf("unexpected error: %v", err) } parsed, podOut, err = tryDecodeSinglePod(yaml, noDefault) if !parsed { t.Errorf("expected to have parsed file: (%s)", string(yaml)) } if err != nil { t.Errorf("unexpected error: %v (%s)", err, string(yaml)) } if !reflect.DeepEqual(pod, podOut) { t.Errorf("expected:n%#vngot:n%#vn%s", pod, podOut, string(yaml)) } } func TestDecodePodList(t *testing.T) { pod := &api.Pod{ TypeMeta: api.TypeMeta{ APIVersion: "", }, ObjectMeta: api.ObjectMeta{ Name:      "test", UID:       "12345", Namespace: "mynamespace", }, Spec: api.PodSpec{ RestartPolicy: api.RestartPolicyAlways, DNSPolicy:     api.DNSClusterFirst, Containers: []api.Container{{ Name:                   "image", Image:                  "test/image", ImagePullPolicy:        "IfNotPresent", TerminationMessagePath: "/dev/termination-log", }}, }, } podList := &api.PodList{ Items: []api.Pod{*pod}, } json, err := testapi.Codec().Encode(podList) if err != nil { t.Errorf("unexpected error: %v", err) } parsed, podListOut, err := tryDecodePodList(json, noDefault) if testapi.Version() == "v1beta1" { // v1beta1 conversion leaves empty lists that should be nil podListOut.Items[0].Spec.Containers[0].Resources.Limits = nil podListOut.Items[0].Spec.Containers[0].Resources.Requests = nil } if !parsed { t.Errorf("expected to have parsed file: (%s)", string(json)) } if err != nil { t.Errorf("unexpected error: %v (%s)", err, string(json)) } if !reflect.DeepEqual(podList, &podListOut) { t.Errorf("expected:n%#vngot:n%#vn%s", podList, &podListOut, string(json)) } externalPodList, err := testapi.Converter().ConvertToVersion(podList, "v1beta3") if err != nil { t.Errorf("unexpected error: %v", err) } yaml, err := yaml.Marshal(externalPodList) if err != nil { t.Errorf("unexpected error: %v", err) } parsed, podListOut, err = tryDecodePodList(yaml, noDefault) if !parsed { t.Errorf("expected to have parsed file: (%s)", string(yaml)) } if err != nil { t.Errorf("unexpected error: %v (%s)", err, string(yaml)) } if !reflect.DeepEqual(podList, &podListOut) { t.Errorf("expected:n%#vngot:n%#vn%s", pod, &podListOut, string(yaml)) } } </ins>


<ins> # Admission Controllers ## What are they? An admission control plug-in is a piece of code that intercepts requests to the Kubernetes API server prior to persistence of the object.  The plug-in code is in the API server process and must be compiled into the binary in order to be used at this time. Each admission control plug-in is run in sequence before a request is accepted into the cluster.  If any of the plug-ins in the sequence reject the request, the entire request is rejected immediately and an error is returned to the end-user. Admission control plug-ins may mutate the incoming object in some cases to apply system configured defaults.  In addition, admission control plug-ins may mutate related resources as part of request processing to do things like increment quota usage. ## Why do I need them? Many advanced features in Kubernetes require an admission control plug-in to be enabled in order to properly support the feature.  As a result, a Kubernetes API server that is not properly configured with the right set of admission control plug-ins is an incomplete server and will not support all the features you expect. ## How do I turn on an admission control plug-in? The Kubernetes API server supports a flag, ```admission_control``` that takes a comma-delimited, ordered list of admission control choices to invoke prior to modifying objects in the cluster. ## What does each plug-in do? ### AlwaysAdmit This plug-in will accept all incoming requests made to the Kubernetes API server. ### AlwaysDeny This plug-in will reject all mutating requests made to the Kubernetes API server.  It's largely intended for testing purposes and is not recommended for usage in a real deployment. ### DenyExecOnPrivileged This plug-in will intercept all requests to exec a command in a pod if that pod has a privileged container. If your cluster supports privileged containers, and you want to restrict the ability of end-users to exec commands in those containers, we strongly encourage enabling this plug-in. ### ServiceAccount This plug-in limits admission of Pod creation requests based on the Pod's ```ServiceAccount```. 1. If the pod does not have a ```ServiceAccount```, it modifies the pod's ```ServiceAccount``` to "default". 2. It ensures that the ```ServiceAccount``` referenced by a pod exists. 3. If ```LimitSecretReferences``` is true, it rejects the pod if the pod references ```Secret``` objects which the pods ```ServiceAccount``` does not reference. 4. If the pod does not contain any ```ImagePullSecrets```, the ```ImagePullSecrets``` of the ```ServiceAccount``` are added to the pod. 5. If ```MountServiceAccountToken``` is true, it adds a ```VolumeMount``` with the pod's ```ServiceAccount``` API token secret to containers in the pod. We strongly recommend using this plug-in if you intend to make use of Kubernetes ```ServiceAccount``` objects. ### SecurityContextDeny This plug-in will deny any ```SecurityContext``` that defines options that were not available on the ```Container```. ### ResourceQuota This plug-in will observe the incoming request and ensure that it does not violate any of the constraints enumerated in the ```ResourceQuota``` object in a ```Namespace```.  If you are using ```ResourceQuota``` objects in your Kubernetes deployment, you MUST use this plug-in to enforce quota constraints. It is strongly encouraged that this plug-in is configured last in the sequence of admission control plug-ins.  This is so that quota is not prematurely incremented only for the request to be rejected later in admission control. ### LimitRanger This plug-in will observe the incoming request and ensure that it does not violate any of the constraints enumerated in the ```LimitRange``` object in a ```Namespace```.  If you are using ```LimitRange``` objects in your Kubernetes deployment, you MUST use this plug-in to enforce those constraints. ### NamespaceExists This plug-in will observe all incoming requests that attempt to create a resource in a Kubernetes ```Namespace``` and reject the request if the ```Namespace``` was not previously created.  We strongly recommend running this plug-in to ensure integrity of your data. ### NamespaceAutoProvision (deprecated) This plug-in will observe all incoming requests that attempt to create a resource in a Kubernetes ```Namespace``` and create a new ```Namespace``` if one did not already exist previously. We strongly recommend ```NamespaceExists``` over ```NamespaceAutoProvision```. ### NamespaceLifecycle This plug-in enforces that a ```Namespace``` that is undergoing termination cannot have new content created in it. A ```Namespace``` deletion kicks off a sequence of operations that remove all content (pods, services, etc.) in that namespace.  In order to enforce integrity of that process, we strongly recommend running this plug-in. Once ```NamespaceAutoProvision``` is deprecated, we anticipate ```NamespaceLifecycle``` and ```NamespaceExists``` will be merged into a single plug-in that enforces the life-cycle of a ```Namespace``` in Kubernetes. ## Is there a recommended set of plug-ins to use? Yes. For Kubernetes 1.0, we strongly recommend running the following set of admission control plug-ins: ```shell --admission_control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota ``` </ins>


KUBE_PROMPT_FOR_UPDATE=y KUBE_SKIP_UPDATE=${KUBE_SKIP_UPDATE-"n"} <del> # Suffix to append to the staging path used for the server tars. Useful if # multiple versions of the server are being used in the same project # simultaneously (e.g. on Jenkins). KUBE_GCS_STAGING_PATH_SUFFIX=${KUBE_GCS_STAGING_PATH_SUFFIX-""} </del> # How long (in seconds) to wait for cluster initialization. KUBE_CLUSTER_INITIALIZATION_TIMEOUT=${KUBE_CLUSTER_INITIALIZATION_TIMEOUT:-300}


gsutil mb "${staging_bucket}" fi <del> local -r staging_path="${staging_bucket}/devel${KUBE_GCS_STAGING_PATH_SUFFIX}" </del> <ins> local -r staging_path="${staging_bucket}/${INSTANCE_PREFIX}-devel" </ins> SERVER_BINARY_TAR_HASH=$(sha1sum-file "${SERVER_BINARY_TAR}") SALT_TAR_HASH=$(sha1sum-file "${SALT_TAR}")


${GCE_SLOW_TESTS[@]:+${GCE_SLOW_TESTS[@]}}  )"} : ${KUBE_GCE_INSTANCE_PREFIX:="e2e-gce-${NODE_NAME}-${EXECUTOR_NUMBER}"} <del> : ${KUBE_GCS_STAGING_PATH_SUFFIX:="-${NODE_NAME}-${EXECUTOR_NUMBER}"} </del> : ${PROJECT:="kubernetes-jenkins-pull"} : ${ENABLE_DEPLOYMENTS:=true} # Override GCE defaults


export KUBE_GCE_ZONE=${E2E_ZONE} export KUBE_GCE_NETWORK=${E2E_NETWORK} export KUBE_GCE_INSTANCE_PREFIX=${KUBE_GCE_INSTANCE_PREFIX:-} <del> export KUBE_GCS_STAGING_PATH_SUFFIX=${KUBE_GCS_STAGING_PATH_SUFFIX:-} </del> export KUBE_GCE_NODE_PROJECT=${KUBE_GCE_NODE_PROJECT:-} export KUBE_GCE_NODE_IMAGE=${KUBE_GCE_NODE_IMAGE:-} export KUBE_OS_DISTRIBUTION=${KUBE_OS_DISTRIBUTION:-}


KUBE_PROXY_TOKEN: $(yaml-quote ${KUBE_PROXY_TOKEN:-}) ADMISSION_CONTROL: $(yaml-quote ${ADMISSION_CONTROL:-}) MASTER_IP_RANGE: $(yaml-quote ${MASTER_IP_RANGE}) <del> CA_CERT: $(yaml-quote ${CA_CERT_BASE64}) </del> <ins> CA_CERT: $(yaml-quote ${CA_CERT_BASE64:-}) </ins> EOF if [[ "${master}" == "true" ]]; then


// available volumes await a claim case api.VolumeAvailable: <ins> // TODO:  remove api.VolumePending phase altogether _, exists, err := volumeIndex.Get(volume) if err != nil { return err } if !exists { volumeIndex.Add(volume) } </ins> if volume.Spec.ClaimRef != nil { _, err := binderClient.GetPersistentVolumeClaim(volume.Spec.ClaimRef.Namespace, volume.Spec.ClaimRef.Name) if err == nil {


} } <ins> func TestMissingFromIndex(t *testing.T) { api.ForTesting_ReferencesAllowBlankSelfLinks = true o := testclient.NewObjects(api.Scheme, api.Scheme) if err := testclient.AddObjectsFromPath("../../examples/persistent-volumes/claims/claim-01.yaml", o, api.Scheme); err != nil { t.Fatal(err) } if err := testclient.AddObjectsFromPath("../../examples/persistent-volumes/volumes/local-01.yaml", o, api.Scheme); err != nil { t.Fatal(err) } client := &testclient.Fake{ReactFn: testclient.ObjectReaction(o, latest.RESTMapper)} pv, err := client.PersistentVolumes().Get("any") if err != nil { t.Error("Unexpected error getting PV from client: %v", err) } claim, error := client.PersistentVolumeClaims("ns").Get("any") if error != nil { t.Errorf("Unexpected error getting PVC from client: %v", err) } volumeIndex := NewPersistentVolumeOrderedIndex() mockClient := &mockBinderClient{ volume: pv, claim:  claim, } // the default value of the PV is Pending. // if has previously been processed by the binder, it's status in etcd would be Available. // Only Pending volumes were being indexed and made ready for claims. pv.Status.Phase = api.VolumeAvailable // adds the volume to the index, making the volume available syncVolume(volumeIndex, mockClient, pv) if pv.Status.Phase != api.VolumeAvailable { t.Errorf("Expected phase %s but got %s", api.VolumeBound, pv.Status.Phase) } // an initial sync for a claim will bind it to an unbound volume, triggers state change err = syncClaim(volumeIndex, mockClient, claim) if err != nil { t.Fatalf("Expected Clam to be bound, instead got an error: %+vn", err) } // state change causes another syncClaim to update statuses syncClaim(volumeIndex, mockClient, claim) // claim updated volume's status, causing an update and syncVolume call syncVolume(volumeIndex, mockClient, pv) if pv.Spec.ClaimRef == nil { t.Errorf("Expected ClaimRef but got nil for pv.Status.ClaimRef: %+vn", pv) } if pv.Status.Phase != api.VolumeBound { t.Errorf("Expected phase %s but got %s", api.VolumeBound, pv.Status.Phase) } if claim.Status.Phase != api.ClaimBound { t.Errorf("Expected phase %s but got %s", api.ClaimBound, claim.Status.Phase) } if len(claim.Status.AccessModes) != len(pv.Spec.AccessModes) { t.Errorf("Expected phase %s but got %s", api.ClaimBound, claim.Status.Phase) } if claim.Status.AccessModes[0] != pv.Spec.AccessModes[0] { t.Errorf("Expected access mode %s but got %s", claim.Status.AccessModes[0], pv.Spec.AccessModes[0]) } // pretend the user deleted their claim mockClient.claim = nil syncVolume(volumeIndex, mockClient, pv) if pv.Status.Phase != api.VolumeReleased { t.Errorf("Expected phase %s but got %s", api.VolumeReleased, pv.Status.Phase) } } </ins> type mockBinderClient struct { volume *api.PersistentVolume claim  *api.PersistentVolumeClaim


<ins> /* Copyright 2015 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package operationmanager import ( "fmt" "sync" ) // Operation Manager is a thread-safe interface for keeping track of multiple pending async operations. type OperationManager interface { // Called when the operation with the given ID has started. // Creates a new channel with specified buffer size tracked with the specified ID. // Returns a read-only version of the newly created channel. // Returns an error if an entry with the specified ID already exists (previous entry must be removed by calling Close). Start(id string, bufferSize uint) (<-chan interface{}, error) // Called when the operation with the given ID has terminated. // Closes and removes the channel associated with ID. // Returns an error if no associated channel exists. Close(id string) error // Attempts to send msg to the channel associated with ID. // Returns an error if no associated channel exists. Send(id string, msg interface{}) error } // Returns a new instance of a channel manager. func NewOperationManager() OperationManager { return &operationManager{ chanMap: make(map[string]chan interface{}), } } type operationManager struct { sync.RWMutex chanMap map[string]chan interface{} } // Called when the operation with the given ID has started. // Creates a new channel with specified buffer size tracked with the specified ID. // Returns a read-only version of the newly created channel. // Returns an error if an entry with the specified ID already exists (previous entry must be removed by calling Close). func (cm *operationManager) Start(id string, bufferSize uint) (<-chan interface{}, error) { cm.Lock() defer cm.Unlock() if _, exists := cm.chanMap[id]; exists { return nil, fmt.Errorf("id %q already exists", id) } cm.chanMap[id] = make(chan interface{}, bufferSize) return cm.chanMap[id], nil } // Called when the operation with the given ID has terminated. // Closes and removes the channel associated with ID. // Returns an error if no associated channel exists. func (cm *operationManager) Close(id string) error { cm.Lock() defer cm.Unlock() if _, exists := cm.chanMap[id]; !exists { return fmt.Errorf("id %q not found", id) } close(cm.chanMap[id]) delete(cm.chanMap, id) return nil } // Attempts to send msg to the channel associated with ID. // Returns an error if no associated channel exists. func (cm *operationManager) Send(id string, msg interface{}) error { cm.RLock() defer cm.RUnlock() if _, exists := cm.chanMap[id]; !exists { return fmt.Errorf("id %q not found", id) } cm.chanMap[id] <- msg return nil } </ins>


<ins> /* Copyright 2015 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Channel Manager keeps track of multiple channels package operationmanager import ( "testing" ) func TestStart(t *testing.T) { // Arrange cm := NewOperationManager() chanId := "testChanId" testMsg := "test message" // Act ch, startErr := cm.Start(chanId, 1 /* bufferSize */) sigErr := cm.Send(chanId, testMsg) // Assert if startErr != nil { t.Fatalf("Unexpected error on Start. Expected: <no error> Actual: <%v>", startErr) } if sigErr != nil { t.Fatalf("Unexpected error on Send. Expected: <no error> Actual: <%v>", sigErr) } if actual := <-ch; actual != testMsg { t.Fatalf("Unexpected testMsg value. Expected: <%v> Actual: <%v>", testMsg, actual) } } func TestStartIdExists(t *testing.T) { // Arrange cm := NewOperationManager() chanId := "testChanId" // Act _, startErr1 := cm.Start(chanId, 1 /* bufferSize */) _, startErr2 := cm.Start(chanId, 1 /* bufferSize */) // Assert if startErr1 != nil { t.Fatalf("Unexpected error on Start1. Expected: <no error> Actual: <%v>", startErr1) } if startErr2 == nil { t.Fatalf("Expected error on Start2. Expected: <id already exists error> Actual: <no error>") } } func TestStartAndAdd2Chans(t *testing.T) { // Arrange cm := NewOperationManager() chanId1 := "testChanId1" chanId2 := "testChanId2" testMsg1 := "test message 1" testMsg2 := "test message 2" // Act ch1, startErr1 := cm.Start(chanId1, 1 /* bufferSize */) ch2, startErr2 := cm.Start(chanId2, 1 /* bufferSize */) sigErr1 := cm.Send(chanId1, testMsg1) sigErr2 := cm.Send(chanId2, testMsg2) // Assert if startErr1 != nil { t.Fatalf("Unexpected error on Start1. Expected: <no error> Actual: <%v>", startErr1) } if startErr2 != nil { t.Fatalf("Unexpected error on Start2. Expected: <no error> Actual: <%v>", startErr2) } if sigErr1 != nil { t.Fatalf("Unexpected error on Send1. Expected: <no error> Actual: <%v>", sigErr1) } if sigErr2 != nil { t.Fatalf("Unexpected error on Send2. Expected: <no error> Actual: <%v>", sigErr2) } if actual := <-ch1; actual != testMsg1 { t.Fatalf("Unexpected testMsg value. Expected: <%v> Actual: <%v>", testMsg1, actual) } if actual := <-ch2; actual != testMsg2 { t.Fatalf("Unexpected testMsg value. Expected: <%v> Actual: <%v>", testMsg2, actual) } } func TestStartAndAdd2ChansAndClose(t *testing.T) { // Arrange cm := NewOperationManager() chanId1 := "testChanId1" chanId2 := "testChanId2" testMsg1 := "test message 1" testMsg2 := "test message 2" // Act ch1, startErr1 := cm.Start(chanId1, 1 /* bufferSize */) ch2, startErr2 := cm.Start(chanId2, 1 /* bufferSize */) sigErr1 := cm.Send(chanId1, testMsg1) sigErr2 := cm.Send(chanId2, testMsg2) cm.Close(chanId1) sigErr3 := cm.Send(chanId1, testMsg1) // Assert if startErr1 != nil { t.Fatalf("Unexpected error on Start1. Expected: <no error> Actual: <%v>", startErr1) } if startErr2 != nil { t.Fatalf("Unexpected error on Start2. Expected: <no error> Actual: <%v>", startErr2) } if sigErr1 != nil { t.Fatalf("Unexpected error on Send1. Expected: <no error> Actual: <%v>", sigErr1) } if sigErr2 != nil { t.Fatalf("Unexpected error on Send2. Expected: <no error> Actual: <%v>", sigErr2) } if sigErr3 == nil { t.Fatalf("Expected error on Send3. Expected: <error> Actual: <no error>", sigErr2) } if actual := <-ch1; actual != testMsg1 { t.Fatalf("Unexpected testMsg value. Expected: <%v> Actual: <%v>", testMsg1, actual) } if actual := <-ch2; actual != testMsg2 { t.Fatalf("Unexpected testMsg value. Expected: <%v> Actual: <%v>", testMsg2, actual) } } </ins>


package gce_pd import ( <del> "errors" </del> "fmt" "os" "path" <ins> "path/filepath" "strings" </ins> "time" "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider" "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/gce" <ins> "github.com/GoogleCloudPlatform/kubernetes/pkg/util" </ins> "github.com/GoogleCloudPlatform/kubernetes/pkg/util/exec" "github.com/GoogleCloudPlatform/kubernetes/pkg/util/mount" <ins> "github.com/GoogleCloudPlatform/kubernetes/pkg/util/operationmanager" </ins> "github.com/golang/glog" ) <ins> const ( diskByIdPath         = "/dev/disk/by-id/" diskGooglePrefix     = "google-" diskScsiGooglePrefix = "scsi-0Google_PersistentDisk_" diskPartitionSuffix  = "-part" diskSDPath           = "/dev/sd" diskSDPattern        = "/dev/sd*" maxChecks            = 10 maxRetries           = 10 checkSleepDuration   = time.Second ) // Singleton operation manager for managing detach clean up go routines var detachCleanupManager = operationmanager.NewOperationManager() </ins> type GCEDiskUtil struct{} // Attaches a disk specified by a volume.GCEPersistentDisk to the current kubelet. // Mounts the disk to it's global path. <del> func (util *GCEDiskUtil) AttachAndMountDisk(pd *gcePersistentDisk, globalPDPath string) error { </del> <ins> func (diskUtil *GCEDiskUtil) AttachAndMountDisk(pd *gcePersistentDisk, globalPDPath string) error { glog.V(5).Infof("AttachAndMountDisk(pd, %q) where pd is %#vrn", globalPDPath, pd) // Terminate any in progress verify detach go routines, this will block until the goroutine is ready to exit because the channel is unbuffered detachCleanupManager.Send(pd.pdName, true) sdBefore, err := filepath.Glob(diskSDPattern) if err != nil { glog.Errorf("Error filepath.Glob("%s"): %vrn", diskSDPattern, err) } sdBeforeSet := util.NewStringSet(sdBefore...) </ins> gce, err := cloudprovider.GetCloudProvider("gce", nil) if err != nil { return err } if err := gce.(*gce_cloud.GCECloud).AttachDisk(pd.pdName, pd.readOnly); err != nil { return err } <del> devicePaths := []string{ path.Join("/dev/disk/by-id/", "google-"+pd.pdName), path.Join("/dev/disk/by-id/", "scsi-0Google_PersistentDisk_"+pd.pdName), } if pd.partition != "" { for i, path := range devicePaths { devicePaths[i] = path + "-part" + pd.partition } } //TODO(jonesdl) There should probably be better method than busy-waiting here. numTries := 0 devicePath := "" // Wait for the disk device to be created for { for _, path := range devicePaths { _, err := os.Stat(path) if err == nil { devicePath = path break } if err != nil && !os.IsNotExist(err) { return err } } if devicePath != "" { break } numTries++ if numTries == 10 { return errors.New("Could not attach disk: Timeout after 10s") } time.Sleep(time.Second) </del> <ins> devicePath, err := verifyAttached(pd, sdBeforeSet, gce) if err != nil { return err </ins> } // Only mount the PD globally once.


func (util *GCEDiskUtil) DetachDisk(pd *gcePersistentDisk) error { // Unmount the global PD mount, which should be the only one. globalPDPath := makeGlobalPDName(pd.plugin.host, pd.pdName) <ins> glog.V(5).Infof("DetachDisk(pd) where pd is %#v and the globalPDPath is %qrn", pd, globalPDPath) // Terminate any in progress verify detach go routines, this will block until the goroutine is ready to exit because the channel is unbuffered detachCleanupManager.Send(pd.pdName, true) </ins> if err := pd.mounter.Unmount(globalPDPath); err != nil { return err }


if err := gce.(*gce_cloud.GCECloud).DetachDisk(pd.pdName); err != nil { return err } <ins> // Verify disk detached, retry if needed. go verifyDetached(pd, gce) return nil } // Verifys the disk device to be created has been succesffully attached, and retries if it fails. func verifyAttached(pd *gcePersistentDisk, sdBeforeSet util.StringSet, gce cloudprovider.Interface) (string, error) { devicePaths := getDiskByIdPaths(pd) for numRetries := 0; numRetries < maxRetries; numRetries++ { for numChecks := 0; numChecks < maxChecks; numChecks++ { if err := udevadmChangeToNewDrives(sdBeforeSet); err != nil { // udevadm errors should not block disk attachment, log and continue glog.Errorf("%v", err) } for _, path := range devicePaths { if pathExists, err := pathExists(path); err != nil { return "", err } else if pathExists { // A device path has succesfully been created for the PD glog.V(5).Infof("Succesfully attached GCE PD %q.", pd.pdName) return path, nil } } // Sleep then check again glog.V(5).Infof("Waiting for GCE PD %q to attach.", pd.pdName) time.Sleep(checkSleepDuration) } // Try attaching the disk again glog.Warningf("Timed out waiting for GCE PD %q to attach. Retrying attach.", pd.pdName) if err := gce.(*gce_cloud.GCECloud).AttachDisk(pd.pdName, pd.readOnly); err != nil { return "", err } } return "", fmt.Errorf("Could not attach GCE PD %q. Timeout waiting for mount paths to be created.", pd.pdName) } // Veify the specified persistent disk device has been succesfully detached, and retries if it fails. // This function is intended to be called asynchronously as a go routine. func verifyDetached(pd *gcePersistentDisk, gce cloudprovider.Interface) { defer util.HandleCrash() // Setting bufferSize to 0 so that when senders send, they are blocked until we recieve. This avoids the need to have a separate exit check. ch, err := detachCleanupManager.Start(pd.pdName, 0 /* bufferSize */) if err != nil { glog.Errorf("Error adding %q to detachCleanupManager: %v", pd.pdName, err) return } defer detachCleanupManager.Close(pd.pdName) devicePaths := getDiskByIdPaths(pd) for numRetries := 0; numRetries < maxRetries; numRetries++ { for numChecks := 0; numChecks < maxChecks; numChecks++ { select { case <-ch: glog.Warningf("Terminating GCE PD %q detach verification. Another attach/detach call was made for this PD.", pd.pdName) return default: allPathsRemoved := true for _, path := range devicePaths { if err := udevadmChangeToDrive(path); err != nil { // udevadm errors should not block disk detachment, log and continue glog.Errorf("%v", err) } if exists, err := pathExists(path); err != nil { glog.Errorf("Error check path: %v", err) return } else { allPathsRemoved = allPathsRemoved && !exists } } if allPathsRemoved { // All paths to the PD have been succefully removed glog.V(5).Infof("Succesfully detached GCE PD %q.", pd.pdName) return } // Sleep then check again glog.V(5).Infof("Waiting for GCE PD %q to detach.", pd.pdName) time.Sleep(checkSleepDuration) } } // Try detaching disk again glog.Warningf("Timed out waiting for GCE PD %q to detach. Retrying detach.", pd.pdName) if err := gce.(*gce_cloud.GCECloud).DetachDisk(pd.pdName); err != nil { glog.Errorf("Error on retry detach PD %q: %v", pd.pdName, err) return } } glog.Errorf("Could not detach GCE PD %q. One or more mount paths was not removed.", pd.pdName) } // Returns list of all /dev/disk/by-id/* paths for given PD. func getDiskByIdPaths(pd *gcePersistentDisk) []string { devicePaths := []string{ path.Join(diskByIdPath, diskGooglePrefix+pd.pdName), path.Join(diskByIdPath, diskScsiGooglePrefix+pd.pdName), } if pd.partition != "" { for i, path := range devicePaths { devicePaths[i] = path + diskPartitionSuffix + pd.partition } } return devicePaths } // Checks if the specified path exists func pathExists(path string) (bool, error) { _, err := os.Stat(path) if err == nil { return true, nil } else if os.IsNotExist(err) { return false, nil } else { return false, err } } // Calls "udevadm trigger --action=change" for newly created "/dev/sd*" drives (exist only in after set). // This is workaround for Issue #7972. Once the underlying issue has been resolved, this may be removed. func udevadmChangeToNewDrives(sdBeforeSet util.StringSet) error { sdAfter, err := filepath.Glob(diskSDPattern) if err != nil { return fmt.Errorf("Error filepath.Glob("%s"): %vrn", diskSDPattern, err) } for _, sd := range sdAfter { if !sdBeforeSet.Has(sd) { return udevadmChangeToDrive(sd) } } return nil } // Calls "udevadm trigger --action=change" on the specified drive. // drivePath must be the the block device path to trigger on, in the format "/dev/sd*", or a symlink to it. // This is workaround for Issue #7972. Once the underlying issue has been resolved, this may be removed. func udevadmChangeToDrive(drivePath string) error { glog.V(5).Infof("udevadmChangeToDrive: drive=%q", drivePath) // Evaluate symlink, if any drive, err := filepath.EvalSymlinks(drivePath) if err != nil { return fmt.Errorf("udevadmChangeToDrive: filepath.EvalSymlinks(%q) failed with %v.", drivePath, err) } glog.V(5).Infof("udevadmChangeToDrive: symlink path is %q", drive) // Check to make sure input is "/dev/sd*" if !strings.Contains(drive, diskSDPath) { return fmt.Errorf("udevadmChangeToDrive: expected input in the form "%s" but drive is %q.", diskSDPattern, drive) } // Call "udevadm trigger --action=change --property-match=DEVNAME=/dev/sd..." _, err = exec.New().Command( "udevadm", "trigger", "--action=change", fmt.Sprintf("--property-match=DEVNAME=%s", drive)).CombinedOutput() if err != nil { return fmt.Errorf("udevadmChangeToDrive: udevadm trigger failed for drive %q with %v.", drive, err) } </ins> return nil }


var _ = Describe("Probing container", func() { framework := Framework{BaseName: "container-probe"} var podClient client.PodInterface <del> probe := nginxProbeBuilder{} </del> <ins> probe := webserverProbeBuilder{} </ins> BeforeEach(func() { framework.beforeEach()


expectNoError(err) startTime := time.Now() <del> expectNoError(wait.Poll(poll, 90*time.Second, func() (bool, error) { </del> <ins> Expect(wait.Poll(poll, 90*time.Second, func() (bool, error) { </ins> p, err := podClient.Get(p.Name) if err != nil { return false, err } <del> return api.IsPodReady(p), nil })) </del> <ins> ready := api.IsPodReady(p) if !ready { Logf("pod is not yet ready; pod has phase %q.", p.Status.Phase) return false, nil } return true, nil })).NotTo(HaveOccurred(), "pod never became ready") </ins> if time.Since(startTime) < 30*time.Second { Failf("Pod became ready before it's initial delay")


isReady, err := podRunningReady(p) expectNoError(err) <del> Expect(isReady).To(BeTrue()) </del> <ins> Expect(isReady).To(BeTrue(), "pod should be ready") </ins> <del> Expect(getRestartCount(p) == 0).To(BeTrue()) </del> <ins> restartCount := getRestartCount(p) Expect(restartCount == 0).To(BeTrue(), "pod should have a restart count of 0 but got %v", restartCount) </ins> }) It("with readiness probe that fails should never be ready and never restart", func() {


expectNoError(err) isReady, err := podRunningReady(p) <del> Expect(isReady).NotTo(BeTrue()) </del> <ins> Expect(isReady).NotTo(BeTrue(), "pod should be not ready") </ins> <del> Expect(getRestartCount(p) == 0).To(BeTrue()) </del> <ins> restartCount := getRestartCount(p) Expect(restartCount == 0).To(BeTrue(), "pod should have a restart count of 0 but got %v", restartCount) </ins> }) })


func makePodSpec(readinessProbe, livenessProbe *api.Probe) *api.Pod { pod := &api.Pod{ <del> ObjectMeta: api.ObjectMeta{Name: "nginx-" + string(util.NewUUID())}, </del> <ins> ObjectMeta: api.ObjectMeta{Name: "test-webserver-" + string(util.NewUUID())}, </ins> Spec: api.PodSpec{ Containers: []api.Container{ { <del> Name:           "nginx", Image:          "nginx", </del> <ins> Name:           "test-webserver", Image:          "gcr.io/google_containers/test-webserver", </ins> LivenessProbe:  livenessProbe, ReadinessProbe: readinessProbe, },


return pod } <del> type nginxProbeBuilder struct { </del> <ins> type webserverProbeBuilder struct { </ins> failing      bool initialDelay bool } <del> func (b nginxProbeBuilder) withFailing() nginxProbeBuilder { </del> <ins> func (b webserverProbeBuilder) withFailing() webserverProbeBuilder { </ins> b.failing = true return b } <del> func (b nginxProbeBuilder) withInitialDelay() nginxProbeBuilder { </del> <ins> func (b webserverProbeBuilder) withInitialDelay() webserverProbeBuilder { </ins> b.initialDelay = true return b } <del> func (b nginxProbeBuilder) build() *api.Probe { </del> <ins> func (b webserverProbeBuilder) build() *api.Probe { </ins> probe := &api.Probe{ Handler: api.Handler{ HTTPGet: &api.HTTPGetAction{


// Remove unidentified containers. for _, container := range unidentifiedContainers { glog.Infof("Removing unidentified dead container %q with ID %q", container.name, container.id) <del> err = cgc.dockerClient.RemoveContainer(docker.RemoveContainerOptions{ID: container.id}) </del> <ins> err = cgc.dockerClient.RemoveContainer(docker.RemoveContainerOptions{ID: container.id, RemoveVolumes: true}) </ins> if err != nil { glog.Warningf("Failed to remove unidentified dead container %q: %v", container.name, err) }


// Remove from oldest to newest (last to first). numToKeep := len(containers) - toRemove for i := numToKeep; i < len(containers); i++ { <del> err := cgc.dockerClient.RemoveContainer(docker.RemoveContainerOptions{ID: containers[i].id}) </del> <ins> err := cgc.dockerClient.RemoveContainer(docker.RemoveContainerOptions{ID: containers[i].id, RemoveVolumes: true}) </ins> if err != nil { glog.Warningf("Failed to remove dead container %q: %v", containers[i].name, err) }


In one terminal, run `gcloud compute ssh master --ssh-flag="-L 8080:127.0.0.1:8080"` and in a second run `gcloud compute ssh master --ssh-flag="-R 8080:127.0.0.1:8080"`. <ins> ### OpenStack These instructions are for running on the command line.  Most of this you can also do through the Horizon dashboard. These instructions were tested on the Ice House release on a Metacloud distribution of OpenStack but should be similar if not the same across other versions/distributions of OpenStack. #### Make sure you can connect with OpenStack Make sure the environment variables are set for OpenStack such as: ```sh OS_TENANT_ID OS_PASSWORD OS_AUTH_URL OS_USERNAME OS_TENANT_NAME ``` Test this works with something like: ``` nova list ``` #### Get a Suitable CoreOS Image You'll need a [suitable version of CoreOS image for OpenStack] (https://coreos.com/os/docs/latest/booting-on-openstack.html) Once you download that, upload it to glance.  An example is shown below: ```sh glance image-create --name CoreOS723  --container-format bare --disk-format qcow2  --file coreos_production_openstack_image.img  --is-public True ``` #### Create security group ```sh nova secgroup-create kubernetes "Kubernetes Security Group" nova secgroup-add-rule kubernetes tcp 22 22   0.0.0.0/0 nova secgroup-add-rule kubernetes tcp 80 80   0.0.0.0/0 ``` #### Provision the Master ```sh nova boot  --image <image_name>  --key-name <my_key>  --flavor <flavor id>  --security-group kubernetes  --user-data files/master.yaml  kube-master ``` ```<image_name>``` is the CoreOS image name.  In our example we can use the image we created in the previous step and put in 'CoreOS723' ```<my_key>``` is the keypair name that you already generated to access the instance. ```<flavor_id>``` is the flavor ID you use to size the instance.  Run ```nova flavor-list``` to get the IDs.  3 on the system this was tested with gives the m1.large size. The important part is to ensure you have the files/master.yml as this is what will do all the post boot configuration. This path is relevant so we are assuming in this example that you are running the nova command in a directory where there is a subdirectory called files that has the master.yml file in it.  Absolute paths also work. Next, assign it a public IP address: ``` nova floating-ip-list ``` Get an IP address that's free and run: ``` nova floating-ip-associate kube-master <ip address> ``` where ```<ip address>``` is the IP address that was available from the ```nova floating-ip-list``` command. #### Provision Worker Nodes Edit ```node.yaml``` and replace all instances of ```<master-private-ip>``` with the private IP address of the master node.  You can get this by runnning ```nova show kube-master``` assuming you named your instance kube master.  This is not the floating IP address you just assigned it. ```sh nova boot  --image <image_name>  --key-name <my_key>  --flavor <flavor id>  --security-group kubernetes  --user-data files/node.yaml  minion01 ``` This is basically the same as the master nodes but with the node.yaml post-boot script instead of the master. </ins> ### VMware Fusion #### Create the master config-drive


Manager: m, } <del> // Export the HTTP endpoint if a port was specified. if port > 0 { err = cadvisorClient.exportHTTP(port) if err != nil { return nil, err } </del> <ins> err = cadvisorClient.exportHTTP(port) if err != nil { return nil, err </ins> } return cadvisorClient, nil }


} func (cc *cadvisorClient) exportHTTP(port uint) error { <ins> // Register the handlers regardless as this registers the prometheus // collector properly. </ins> mux := http.NewServeMux() err := cadvisorHttp.RegisterHandlers(mux, cc, "", "", "", "", "/metrics") if err != nil { return err } <del> serv := &http.Server{ Addr:    fmt.Sprintf(":%d", port), Handler: mux, } // TODO(vmarmol): Remove this when the cAdvisor port is once again free. // If export failed, retry in the background until we are able to bind. // This allows an existing cAdvisor to be killed before this one registers. go func() { defer util.HandleCrash() err := serv.ListenAndServe() for err != nil { glog.Infof("Failed to register cAdvisor on port %d, retrying. Error: %v", port, err) time.Sleep(time.Minute) err = serv.ListenAndServe() </del> <ins> // Only start the http server if port > 0 if port > 0 { serv := &http.Server{ Addr:    fmt.Sprintf(":%d", port), Handler: mux, </ins> } <del> }() </del> <ins> // TODO(vmarmol): Remove this when the cAdvisor port is once again free. // If export failed, retry in the background until we are able to bind. // This allows an existing cAdvisor to be killed before this one registers. go func() { defer util.HandleCrash() err := serv.ListenAndServe() for err != nil { glog.Infof("Failed to register cAdvisor on port %d, retrying. Error: %v", port, err) time.Sleep(time.Minute) err = serv.ListenAndServe() } }() } </ins> return nil }


# KUBE_TEST_API_VERSIONS=${KUBE_TEST_API_VERSIONS:-"v1,experimental/v1alpha1"} KUBE_TEST_API_VERSIONS=${KUBE_TEST_API_VERSIONS:-"v1,experimental/v1alpha1"} <ins> # Give integration tests longer to run KUBE_TIMEOUT=${KUBE_TIMEOUT:--timeout 240s} </ins> KUBE_INTEGRATION_TEST_MAX_CONCURRENCY=${KUBE_INTEGRATION_TEST_MAX_CONCURRENCY:-"-1"} LOG_LEVEL=${LOG_LEVEL:-2}


# KUBE_RACE="-race" KUBE_GOFLAGS="-tags 'integration no-docker' "  KUBE_RACE=""  <ins> KUBE_TIMEOUT="${KUBE_TIMEOUT}"  </ins> KUBE_TEST_API_VERSIONS="$1"  KUBE_API_VERSIONS="v1,experimental/v1alpha1"  "${KUBE_ROOT}/hack/test-go.sh" test/integration


func rcByNameContainer(name string, replicas int, image string, labels map[string]string, c api.Container) *api.ReplicationController { // Add "name": name to the labels, overwriting if it exists. labels["name"] = name <ins> gracePeriod := int64(0) </ins> return &api.ReplicationController{ TypeMeta: unversioned.TypeMeta{ Kind:       "ReplicationController",


Labels: labels, }, Spec: api.PodSpec{ <del> Containers: []api.Container{c}, </del> <ins> Containers:                    []api.Container{c}, TerminationGracePeriodSeconds: &gracePeriod, </ins> }, }, },


// In case of failure or too long waiting time, an error is returned. func waitForRCPodToDisappear(c *client.Client, ns, rcName, podName string) error { label := labels.SelectorFromSet(labels.Set(map[string]string{"name": rcName})) <del> return waitForPodToDisappear(c, ns, podName, label, 20*time.Second, 5*time.Minute) </del> <ins> // NodeController evicts pod after 5 minutes, so we need timeout greater than that. // Additionally, there can be non-zero grace period, so we are setting 10 minutes // to be on the safe size. return waitForPodToDisappear(c, ns, podName, label, 20*time.Second, 10*time.Minute) </ins> } // waitForService waits until the service appears (exist == true), or disappears (exist == false)


return fmt.Errorf("pod %s is not running and cannot be attached to; current phase is %s", p.PodName, pod.Status.Phase) } <del> containerName := p.ContainerName if len(containerName) == 0 { glog.V(4).Infof("defaulting container name to %s", pod.Spec.Containers[0].Name) containerName = pod.Spec.Containers[0].Name } </del> // TODO: refactor with terminal helpers from the edit utility once that is merged var stdin io.Reader tty := p.TTY


Name(pod.Name). Namespace(pod.Namespace). SubResource("attach"). <del> Param("container", containerName) </del> <ins> Param("container", p.GetContainerName(pod)) </ins> return p.Attach.Attach(req, p.Config, stdin, p.Out, p.Err, tty) } <ins> // GetContainerName returns the name of the container to attach to, with a fallback. func (p *AttachOptions) GetContainerName(pod *api.Pod) string { if len(p.ContainerName) > 0 { return p.ContainerName } glog.V(4).Infof("defaulting container name to %s", pod.Spec.Containers[0].Name) return pod.Spec.Containers[0].Name } </ins>


return err } if status == api.PodSucceeded || status == api.PodFailed { <del> return handleLog(c, pod.Namespace, pod.Name, &api.PodLogOptions{Container: pod.Spec.Containers[0].Name}, opts.Out) </del> <ins> return handleLog(c, pod.Namespace, pod.Name, &api.PodLogOptions{Container: opts.GetContainerName(pod)}, opts.Out) </ins> } opts.Client = c opts.PodName = pod.Name opts.Namespace = pod.Namespace <del> return opts.Run() </del> <ins> if err := opts.Run(); err != nil { fmt.Fprintf(opts.Out, "Error attaching, falling back to logs: %vn", err) return handleLog(c, pod.Namespace, pod.Name, &api.PodLogOptions{Container: opts.GetContainerName(pod)}, opts.Out) } return nil </ins> } func getRestartPolicy(cmd *cobra.Command, interactive bool) (api.RestartPolicy, error) {


if containerChanges.StartInfraContainer && (len(containerChanges.ContainersToStart) > 0) { glog.V(4).Infof("Creating pod infra container for %q", podFullName) podInfraContainerID, err = dm.createPodInfraContainer(pod) <ins> if err != nil { glog.Errorf("Failed to create pod infra container: %v; Skipping pod %q", err, podFullName) return err } </ins> // Call the networking plugin <del> if err == nil { err = dm.networkPlugin.SetUpPod(pod.Namespace, pod.Name, podInfraContainerID) } </del> <ins> err = dm.networkPlugin.SetUpPod(pod.Namespace, pod.Name, podInfraContainerID) </ins> if err != nil { glog.Errorf("Failed to create pod infra container: %v; Skipping pod %q", err, podFullName) <ins> // Delete infra container if delErr := dm.KillContainerInPod(kubecontainer.ContainerID{ ID:   string(podInfraContainerID), Type: "docker"}, nil, pod); delErr != nil { glog.Warningf("Clear infra container failed for pod %q: %v", podFullName, delErr) } </ins> return err }


package flocker import ( <ins> "io/ioutil" "os" </ins> "testing" flockerClient "github.com/ClusterHQ/flocker-go"


const pluginName = "kubernetes.io/flocker" <del> func newInitializedVolumePlugMgr() volume.VolumePluginMgr { </del> <ins> func newInitializedVolumePlugMgr(t *testing.T) (volume.VolumePluginMgr, string) { </ins> plugMgr := volume.VolumePluginMgr{} <del> plugMgr.InitPlugins(ProbeVolumePlugins(), volume.NewFakeVolumeHost("/foo/bar", nil, nil)) return plugMgr </del> <ins> dir, err := ioutil.TempDir("", "flocker") assert.NoError(t, err) plugMgr.InitPlugins(ProbeVolumePlugins(), volume.NewFakeVolumeHost(dir, nil, nil)) return plugMgr, dir </ins> } func TestGetByName(t *testing.T) { assert := assert.New(t) <del> plugMgr := newInitializedVolumePlugMgr() </del> <ins> plugMgr, _ := newInitializedVolumePlugMgr(t) </ins> plug, err := plugMgr.FindPluginByName(pluginName) assert.NotNil(plug, "Can't find the plugin by name")


func TestCanSupport(t *testing.T) { assert := assert.New(t) <del> plugMgr := newInitializedVolumePlugMgr() </del> <ins> plugMgr, _ := newInitializedVolumePlugMgr(t) </ins> plug, err := plugMgr.FindPluginByName(pluginName) assert.NoError(err)


func TestNewBuilder(t *testing.T) { assert := assert.New(t) <del> plugMgr := newInitializedVolumePlugMgr() </del> <ins> plugMgr, _ := newInitializedVolumePlugMgr(t) </ins> plug, err := plugMgr.FindPluginByName(pluginName) assert.NoError(err)


assert := assert.New(t) <del> plugMgr := newInitializedVolumePlugMgr() </del> <ins> plugMgr, rootDir := newInitializedVolumePlugMgr(t) if rootDir != "" { defer os.RemoveAll(rootDir) } </ins> plug, err := plugMgr.FindPluginByName(flockerPluginName) assert.NoError(err)


kube::test::get_object_assert 'rc mock2' "{{${labels_field}.status}}" 'replaced' fi fi <ins> # Command: kubectl edit multiple resources temp_editor="${KUBE_TEMP}/tmp-editor.sh" echo -e '#!/bin/bashnsed -i "s/status: replaced/status: edited/g" $@' > "${temp_editor}" chmod +x "${temp_editor}" EDITOR="${temp_editor}" kubectl edit "${kube_flags[@]}" -f "${file}" # Post-condition: mock service (and mock2) and mock rc (and mock2) are edited if [ "$has_svc" = true ]; then kube::test::get_object_assert 'services mock' "{{${labels_field}.status}}" 'edited' if [ "$two_svcs" = true ]; then kube::test::get_object_assert 'services mock2' "{{${labels_field}.status}}" 'edited' fi fi if [ "$has_rc" = true ]; then kube::test::get_object_assert 'rc mock' "{{${labels_field}.status}}" 'edited' if [ "$two_rcs" = true ]; then kube::test::get_object_assert 'rc mock2' "{{${labels_field}.status}}" 'edited' fi fi # cleaning rm "${temp_editor}" </ins> # Command # We need to set --overwrite, because otherwise, if the first attempt to run "kubectl label" # fails on some, but not all, of the resources, retries will fail because it tries to modify


} func (a genericAccessor) SetAnnotations(annotations map[string]string) { <ins> if a.annotations == nil { emptyAnnotations := make(map[string]string) a.annotations = &emptyAnnotations } </ins> *a.annotations = annotations }


defaultVersion := cmdutil.OutputVersion(cmd, clientConfig.Version) results := editResults{} for { <del> obj, err := resource.AsVersionedObject(infos, false, defaultVersion) </del> <ins> objs, err := resource.AsVersionedObjects(infos, defaultVersion) </ins> if err != nil { return preservedFile(err, results.file, out) } <ins> // if input object is a list, traverse and edit each item one at a time for _, obj := range objs { // TODO: add an annotating YAML printer that can print inline comments on each field, //   including descriptions or validation errors // generate the file to edit buf := &bytes.Buffer{} if err := results.header.writeTo(buf); err != nil { return preservedFile(err, results.file, out) } if err := printer.PrintObj(obj, buf); err != nil { return preservedFile(err, results.file, out) } original := buf.Bytes() </ins> <del> // TODO: add an annotating YAML printer that can print inline comments on each field, //   including descriptions or validation errors // generate the file to edit buf := &bytes.Buffer{} if err := results.header.writeTo(buf); err != nil { return preservedFile(err, results.file, out) } if err := printer.PrintObj(obj, buf); err != nil { return preservedFile(err, results.file, out) } original := buf.Bytes() // launch the editor edit := editor.NewDefaultEditor() edited, file, err := edit.LaunchTempFile("kubectl-edit-", ext, buf) if err != nil { return preservedFile(err, results.file, out) } </del> <ins> // launch the editor edit := editor.NewDefaultEditor() edited, file, err := edit.LaunchTempFile("kubectl-edit-", ext, buf) if err != nil { return preservedFile(err, results.file, out) } </ins> <del> // cleanup any file from the previous pass if len(results.file) > 0 { os.Remove(results.file) } </del> <ins> // cleanup any file from the previous pass if len(results.file) > 0 { os.Remove(results.file) } </ins> <del> glog.V(4).Infof("User edited:n%s", string(edited)) fmt.Printf("User edited:n%s", string(edited)) lines, err := hasLines(bytes.NewBuffer(edited)) if err != nil { return preservedFile(err, file, out) } if bytes.Equal(original, edited) { if len(results.edit) > 0 { preservedFile(nil, file, out) } else { os.Remove(file) </del> <ins> glog.V(4).Infof("User edited:n%s", string(edited)) lines, err := hasLines(bytes.NewBuffer(edited)) if err != nil { return preservedFile(err, file, out) </ins> } <del> fmt.Fprintln(out, "Edit cancelled, no changes made.") return nil } if !lines { if len(results.edit) > 0 { preservedFile(nil, file, out) } else { os.Remove(file) </del> <ins> // Compare content without comments if bytes.Equal(stripComments(original), stripComments(edited)) { if len(results.edit) > 0 { preservedFile(nil, file, out) } else { os.Remove(file) } fmt.Fprintln(out, "Edit cancelled, no changes made.") continue } if !lines { if len(results.edit) > 0 { preservedFile(nil, file, out) } else { os.Remove(file) } fmt.Fprintln(out, "Edit cancelled, saved file was empty.") continue </ins> } <del> fmt.Fprintln(out, "Edit cancelled, saved file was empty.") return nil } results = editResults{ file: file, } </del> <ins> results = editResults{ file: file, } </ins> <del> // parse the edited file updates, err := rmap.InfoForData(edited, "edited-file") if err != nil { return preservedFile(err, file, out) } </del> <ins> // parse the edited file updates, err := rmap.InfoForData(edited, "edited-file") if err != nil { return fmt.Errorf("The edited file had a syntax error: %v", err) } </ins> <del> // annotate the edited object for kubectl apply if err := kubectl.UpdateApplyAnnotation(updates); err != nil { return preservedFile(err, file, out) } </del> <ins> // annotate the edited object for kubectl apply if err := kubectl.UpdateApplyAnnotation(updates); err != nil { return preservedFile(err, file, out) } </ins> <del> visitor := resource.NewFlattenListVisitor(updates, rmap) </del> <ins> visitor := resource.NewFlattenListVisitor(updates, rmap) </ins> <del> // need to make sure the original namespace wasn't changed while editing if err = visitor.Visit(resource.RequireNamespace(cmdNamespace)); err != nil { return preservedFile(err, file, out) } </del> <ins> // need to make sure the original namespace wasn't changed while editing if err = visitor.Visit(resource.RequireNamespace(cmdNamespace)); err != nil { return preservedFile(err, file, out) } </ins> <del> // use strategic merge to create a patch originalJS, err := yaml.ToJSON(original) if err != nil { return preservedFile(err, file, out) } editedJS, err := yaml.ToJSON(edited) if err != nil { return preservedFile(err, file, out) } patch, err := strategicpatch.CreateStrategicMergePatch(originalJS, editedJS, obj) // TODO: change all jsonmerge to strategicpatch // for checking preconditions preconditions := []jsonmerge.PreconditionFunc{} if err != nil { glog.V(4).Infof("Unable to calculate diff, no merge is possible: %v", err) return preservedFile(err, file, out) } else { preconditions = append(preconditions, jsonmerge.RequireKeyUnchanged("apiVersion")) preconditions = append(preconditions, jsonmerge.RequireKeyUnchanged("kind")) preconditions = append(preconditions, jsonmerge.RequireMetadataKeyUnchanged("name")) results.version = defaultVersion } </del> <ins> // use strategic merge to create a patch originalJS, err := yaml.ToJSON(original) if err != nil { return preservedFile(err, file, out) } editedJS, err := yaml.ToJSON(edited) if err != nil { return preservedFile(err, file, out) } patch, err := strategicpatch.CreateStrategicMergePatch(originalJS, editedJS, obj) // TODO: change all jsonmerge to strategicpatch // for checking preconditions preconditions := []jsonmerge.PreconditionFunc{} if err != nil { glog.V(4).Infof("Unable to calculate diff, no merge is possible: %v", err) return preservedFile(err, file, out) } else { preconditions = append(preconditions, jsonmerge.RequireKeyUnchanged("apiVersion")) preconditions = append(preconditions, jsonmerge.RequireKeyUnchanged("kind")) preconditions = append(preconditions, jsonmerge.RequireMetadataKeyUnchanged("name")) results.version = defaultVersion } </ins> <del> if hold, msg := jsonmerge.TestPreconditionsHold(patch, preconditions); !hold { fmt.Fprintf(out, "error: %s", msg) return preservedFile(nil, file, out) } </del> <ins> if hold, msg := jsonmerge.TestPreconditionsHold(patch, preconditions); !hold { fmt.Fprintf(out, "error: %s", msg) return preservedFile(nil, file, out) } </ins> <del> err = visitor.Visit(func(info *resource.Info, err error) error { patched, err := resource.NewHelper(info.Client, info.Mapping).Patch(info.Namespace, info.Name, api.StrategicMergePatchType, patch) if err != nil { fmt.Fprintln(out, results.addError(err, info)) </del> <ins> err = visitor.Visit(func(info *resource.Info, err error) error { patched, err := resource.NewHelper(info.Client, info.Mapping).Patch(info.Namespace, info.Name, api.StrategicMergePatchType, patch) if err != nil { fmt.Fprintln(out, results.addError(err, info)) return nil } info.Refresh(patched, true) cmdutil.PrintSuccess(mapper, false, out, info.Mapping.Resource, info.Name, "edited") </ins> return nil <ins> }) if err != nil { return preservedFile(err, file, out) </ins> } <del> info.Refresh(patched, true) cmdutil.PrintSuccess(mapper, false, out, info.Mapping.Resource, info.Name, "edited") return nil }) if err != nil { return preservedFile(err, file, out) } if results.retryable > 0 { fmt.Fprintf(out, "You can run `kubectl replace -f %s` to try this update again.n", file) return errExit } if results.conflict > 0 { fmt.Fprintf(out, "You must update your local resource version and run `kubectl replace -f %s` to overwrite the remote changes.n", file) return errExit </del> <ins> if results.retryable > 0 { fmt.Fprintf(out, "You can run `kubectl replace -f %s` to try this update again.n", file) return errExit } if results.conflict > 0 { fmt.Fprintf(out, "You must update your local resource version and run `kubectl replace -f %s` to overwrite the remote changes.n", file) return errExit } if len(results.edit) == 0 { if results.notfound == 0 { os.Remove(file) } else { fmt.Fprintf(out, "The edits you made on deleted resources have been saved to %qn", file) } } </ins> } if len(results.edit) == 0 { <del> if results.notfound == 0 { os.Remove(file) } else { fmt.Fprintf(out, "The edits you made on deleted resources have been saved to %qn", file) } </del> return nil }


} return false, nil } <ins> // stripComments will transform a YAML file into JSON, thus dropping any comments // in it. Note that if the given file has a syntax error, the transformation will // fail and we will manually drop all comments from the file. func stripComments(file []byte) []byte { stripped, err := yaml.ToJSON(file) if err != nil { stripped = manualStrip(file) } return stripped } // manualStrip is used for dropping comments from a YAML file func manualStrip(file []byte) []byte { stripped := []byte{} for _, line := range bytes.Split(file, []byte("n")) { if bytes.HasPrefix(bytes.TrimSpace(line), []byte("#")) { continue } stripped = append(stripped, line...) stripped = append(stripped, 'n') } return stripped } </ins>


command := exec.Command("mount", mountArgs...) output, err := command.CombinedOutput() if err != nil { <del> glog.Errorf("Mount failed: %vnMounting arguments: %s %s %s %vnOutput: %sn", </del> <ins> return fmt.Errorf("Mount failed: %vnMounting arguments: %s %s %s %vnOutput: %sn", </ins> err, source, target, fstype, options, string(output)) } return err


command := exec.Command("umount", target) output, err := command.CombinedOutput() if err != nil { <del> glog.Errorf("Unmount failed: %vnUnmounting arguments: %snOutput: %sn", err, target, string(output)) return err </del> <ins> return fmt.Errorf("Unmount failed: %vnUnmounting arguments: %snOutput: %sn", err, target, string(output)) </ins> } return nil }


#   google         - Heapster, Google Cloud Monitoring, and Google Cloud Logging #   googleinfluxdb - Enable influxdb and google (except GCM) #   standalone     - Heapster only. Metrics available via Heapster REST API. <del> ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-googleinfluxdb}" </del> <ins> ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-influxdb}" </ins> # Optional: Enable node logging. ENABLE_NODE_LOGGING="${KUBE_ENABLE_NODE_LOGGING:-true}"


AUTOSCALER_MIN_NODES="${KUBE_AUTOSCALER_MIN_NODES:-1}" AUTOSCALER_MAX_NODES="${KUBE_AUTOSCALER_MAX_NODES:-${NUM_MINIONS}}" TARGET_NODE_UTILIZATION="${KUBE_TARGET_NODE_UTILIZATION:-0.7}" <ins> ENABLE_CLUSTER_MONITORING=googleinfluxdb </ins> fi # Optional: Enable deployment experimental feature, not ready for production use.


# Kubernetes Cluster Admin Guide: Cluster Components <ins> **Table of Contents** <!-- BEGIN MUNGE: GENERATED_TOC --> - [Kubernetes Cluster Admin Guide: Cluster Components](#kubernetes-cluster-admin-guide-cluster-components) - [Master Components](#master-components) - [kube-apiserver](#kube-apiserver) - [etcd](#etcd) - [kube-controller-manager](#kube-controller-manager) - [kube-scheduler](#kube-scheduler) - [addons](#addons) - [DNS](#dns) - [User interface](#user-interface) - [Container Resource Monitoring](#container-resource-monitoring) - [Cluster-level Logging](#cluster-level-logging) - [Node components](#node-components) - [kubelet](#kubelet) - [kube-proxy](#kube-proxy) - [docker](#docker) - [rkt](#rkt) - [monit](#monit) - [fluentd](#fluentd) <!-- END MUNGE: GENERATED_TOC --> </ins> This document outlines the various binary components that need to run to deliver a functioning Kubernetes cluster.


Addon objects are created in the "kube-system" namespace. <del> Example addons are: * [DNS](http://releases.k8s.io/HEAD/cluster/addons/dns/) provides cluster local DNS. * [kube-ui](http://releases.k8s.io/HEAD/cluster/addons/kube-ui/) provides a graphical UI for the cluster. * [fluentd-elasticsearch](http://releases.k8s.io/HEAD/cluster/addons/fluentd-elasticsearch/) provides log storage. Also see the [gcp version](http://releases.k8s.io/HEAD/cluster/addons/fluentd-gcp/). * [cluster-monitoring](http://releases.k8s.io/HEAD/cluster/addons/cluster-monitoring/) provides monitoring for the cluster. </del> <ins> #### DNS While the other addons are not strictly required, all Kubernetes clusters should have [cluster DNS](dns.md), as many examples rely on it. Cluster DNS is a DNS server, in addition to the other DNS server(s) in your environment, which serves DNS records for Kubernetes services. Containers started by Kubernetes automatically include this DNS server in their DNS searches. #### User interface The kube-ui provides a read-only overview of the cluster state.  Access [the UI using kubectl proxy](../user-guide/connecting-to-applications-proxy.md#connecting-to-the-kube-ui-service-from-your-local-workstation) #### Container Resource Monitoring [Container Resource Monitoring](../user-guide/monitoring.md) records generic time-series metrics about containers in a central database, and provides a UI for browsing that data. #### Cluster-level Logging [Container Logging](../user-guide/monitoring.md) saves container logs to a central log store with search/browsing interface.  There are two implementations: * [Cluster-level logging to Google Cloud Logging]( docs/user-guide/logging.md#cluster-level-logging-to-google-cloud-logging) * [Cluster-level Logging with Elasticsearch and Kibana]( docs/user-guide/logging.md#cluster-level-logging-with-elasticsearch-and-kibana) </ins> ## Node components


`monit` is a lightweight process babysitting system for keeping kubelet and docker running. <ins> ### fluentd `fluentd` is a daemon which helps provide [cluster-level logging](#cluster-level-logging). </ins> <!-- BEGIN MUNGE: GENERATED_ANALYTICS --> [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/admin/cluster-components.md?pixel)]()


- [Scheduler pod template](#scheduler-pod-template) - [Controller Manager Template](#controller-manager-template) - [Starting and Verifying Apiserver, Scheduler, and Controller Manager](#starting-and-verifying-apiserver-scheduler-and-controller-manager) <del> - [Logging](#logging) - [Monitoring](#monitoring) - [DNS](#dns) </del> <ins> - [Starting Cluster Services](#starting-cluster-services) </ins> - [Troubleshooting](#troubleshooting) - [Running validate-cluster](#running-validate-cluster) - [Inspect pods and services](#inspect-pods-and-services)


- Otherwise, if taking the firewall-based security approach - `--api-servers=http://$MASTER_IP` - `--config=/etc/kubernetes/manifests` <del> - `--cluster-dns=` to the address of the DNS server you will setup (see [Starting Addons](#starting-addons).) </del> <ins> - `--cluster-dns=` to the address of the DNS server you will setup (see [Starting Cluster Services](#starting-cluster-services).) </ins> - `--cluster-domain=` to the dns domain prefix to use for cluster DNS addresses. - `--docker-root=` - `--root-dir=`


You should soon be able to see all your nodes by running the `kubectl get nodes` command. Otherwise, you will need to manually create node objects. <del> ### Logging **TODO** talk about starting Logging. ### Monitoring **TODO** talk about starting Monitoring. ### DNS **TODO** talk about starting DNS. </del> <ins> ### Starting Cluster Services You will want to complete your Kubernetes clusters by adding cluster-wide services.  These are sometimes called *addons*, and [an overview of their purpose is in the admin guide]( ../../docs/admin/cluster-components.md#addons). Notes for setting up each cluster service are given below: * Cluster DNS: * required for many kubernetes examples * [Setup instructions](http://releases.k8s.io/HEAD/cluster/addons/dns/) * [Admin Guide](../admin/dns.md) * Cluster-level Logging * Multiple implementations with different storage backends and UIs. * [Elasticsearch Backend Setup Instructions](http://releases.k8s.io/HEAD/cluster/addons/fluentd-elasticsearch/) * [Google Cloud Logging Backend Setup Instructions](http://releases.k8s.io/HEAD/cluster/addons/fluentd-gcp/). * Both require running fluentd on each node. * [User Guide](../user-guide/logging.md) * Container Resource Monitoring * [Setup instructions](http://releases.k8s.io/HEAD/cluster/addons/cluster-monitoring/) * GUI * [Setup instructions](http://releases.k8s.io/HEAD/cluster/addons/kube-ui/) cluster. </ins> ## Troubleshooting


# Connecting to applications: kubectl proxy and apiserver proxy <ins> <!-- BEGIN MUNGE: GENERATED_TOC --> - [Connecting to applications: kubectl proxy and apiserver proxy](#connecting-to-applications-kubectl-proxy-and-apiserver-proxy) - [Getting the apiserver proxy URL of kube-ui](#getting-the-apiserver-proxy-url-of-kube-ui) - [Connecting to the kube-ui service from your local workstation](#connecting-to-the-kube-ui-service-from-your-local-workstation) <!-- END MUNGE: GENERATED_TOC --> </ins> You have seen the [basics](accessing-the-cluster.md) about `kubectl proxy` and `apiserver proxy`. This guide shows how to use them together to access a service([kube-ui](ui.md)) running on the Kubernetes cluster from your workstation.


# Logging <ins> **Table of Contents** <!-- BEGIN MUNGE: GENERATED_TOC --> - [Logging](#logging) - [Logging by Kubernetes Components](#logging-by-kubernetes-components) - [Examining the logs of running containers](#examining-the-logs-of-running-containers) - [Cluster level logging to Google Cloud Logging](#cluster-level-logging-to-google-cloud-logging) - [Cluster level logging with Elasticsearch and Kibana](#cluster-level-logging-with-elasticsearch-and-kibana) - [Ingesting Application Log Files](#ingesting-application-log-files) - [Known issues](#known-issues) <!-- END MUNGE: GENERATED_TOC --> </ins> ## Logging by Kubernetes Components Kubernetes components, such as kubelet and apiserver, use the [glog](https://godoc.org/github.com/golang/glog) logging library.  Developer conventions for logging severity are described in [docs/devel/logging.md](../devel/logging.md).


} if (opts.Watch || forceWatch) && rw != nil { <ins> glog.Infof("Started to log from %v for %v", ctx, req.Request.URL.RequestURI()) </ins> watcher, err := rw.Watch(ctx, &opts) if err != nil { scope.err(err, res.ResponseWriter, req.Request)


-o -path './test/e2e/*'  -o -path './test/e2e_node/*'  -o -path './test/integration/*'  <ins> -o -path './test/component/scheduler/perf/*'  </ins> ) -prune  ) -name '*_test.go' -print0 | xargs -0n1 dirname | sed 's|^./||' | sort -u )


PLEASE NOTE: This document applies to the HEAD of the source tree
 If you are using a released version of Kubernetes, you should refer to the docs that go with that version.  The latest release of this document can be found [here](http://releases.k8s.io/release-1.1/docs/proposals/choosing-scheduler.md). Documentation for other releases can be found at [releases.k8s.io](http://releases.k8s.io).  --   Scheduler Performance Test ====== Motivation ------ We already have a performance testing system -- Kubemark. However, Kubemark requires setting up and bootstrapping a whole cluster, which takes a lot of time. We want to have a standard way to reproduce scheduling latency metrics result and benchmark scheduler as simple and fast as possible. We have the following goals: - Save time on testing - The test and benchmark can be run in a single box. We only set up components necessary to scheduling without booting up a cluster. - Profiling runtime metrics to find out bottleneck - Write scheduler integration test but focus on performance measurement. Take advantage of go profiling tools and collect fine-grained metrics, like cpu-profiling, memory-profiling and block-profiling. - Reproduce test result easily - We want to have a known place to do the performance related test for scheduler. Developers should just run one script to collect all the information they need. Currently the test suite has the following: - density test (by adding a new Go test) - schedule 30k pods on 1000 (fake) nodes and 3k pods on 100 (fake) nodes - print out scheduling rate every second - let you learn the rate changes vs number of scheduled pods - benchmark - make use of `go test -bench` and report nanosecond/op. - schedule b.N pods when the cluster has N nodes and P scheduled pods. Since it takes relatively long time to finish one round, b.N is small: 10 - 100. How To Run ------ ``` cd kubernetes/test/component/scheduler/perf ./test-performance.sh ```  [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/test/component/scheduler/perf/README.md?pixel)]()


<ins> /* Copyright 2015 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package benchmark import ( "testing" "time" ) // BenchmarkScheduling100Nodes0Pods benchmarks the scheduling rate // when the cluster has 100 nodes and 0 scheduled pods func BenchmarkScheduling100Nodes0Pods(b *testing.B) { benchmarkScheduling(100, 0, b) } // BenchmarkScheduling100Nodes1000Pods benchmarks the scheduling rate // when the cluster has 100 nodes and 1000 scheduled pods func BenchmarkScheduling100Nodes1000Pods(b *testing.B) { benchmarkScheduling(100, 1000, b) } // BenchmarkScheduling1000Nodes0Pods benchmarks the scheduling rate // when the cluster has 1000 nodes and 0 scheduled pods func BenchmarkScheduling1000Nodes0Pods(b *testing.B) { benchmarkScheduling(1000, 0, b) } // BenchmarkScheduling1000Nodes1000Pods benchmarks the scheduling rate // when the cluster has 1000 nodes and 1000 scheduled pods func BenchmarkScheduling1000Nodes1000Pods(b *testing.B) { benchmarkScheduling(1000, 1000, b) } // benchmarkScheduling benchmarks scheduling rate with specific number of nodes // and specific number of pods already scheduled. Since an operation takes relatively // long time, b.N should be small: 10 - 100. func benchmarkScheduling(numNodes, numScheduledPods int, b *testing.B) { schedulerConfigFactory, finalFunc := mustSetupScheduler() defer finalFunc() c := schedulerConfigFactory.Client makeNodes(c, numNodes) makePods(c, numScheduledPods) for { scheduled := schedulerConfigFactory.ScheduledPodLister.Store.List() if len(scheduled) >= numScheduledPods { break } time.Sleep(1 * time.Second) } // start benchmark b.ResetTimer() makePods(c, b.N) for { // This can potentially affect performance of scheduler, since List() is done under mutex. // TODO: Setup watch on apiserver and wait until all pods scheduled. scheduled := schedulerConfigFactory.ScheduledPodLister.Store.List() if len(scheduled) >= numScheduledPods+b.N { break } // Note: This might introduce slight deviation in accuracy of benchmark results. // Since the total amount of time is relatively large, it might not be a concern. time.Sleep(100 * time.Millisecond) } } </ins>


<ins> /* Copyright 2015 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package benchmark import ( "fmt" "testing" "time" ) // TestSchedule100Node3KPods schedules 3k pods on 100 nodes. func TestSchedule100Node3KPods(t *testing.T) { schedulePods(100, 3000) } // TestSchedule1000Node30KPods schedules 30k pods on 1000 nodes. func TestSchedule1000Node30KPods(t *testing.T) { schedulePods(1000, 30000) } // schedulePods schedules specific number of pods on specific number of nodes. // This is used to learn the scheduling throughput on various // sizes of cluster and changes as more and more pods are scheduled. // It won't stop until all pods are scheduled. func schedulePods(numNodes, numPods int) { schedulerConfigFactory, destroyFunc := mustSetupScheduler() defer destroyFunc() c := schedulerConfigFactory.Client makeNodes(c, numNodes) makePods(c, numPods) prev := 0 start := time.Now() for { // This can potentially affect performance of scheduler, since List() is done under mutex. // Listing 10000 pods is an expensive operation, so running it frequently may impact scheduler. // TODO: Setup watch on apiserver and wait until all pods scheduled. scheduled := schedulerConfigFactory.ScheduledPodLister.Store.List() fmt.Printf("%dstrate: %dttotal: %dn", time.Since(start)/time.Second, len(scheduled)-prev, len(scheduled)) if len(scheduled) >= numPods { return } prev = len(scheduled) time.Sleep(1 * time.Second) } } </ins>


<ins> #!/usr/bin/env bash # Copyright 2014 The Kubernetes Authors All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -o errexit set -o nounset set -o pipefail pushd "../../../.." source "./hack/lib/util.sh" source "./hack/lib/logging.sh" source "./hack/lib/etcd.sh" popd cleanup() { kube::etcd::cleanup kube::log::status "performance test cleanup complete" } trap cleanup EXIT kube::etcd::start kube::log::status "performance test start" # TODO: set log-dir and prof output dir. DIR_BASENAME=$(basename `pwd`) go test -c -o "${DIR_BASENAME}.test" # We are using the benchmark suite to do profiling. Because it only runs a few pods and # theoretically it has less variance. "./${DIR_BASENAME}.test" -test.bench=. -test.run=xxxx -test.cpuprofile=prof.out -logtostderr=false kube::log::status "benchmark tests finished" # Running density tests. It might take a long time. "./${DIR_BASENAME}.test" -test.run=. -test.timeout=60m kube::log::status "density tests finished" </ins>


<ins> /* Copyright 2015 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package benchmark import ( "net/http" "net/http/httptest" "github.com/golang/glog" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/api/testapi" "k8s.io/kubernetes/pkg/client/record" client "k8s.io/kubernetes/pkg/client/unversioned" "k8s.io/kubernetes/pkg/master" "k8s.io/kubernetes/plugin/pkg/scheduler" _ "k8s.io/kubernetes/plugin/pkg/scheduler/algorithmprovider" "k8s.io/kubernetes/plugin/pkg/scheduler/factory" "k8s.io/kubernetes/test/integration/framework" ) // mustSetupScheduler starts the following components: // - k8s api server (a.k.a. master) // - scheduler // It returns scheduler config factory and destroyFunc which should be used to // remove resources after finished. // Notes on rate limiter: //   - The BindPodsRateLimiter is nil, meaning no rate limits. //   - client rate limit is set to 5000. func mustSetupScheduler() (schedulerConfigFactory *factory.ConfigFactory, destroyFunc func()) { framework.DeleteAllEtcdKeys() var m *master.Master masterConfig := framework.NewIntegrationTestMasterConfig() m = master.New(masterConfig) s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { m.Handler.ServeHTTP(w, req) })) c := client.NewOrDie(&client.Config{ Host:         s.URL, GroupVersion: testapi.Default.GroupVersion(), QPS:          5000.0, Burst:        5000, }) schedulerConfigFactory = factory.NewConfigFactory(c, nil) schedulerConfig, err := schedulerConfigFactory.Create() if err != nil { panic("Couldn't create scheduler config") } eventBroadcaster := record.NewBroadcaster() schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: "scheduler"}) eventBroadcaster.StartRecordingToSink(c.Events("")) scheduler.New(schedulerConfig).Run() destroyFunc = func() { glog.Infof("destroying") close(schedulerConfig.StopEverything) s.Close() glog.Infof("destroyed") } return } func makeNodes(c client.Interface, nodeCount int) { glog.Infof("making %d nodes", nodeCount) baseNode := &api.Node{ ObjectMeta: api.ObjectMeta{ GenerateName: "scheduler-test-node-", }, Spec: api.NodeSpec{ ExternalID: "foobar", }, Status: api.NodeStatus{ Capacity: api.ResourceList{ api.ResourcePods:   *resource.NewQuantity(32, resource.DecimalSI), api.ResourceCPU:    resource.MustParse("4"), api.ResourceMemory: resource.MustParse("32Gi"), }, Phase: api.NodeRunning, Conditions: []api.NodeCondition{ {Type: api.NodeReady, Status: api.ConditionTrue}, }, }, } for i := 0; i < nodeCount; i++ { if _, err := c.Nodes().Create(baseNode); err != nil { panic("error creating node: " + err.Error()) } } } // makePods will setup specified number of scheduled pods. // Currently it goes through scheduling path and it's very slow to setup large number of pods. // TODO: Setup pods evenly on all nodes and quickly/non-linearly. func makePods(c client.Interface, podCount int) { glog.Infof("making %d pods", podCount) basePod := &api.Pod{ ObjectMeta: api.ObjectMeta{ GenerateName: "scheduler-test-pod-", }, Spec: api.PodSpec{ Containers: []api.Container{{ Name:  "pause", Image: "gcr.io/google_containers/pause:1.0", Resources: api.ResourceRequirements{ Limits: api.ResourceList{ api.ResourceCPU:    resource.MustParse("100m"), api.ResourceMemory: resource.MustParse("500Mi"), }, Requests: api.ResourceList{ api.ResourceCPU:    resource.MustParse("100m"), api.ResourceMemory: resource.MustParse("500Mi"), }, }, }}, }, } threads := 30 remaining := make(chan int, 1000) go func() { for i := 0; i < podCount; i++ { remaining <- i } close(remaining) }() for i := 0; i < threads; i++ { go func() { for { _, ok := <-remaining if !ok { return } for { _, err := c.Pods("default").Create(basePod) if err == nil { break } } } }() } } </ins>


"fmt" "net/http" "net/http/httptest" <del> "sync" </del> "testing" "time"


} } } <del> func BenchmarkScheduling(b *testing.B) { framework.DeleteAllEtcdKeys() var m *master.Master s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { m.Handler.ServeHTTP(w, req) })) defer s.Close() masterConfig := framework.NewIntegrationTestMasterConfig() m = master.New(masterConfig) c := client.NewOrDie(&client.Config{ Host:         s.URL, GroupVersion: testapi.Default.GroupVersion(), QPS:          5000.0, Burst:        5000, }) schedulerConfigFactory := factory.NewConfigFactory(c, nil) schedulerConfig, err := schedulerConfigFactory.Create() if err != nil { b.Fatalf("Couldn't create scheduler config: %v", err) } eventBroadcaster := record.NewBroadcaster() schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: "scheduler"}) eventBroadcaster.StartRecordingToSink(c.Events("")) scheduler.New(schedulerConfig).Run() defer close(schedulerConfig.StopEverything) makeNNodes(c, 1000) N := b.N b.ResetTimer() makeNPods(c, N) for { objs := schedulerConfigFactory.ScheduledPodLister.Store.List() if len(objs) >= N { fmt.Printf("%v pods scheduled.n", len(objs)) /* // To prove that this actually works: for _, o := range objs { fmt.Printf("%sn", o.(*api.Pod).Spec.NodeName) } */ break } time.Sleep(time.Millisecond) } b.StopTimer() } func makeNNodes(c client.Interface, N int) { baseNode := &api.Node{ ObjectMeta: api.ObjectMeta{ GenerateName: "scheduler-test-node-", }, Spec: api.NodeSpec{ ExternalID: "foobar", }, Status: api.NodeStatus{ Capacity: api.ResourceList{ api.ResourcePods:   *resource.NewQuantity(32, resource.DecimalSI), api.ResourceCPU:    resource.MustParse("4"), api.ResourceMemory: resource.MustParse("32Gi"), }, Phase: api.NodeRunning, Conditions: []api.NodeCondition{ {Type: api.NodeReady, Status: api.ConditionTrue}, }, }, } for i := 0; i < N; i++ { if _, err := c.Nodes().Create(baseNode); err != nil { panic("error creating node: " + err.Error()) } } } func makeNPods(c client.Interface, N int) { basePod := &api.Pod{ ObjectMeta: api.ObjectMeta{ GenerateName: "scheduler-test-pod-", }, Spec: api.PodSpec{ Containers: []api.Container{{ Name:  "pause", Image: "gcr.io/google_containers/pause:1.0", Resources: api.ResourceRequirements{ Limits: api.ResourceList{ api.ResourceCPU:    resource.MustParse("100m"), api.ResourceMemory: resource.MustParse("500Mi"), }, Requests: api.ResourceList{ api.ResourceCPU:    resource.MustParse("100m"), api.ResourceMemory: resource.MustParse("500Mi"), }, }, }}, }, } wg := sync.WaitGroup{} threads := 30 wg.Add(threads) remaining := make(chan int, N) go func() { for i := 0; i < N; i++ { remaining <- i } close(remaining) }() for i := 0; i < threads; i++ { go func() { defer wg.Done() for { _, ok := <-remaining if !ok { return } for { _, err := c.Pods("default").Create(basePod) if err == nil { break } } } }() } wg.Wait() } </del>


func parseEnvs(envArray []string) ([]api.EnvVar, error) { envs := []api.EnvVar{} for _, env := range envArray { <del> parts := strings.Split(env, "=") if len(parts) != 2 || !validation.IsCIdentifier(parts[0]) || len(parts[1]) == 0 { </del> <ins> pos := strings.Index(env, "=") if pos == -1 { </ins> return nil, fmt.Errorf("invalid env: %v", env) } <del> envVar := api.EnvVar{Name: parts[0], Value: parts[1]} </del> <ins> name := env[:pos] value := env[pos+1:] if len(name) == 0 || !validation.IsCIdentifier(name) || len(value) == 0 { return nil, fmt.Errorf("invalid env: %v", env) } envVar := api.EnvVar{Name: name, Value: value} </ins> envs = append(envs, envVar) } return envs, nil


} } } <ins> func TestParseEnv(t *testing.T) { tests := []struct { envArray  []string expected  []api.EnvVar expectErr bool test      string }{ { envArray: []string{ "THIS_ENV=isOK", "HAS_COMMAS=foo,bar", "HAS_EQUALS=jJnro54iUu75xNy==", }, expected: []api.EnvVar{ { Name:  "THIS_ENV", Value: "isOK", }, { Name:  "HAS_COMMAS", Value: "foo,bar", }, { Name:  "HAS_EQUALS", Value: "jJnro54iUu75xNy==", }, }, expectErr: false, test:      "test case 1", }, { envArray: []string{ "WITH_OUT_EQUALS", }, expected:  []api.EnvVar{}, expectErr: true, test:      "test case 2", }, { envArray: []string{ "WITH_OUT_VALUES=", }, expected:  []api.EnvVar{}, expectErr: true, test:      "test case 3", }, { envArray: []string{ "=WITH_OUT_NAME", }, expected:  []api.EnvVar{}, expectErr: true, test:      "test case 4", }, } for _, test := range tests { envs, err := parseEnvs(test.envArray) if !test.expectErr && err != nil { t.Errorf("unexpected error: %v (%s)", err, test.test) } if test.expectErr && err != nil { continue } if !reflect.DeepEqual(envs, test.expected) { t.Errorf("nexpected:n%#vnsaw:n%#v (%s)", test.expected, envs, test.test) } } } </ins>


eventCorrelator := NewEventCorrelator(util.RealClock{}) return eventBroadcaster.StartEventWatcher( func(event *api.Event) { <del> // Make a copy before modification, because there could be multiple listeners. // Events are safe to copy like this. eventCopy := *event event = &eventCopy result, err := eventCorrelator.EventCorrelate(event) if err != nil { util.HandleError(err) } if result.Skip { return } tries := 0 for { if recordEvent(sink, result.Event, result.Patch, result.Event.Count > 1, eventCorrelator) { break } tries++ if tries >= maxTriesPerEvent { glog.Errorf("Unable to write event '%#v' (retry limit exceeded!)", event) break } // Randomize the first sleep so that various clients won't all be // synced up if the master goes down. if tries == 1 { time.Sleep(time.Duration(float64(sleepDuration) * randGen.Float64())) } else { time.Sleep(sleepDuration) } } </del> <ins> recordToSink(sink, event, eventCorrelator, randGen) </ins> }) } <ins> func recordToSink(sink EventSink, event *api.Event, eventCorrelator *EventCorrelator, randGen *rand.Rand) { // Make a copy before modification, because there could be multiple listeners. // Events are safe to copy like this. eventCopy := *event event = &eventCopy result, err := eventCorrelator.EventCorrelate(event) if err != nil { util.HandleError(err) } if result.Skip { return } tries := 0 for { if recordEvent(sink, result.Event, result.Patch, result.Event.Count > 1, eventCorrelator) { break } tries++ if tries >= maxTriesPerEvent { glog.Errorf("Unable to write event '%#v' (retry limit exceeded!)", event) break } // Randomize the first sleep so that various clients won't all be // synced up if the master goes down. if tries == 1 { time.Sleep(time.Duration(float64(sleepDuration) * randGen.Float64())) } else { time.Sleep(sleepDuration) } } } </ins> func isKeyNotFoundError(err error) bool { statusErr, _ := err.(*errors.StatusError) // At the moment the server is returning 500 instead of a more specific


import ( "encoding/json" "fmt" <del> "runtime" </del> <ins> "math/rand" </ins> "strconv" "testing" "time"


} func TestWriteEventError(t *testing.T) { <del> ref := &api.ObjectReference{ Kind:       "Pod", Name:       "foo", Namespace:  "baz", UID:        "bar", APIVersion: "version", } </del> type entry struct { timesToSendError int <del> attemptsMade     int </del> attemptsWanted   int err              error }


err:              fmt.Errorf("A weird error"), }, } <del> done := make(chan struct{}) eventBroadcaster := NewBroadcaster() defer eventBroadcaster.StartRecordingToSink( &testEventSink{ </del> <ins> eventCorrelator := NewEventCorrelator(util.RealClock{}) randGen := rand.New(rand.NewSource(time.Now().UnixNano())) for caseName, ent := range table { attempts := 0 sink := &testEventSink{ </ins> OnCreate: func(event *api.Event) (*api.Event, error) { <del> if event.Message == "finished" { close(done) return event, nil } item, ok := table[event.Message] if !ok { t.Errorf("Unexpected event: %#v", event) return event, nil } item.attemptsMade++ if item.attemptsMade < item.timesToSendError { return nil, item.err </del> <ins> attempts++ if attempts < ent.timesToSendError { return nil, ent.err </ins> } return event, nil }, <del> }, ).Stop() clock := &util.FakeClock{time.Now()} recorder := recorderWithFakeClock(api.EventSource{Component: "eventTest"}, eventBroadcaster, clock) for caseName := range table { clock.Step(1 * time.Second) recorder.Event(ref, api.EventTypeNormal, "Reason", caseName) runtime.Gosched() } recorder.Event(ref, api.EventTypeNormal, "Reason", "finished") <-done for caseName, item := range table { if e, a := item.attemptsWanted, item.attemptsMade; e != a { t.Errorf("case %v: wanted %v, got %v attempts", caseName, e, a) </del> <ins> } ev := &api.Event{} recordToSink(sink, ev, eventCorrelator, randGen) if attempts != ent.attemptsWanted { t.Errorf("case %v: wanted %d, got %d attempts", caseName, ent.attemptsWanted, attempts) </ins> } } }


var BindingSaturationReportInterval = 1 * time.Second var ( <del> E2eSchedulingLatency = prometheus.NewSummary( prometheus.SummaryOpts{ </del> <ins> E2eSchedulingLatency = prometheus.NewHistogram( prometheus.HistogramOpts{ </ins> Subsystem: schedulerSubsystem, Name:      "e2e_scheduling_latency_microseconds", Help:      "E2e scheduling latency (scheduling algorithm + binding)", <del> MaxAge:    time.Hour, </del> <ins> Buckets:   prometheus.ExponentialBuckets(1000, 2, 15), </ins> }, ) <del> SchedulingAlgorithmLatency = prometheus.NewSummary( prometheus.SummaryOpts{ </del> <ins> SchedulingAlgorithmLatency = prometheus.NewHistogram( prometheus.HistogramOpts{ </ins> Subsystem: schedulerSubsystem, Name:      "scheduling_algorithm_latency_microseconds", Help:      "Scheduling algorithm latency", <del> MaxAge:    time.Hour, </del> <ins> Buckets:   prometheus.ExponentialBuckets(1000, 2, 15), </ins> }, ) <del> BindingLatency = prometheus.NewSummary( prometheus.SummaryOpts{ </del> <ins> BindingLatency = prometheus.NewHistogram( prometheus.HistogramOpts{ </ins> Subsystem: schedulerSubsystem, Name:      "binding_latency_microseconds", Help:      "Binding latency", <del> MaxAge:    time.Hour, </del> <ins> Buckets:   prometheus.ExponentialBuckets(1000, 2, 15), </ins> }, ) BindingRateLimiterSaturation = prometheus.NewGauge(


glog.V(3).Infof("Attempting to schedule: %+v", pod) start := time.Now() <del> defer func() { metrics.E2eSchedulingLatency.Observe(metrics.SinceInMicroseconds(start)) }() </del> dest, err := s.config.Algorithm.Schedule(pod, s.config.NodeLister) <del> metrics.SchedulingAlgorithmLatency.Observe(metrics.SinceInMicroseconds(start)) </del> if err != nil { glog.V(1).Infof("Failed to schedule: %+v", pod) s.config.Recorder.Eventf(pod, api.EventTypeWarning, "FailedScheduling", "%v", err) s.config.Error(pod, err) return } <ins> metrics.SchedulingAlgorithmLatency.Observe(metrics.SinceInMicroseconds(start)) </ins> b := &api.Binding{ ObjectMeta: api.ObjectMeta{Namespace: pod.Namespace, Name: pod.Name}, Target: api.ObjectReference{


s.config.Modeler.LockedAction(func() { bindingStart := time.Now() err := s.config.Binder.Bind(b) <del> metrics.BindingLatency.Observe(metrics.SinceInMicroseconds(bindingStart)) </del> if err != nil { glog.V(1).Infof("Failed to bind pod: %+v", err) s.config.Recorder.Eventf(pod, api.EventTypeNormal, "FailedScheduling", "Binding rejected: %v", err) s.config.Error(pod, err) return } <ins> metrics.BindingLatency.Observe(metrics.SinceInMicroseconds(bindingStart)) </ins> s.config.Recorder.Eventf(pod, api.EventTypeNormal, "Scheduled", "Successfully assigned %v to %v", pod.Name, dest) // tell the model to assume that this binding took effect. assumed := *pod assumed.Spec.NodeName = dest s.config.Modeler.AssumePod(&assumed) }) <ins> metrics.E2eSchedulingLatency.Observe(metrics.SinceInMicroseconds(start)) </ins> }


apiVersion: v1 kind: ReplicationController metadata: <del> name: heapster-v11 </del> <ins> name: heapster-v12 </ins> namespace: kube-system labels: k8s-app: heapster <del> version: v11 </del> <ins> version: v12 </ins> kubernetes.io/cluster-service: "true" spec: replicas: 1 selector: k8s-app: heapster <del> version: v11 </del> <ins> version: v12 </ins> template: metadata: labels: k8s-app: heapster <del> version: v11 </del> <ins> version: v12 </ins> kubernetes.io/cluster-service: "true" spec: containers: <del> - image: gcr.io/google_containers/heapster:v0.18.4 </del> <ins> - image: gcr.io/google_containers/heapster:v0.18.5 </ins> name: heapster resources: # keep request = limit to keep this container in guaranteed class


By("restricting to a time range") time.Sleep(1500 * time.Millisecond) // ensure that startup logs on the node are seen as older than 1s <del> out = runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--since=1s") recent := len(strings.Split(out, "n")) out = runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--since=24h") older := len(strings.Split(out, "n")) Expect(recent).To(BeNumerically("<", older)) </del> <ins> recent_out := runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--since=1s") recent := len(strings.Split(recent_out, "n")) older_out := runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--since=24h") older := len(strings.Split(older_out, "n")) Expect(recent).To(BeNumerically("<", older), "expected recent(%v) to be less than older(%v)nrecent lines:n%vnolder lines:n%vn", recent, older, recent_out, older_out) </ins> }) }) })


readonly KUBE_NODE_BINARIES=("${KUBE_NODE_TARGETS[@]##*/}") readonly KUBE_NODE_BINARIES_WIN=("${KUBE_NODE_BINARIES[@]/%/.exe}") <del> if [[ "${KUBE_FASTBUILD:-}" == "true" ]]; then </del> <ins> if [[ -n "${KUBE_BUILD_PLATFORMS:-}" ]]; then readonly KUBE_SERVER_PLATFORMS=(${KUBE_BUILD_PLATFORMS}) readonly KUBE_NODE_PLATFORMS=(${KUBE_BUILD_PLATFORMS}) readonly KUBE_TEST_PLATFORMS=(${KUBE_BUILD_PLATFORMS}) readonly KUBE_CLIENT_PLATFORMS=(${KUBE_BUILD_PLATFORMS}) elif [[ "${KUBE_FASTBUILD:-}" == "true" ]]; then </ins> readonly KUBE_SERVER_PLATFORMS=(linux/amd64) readonly KUBE_NODE_PLATFORMS=(linux/amd64) if [[ "${KUBE_BUILDER_OS:-}" == "darwin"* ]]; then


pendingActionSet.Insert( strings.Join([]string{"delete-collection", "daemonsets", ""}, "-"), strings.Join([]string{"delete-collection", "deployments", ""}, "-"), <ins> strings.Join([]string{"delete-collection", "replicasets", ""}, "-"), </ins> strings.Join([]string{"delete-collection", "jobs", ""}, "-"), strings.Join([]string{"delete-collection", "horizontalpodautoscalers", ""}, "-"), strings.Join([]string{"delete-collection", "ingresses", ""}, "-"),


mockClient := fake.NewSimpleClientset(testInput.testNamespace) if containsVersion(versions, "extensions/v1beta1") { resources := []unversioned.APIResource{} <del> for _, resource := range []string{"daemonsets", "deployments", "jobs", "horizontalpodautoscalers", "ingresses"} { </del> <ins> for _, resource := range []string{"daemonsets", "deployments", "replicasets", "jobs", "horizontalpodautoscalers", "ingresses"} { </ins> resources = append(resources, unversioned.APIResource{Name: resource}) } mockClient.Resources = map[string]*unversioned.APIResourceList{


return estimate, err } } <ins> if containsResource(resources, "replicasets") { err = deleteReplicaSets(kubeClient.Extensions(), namespace) if err != nil { return estimate, err } } </ins> } return estimate, nil }


return expClient.Deployments(ns).DeleteCollection(nil, api.ListOptions{}) } <ins> func deleteReplicaSets(expClient extensions_unversioned.ExtensionsInterface, ns string) error { return expClient.ReplicaSets(ns).DeleteCollection(nil, api.ListOptions{}) } </ins> func deleteIngress(expClient extensions_unversioned.ExtensionsInterface, ns string) error { return expClient.Ingresses(ns).DeleteCollection(nil, api.ListOptions{}) }


func TestClient(t *testing.T) { _, s := framework.RunAMaster(t) <del> defer s.Close() </del> <ins> // TODO: Uncomment when fix #19254 // defer s.Close() </ins> ns := api.NamespaceDefault framework.DeleteAllEtcdKeys()


func TestSingleWatch(t *testing.T) { _, s := framework.RunAMaster(t) <del> defer s.Close() </del> <ins> // TODO: Uncomment when fix #19254 // defer s.Close() </ins> ns := "blargh" deleteAllEtcdKeys()


framework.DeleteAllEtcdKeys() defer framework.DeleteAllEtcdKeys() _, s := framework.RunAMaster(t) <del> defer s.Close() </del> <ins> // TODO: Uncomment when fix #19254 // defer s.Close() </ins> ns := api.NamespaceDefault client := client.NewOrDie(&client.Config{Host: s.URL, ContentConfig: client.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}})


func TestExperimentalPrefix(t *testing.T) { _, s := framework.RunAMaster(t) <del> defer s.Close() </del> <ins> // TODO: Uncomment when fix #19254 // defer s.Close() </ins> resp, err := http.Get(s.URL + "/apis/extensions/") if err != nil {


func TestWatchSucceedsWithoutArgs(t *testing.T) { _, s := framework.RunAMaster(t) <del> defer s.Close() </del> <ins> // TODO: Uncomment when fix #19254 // defer s.Close() </ins> resp, err := http.Get(s.URL + "/api/v1/namespaces?watch=1") if err != nil {


func TestAccept(t *testing.T) { _, s := framework.RunAMaster(t) <del> defer s.Close() </del> <ins> // TODO: Uncomment when fix #19254 // defer s.Close() </ins> resp, err := http.Get(s.URL + "/api/") if err != nil {


func TestMasterProcessMetrics(t *testing.T) { _, s := framework.RunAMaster(t) <del> defer s.Close() </del> <ins> // TODO: Uncomment when fix #19254 // defer s.Close() </ins> metrics, err := scrapeMetrics(s) if err != nil {


func TestApiserverMetrics(t *testing.T) { _, s := framework.RunAMaster(t) <del> defer s.Close() </del> <ins> // TODO: Uncomment when fix #19254 // defer s.Close() </ins> // Make a request to the apiserver to ensure there's at least one data point // for the metrics we're expecting -- otherwise, they won't be exported.


func TestPersistentVolumeRecycler(t *testing.T) { _, s := framework.RunAMaster(t) <del> defer s.Close() </del> <ins> // TODO: Uncomment when fix #19254 // defer s.Close() </ins> deleteAllEtcdKeys() // Use higher QPS and Burst, there is a test for race condition below, which


"apiVersion": "v1", "kind": "ResourceQuota", "metadata": { <del> "name": "quota", </del> <ins> "name": "quota" </ins> }, "spec": { "hard": {


"pods": "10", "services": "5", "replicationcontrollers":"20", <del> "resourcequotas":"1", }, </del> <ins> "resourcequotas":"1" } </ins> } } EOF


trigger scaling up and down the number of pods in your application. * New GUI (dashboard) allows you to get started quickly and enables the same functionality found in the CLI as a more approachable and discoverable way of  interacting with the system. Note: the GUI is eanbled by default for new cluster creation, however, it does not   interacting with the system. Note: the GUI is enabled by default in 1.2 clusters.        ## Other notable improvements


local output=`check-minion ${minion_ip}` echo $output if [[ "${output}" != "working" ]]; then <del> if (( attempt > 9 )); then </del> <ins> if (( attempt > 20 )); then </ins> echo echo -e "${color_red}Your cluster is unlikely to work correctly." >&2 echo "Please run ./cluster/kube-down.sh and re-create the" >&2


// ServerResourcesForGroupVersion returns the supported resources for a group and version. func (d *DiscoveryClient) ServerResourcesForGroupVersion(groupVersion string) (resources *unversioned.APIResourceList, err error) { url := url.URL{} <del> if groupVersion == "v1" { </del> <ins> if len(groupVersion) == 0 { return nil, fmt.Errorf("groupVersion shouldn't be empty") } else if groupVersion == "v1" { </ins> url.Path = "/api/" + groupVersion } else { url.Path = "/apis/" + groupVersion


"k8s.io/kubernetes/pkg/client/typed/discovery" "k8s.io/kubernetes/pkg/client/typed/dynamic" "k8s.io/kubernetes/pkg/runtime" <ins> utilerrors "k8s.io/kubernetes/pkg/util/errors" </ins> "k8s.io/kubernetes/pkg/util/sets" "github.com/golang/glog"


if err != nil { return results, err } <ins> allErrs := []error{} </ins> for _, apiGroup := range serverGroupList.Groups { preferredVersion := apiGroup.PreferredVersion apiResourceList, err := discoveryClient.ServerResourcesForGroupVersion(preferredVersion.GroupVersion) if err != nil { <del> return results, err </del> <ins> allErrs = append(allErrs, err) continue </ins> } groupVersion := unversioned.GroupVersion{Group: apiGroup.Name, Version: preferredVersion.Version} for _, apiResource := range apiResourceList.APIResources {


results = append(results, groupVersion.WithResource(apiResource.Name)) } } <del> return results, nil </del> <ins> return results, utilerrors.NewAggregate(allErrs) </ins> }


* `-f`: Resource file * also used for `--follow` in `logs`, but should be deprecated in favor of `-F` <ins> * `-n`: Namespace scope </ins> * `-l`: Label selector * also used for `--labels` in `expose`, but should be deprecated * `-L`: Label columns


kubectl create "${kube_flags[@]}" --namespace=other -f docs/admin/limitrange/valid-pod.yaml # Post-condition: valid-pod POD is created kube::test::get_object_assert 'pods --namespace=other' "{{range.items}}{{$id_field}}:{{end}}" 'valid-pod:' <ins> # Post-condition: verify shorthand `-n other` has the same results as `--namespace=other` kube::test::get_object_assert 'pods -n other' "{{range.items}}{{$id_field}}:{{end}}" 'valid-pod:' </ins> ### Delete POD valid-pod in specific namespace # Pre-condition: valid-pod POD exists


return ContextOverrideFlags{ ClusterName:  FlagInfo{prefix + FlagClusterName, "", "", "The name of the kubeconfig cluster to use"}, AuthInfoName: FlagInfo{prefix + FlagAuthInfoName, "", "", "The name of the kubeconfig user to use"}, <del> Namespace:    FlagInfo{prefix + FlagNamespace, "", "", "If present, the namespace scope for this CLI request"}, </del> <ins> Namespace:    FlagInfo{prefix + FlagNamespace, "n", "", "If present, the namespace scope for this CLI request"}, </ins> } }