Question:
When trying to edit multiple resources, for example: will panic. Ref as the possible cause cc
if was the cause, any fix needs to happen in 1.1 as well, was already cherry picked there
Also shows that we need a test for this case.
is the cause. When we edit multiple resources (i.e., list-type objects), the list's Accessor.annotations (type ) is nil and will panic when UpdateApplyAnnotation tries to SetAnnotations ().

defaultVersion := cmdutil.OutputVersion(cmd, clientConfig.Version) results := editResults{} for { <del> obj, err := resource.AsVersionedObject(infos, false, defaultVersion) </del> <ins> objs, err := resource.AsVersionedObjects(infos, defaultVersion) </ins> if err != nil { return preservedFile(err, results.file, out) } <ins> // if input object is a list, traverse and edit each item one at a time for _, obj := range objs { // TODO: add an annotating YAML printer that can print inline comments on each field, //   including descriptions or validation errors // generate the file to edit buf := &bytes.Buffer{} if err := results.header.writeTo(buf); err != nil { return preservedFile(err, results.file, out) } if err := printer.PrintObj(obj, buf); err != nil { return preservedFile(err, results.file, out) } original := buf.Bytes() </ins> <del> // TODO: add an annotating YAML printer that can print inline comments on each field, //   including descriptions or validation errors // generate the file to edit buf := &bytes.Buffer{} if err := results.header.writeTo(buf); err != nil { return preservedFile(err, results.file, out) } if err := printer.PrintObj(obj, buf); err != nil { return preservedFile(err, results.file, out) } original := buf.Bytes() // launch the editor edit := editor.NewDefaultEditor() edited, file, err := edit.LaunchTempFile("kubectl-edit-", ext, buf) if err != nil { return preservedFile(err, results.file, out) } </del> <ins> // launch the editor edit := editor.NewDefaultEditor() edited, file, err := edit.LaunchTempFile("kubectl-edit-", ext, buf) if err != nil { return preservedFile(err, results.file, out) } </ins> <del> // cleanup any file from the previous pass if len(results.file) > 0 { os.Remove(results.file) } </del> <ins> // cleanup any file from the previous pass if len(results.file) > 0 { os.Remove(results.file) } </ins> <del> glog.V(4).Infof("User edited:n%s", string(edited)) fmt.Printf("User edited:n%s", string(edited)) lines, err := hasLines(bytes.NewBuffer(edited)) if err != nil { return preservedFile(err, file, out) } if bytes.Equal(original, edited) { if len(results.edit) > 0 { preservedFile(nil, file, out) } else { os.Remove(file) </del> <ins> glog.V(4).Infof("User edited:n%s", string(edited)) lines, err := hasLines(bytes.NewBuffer(edited)) if err != nil { return preservedFile(err, file, out) </ins> } <del> fmt.Fprintln(out, "Edit cancelled, no changes made.") return nil } if !lines { if len(results.edit) > 0 { preservedFile(nil, file, out) } else { os.Remove(file) </del> <ins> // Compare content without comments if bytes.Equal(stripComments(original), stripComments(edited)) { if len(results.edit) > 0 { preservedFile(nil, file, out) } else { os.Remove(file) } fmt.Fprintln(out, "Edit cancelled, no changes made.") continue } if !lines { if len(results.edit) > 0 { preservedFile(nil, file, out) } else { os.Remove(file) } fmt.Fprintln(out, "Edit cancelled, saved file was empty.") continue </ins> } <del> fmt.Fprintln(out, "Edit cancelled, saved file was empty.") return nil } results = editResults{ file: file, } </del> <ins> results = editResults{ file: file, } </ins> <del> // parse the edited file updates, err := rmap.InfoForData(edited, "edited-file") if err != nil { return preservedFile(err, file, out) } </del> <ins> // parse the edited file updates, err := rmap.InfoForData(edited, "edited-file") if err != nil { return fmt.Errorf("The edited file had a syntax error: %v", err) } </ins> <del> // annotate the edited object for kubectl apply if err := kubectl.UpdateApplyAnnotation(updates); err != nil { return preservedFile(err, file, out) } </del> <ins> // annotate the edited object for kubectl apply if err := kubectl.UpdateApplyAnnotation(updates); err != nil { return preservedFile(err, file, out) } </ins> <del> visitor := resource.NewFlattenListVisitor(updates, rmap) </del> <ins> visitor := resource.NewFlattenListVisitor(updates, rmap) </ins> <del> // need to make sure the original namespace wasn't changed while editing if err = visitor.Visit(resource.RequireNamespace(cmdNamespace)); err != nil { return preservedFile(err, file, out) } </del> <ins> // need to make sure the original namespace wasn't changed while editing if err = visitor.Visit(resource.RequireNamespace(cmdNamespace)); err != nil { return preservedFile(err, file, out) } </ins> <del> // use strategic merge to create a patch originalJS, err := yaml.ToJSON(original) if err != nil { return preservedFile(err, file, out) } editedJS, err := yaml.ToJSON(edited) if err != nil { return preservedFile(err, file, out) } patch, err := strategicpatch.CreateStrategicMergePatch(originalJS, editedJS, obj) // TODO: change all jsonmerge to strategicpatch // for checking preconditions preconditions := []jsonmerge.PreconditionFunc{} if err != nil { glog.V(4).Infof("Unable to calculate diff, no merge is possible: %v", err) return preservedFile(err, file, out) } else { preconditions = append(preconditions, jsonmerge.RequireKeyUnchanged("apiVersion")) preconditions = append(preconditions, jsonmerge.RequireKeyUnchanged("kind")) preconditions = append(preconditions, jsonmerge.RequireMetadataKeyUnchanged("name")) results.version = defaultVersion } </del> <ins> // use strategic merge to create a patch originalJS, err := yaml.ToJSON(original) if err != nil { return preservedFile(err, file, out) } editedJS, err := yaml.ToJSON(edited) if err != nil { return preservedFile(err, file, out) } patch, err := strategicpatch.CreateStrategicMergePatch(originalJS, editedJS, obj) // TODO: change all jsonmerge to strategicpatch // for checking preconditions preconditions := []jsonmerge.PreconditionFunc{} if err != nil { glog.V(4).Infof("Unable to calculate diff, no merge is possible: %v", err) return preservedFile(err, file, out) } else { preconditions = append(preconditions, jsonmerge.RequireKeyUnchanged("apiVersion")) preconditions = append(preconditions, jsonmerge.RequireKeyUnchanged("kind")) preconditions = append(preconditions, jsonmerge.RequireMetadataKeyUnchanged("name")) results.version = defaultVersion } </ins> <del> if hold, msg := jsonmerge.TestPreconditionsHold(patch, preconditions); !hold { fmt.Fprintf(out, "error: %s", msg) return preservedFile(nil, file, out) } </del> <ins> if hold, msg := jsonmerge.TestPreconditionsHold(patch, preconditions); !hold { fmt.Fprintf(out, "error: %s", msg) return preservedFile(nil, file, out) } </ins> <del> err = visitor.Visit(func(info *resource.Info, err error) error { patched, err := resource.NewHelper(info.Client, info.Mapping).Patch(info.Namespace, info.Name, api.StrategicMergePatchType, patch) if err != nil { fmt.Fprintln(out, results.addError(err, info)) </del> <ins> err = visitor.Visit(func(info *resource.Info, err error) error { patched, err := resource.NewHelper(info.Client, info.Mapping).Patch(info.Namespace, info.Name, api.StrategicMergePatchType, patch) if err != nil { fmt.Fprintln(out, results.addError(err, info)) return nil } info.Refresh(patched, true) cmdutil.PrintSuccess(mapper, false, out, info.Mapping.Resource, info.Name, "edited") </ins> return nil <ins> }) if err != nil { return preservedFile(err, file, out) </ins> } <del> info.Refresh(patched, true) cmdutil.PrintSuccess(mapper, false, out, info.Mapping.Resource, info.Name, "edited") return nil }) if err != nil { return preservedFile(err, file, out) } if results.retryable > 0 { fmt.Fprintf(out, "You can run `kubectl replace -f %s` to try this update again.n", file) return errExit } if results.conflict > 0 { fmt.Fprintf(out, "You must update your local resource version and run `kubectl replace -f %s` to overwrite the remote changes.n", file) return errExit </del> <ins> if results.retryable > 0 { fmt.Fprintf(out, "You can run `kubectl replace -f %s` to try this update again.n", file) return errExit } if results.conflict > 0 { fmt.Fprintf(out, "You must update your local resource version and run `kubectl replace -f %s` to overwrite the remote changes.n", file) return errExit } if len(results.edit) == 0 { if results.notfound == 0 { os.Remove(file) } else { fmt.Fprintf(out, "The edits you made on deleted resources have been saved to %qn", file) } } </ins> } if len(results.edit) == 0 { <del> if results.notfound == 0 { os.Remove(file) } else { fmt.Fprintf(out, "The edits you made on deleted resources have been saved to %qn", file) } </del> return nil }


Question:
When trying to edit multiple resources, for example: will panic. Ref as the possible cause cc
if was the cause, any fix needs to happen in 1.1 as well, was already cherry picked there
Also shows that we need a test for this case.
is the cause. When we edit multiple resources (i.e., list-type objects), the list's Accessor.annotations (type ) is nil and will panic when UpdateApplyAnnotation tries to SetAnnotations ().

} return false, nil } <ins> // stripComments will transform a YAML file into JSON, thus dropping any comments // in it. Note that if the given file has a syntax error, the transformation will // fail and we will manually drop all comments from the file. func stripComments(file []byte) []byte { stripped, err := yaml.ToJSON(file) if err != nil { stripped = manualStrip(file) } return stripped } // manualStrip is used for dropping comments from a YAML file func manualStrip(file []byte) []byte { stripped := []byte{} for _, line := range bytes.Split(file, []byte("n")) { if bytes.HasPrefix(bytes.TrimSpace(line), []byte("#")) { continue } stripped = append(stripped, line...) stripped = append(stripped, 'n') } return stripped } </ins>


Question: We have to spend times on figuring out what are the mount error code means from issues and . In , the error message is and is great for diagnosing mount problem. The console message better be returned rather than just error code.

command := exec.Command("mount", mountArgs...) output, err := command.CombinedOutput() if err != nil { <del> glog.Errorf("Mount failed: %vnMounting arguments: %s %s %s %vnOutput: %sn", </del> <ins> return fmt.Errorf("Mount failed: %vnMounting arguments: %s %s %s %vnOutput: %sn", </ins> err, source, target, fstype, options, string(output)) } return err


Question: We have to spend times on figuring out what are the mount error code means from issues and . In , the error message is and is great for diagnosing mount problem. The console message better be returned rather than just error code.

command := exec.Command("umount", target) output, err := command.CombinedOutput() if err != nil { <del> glog.Errorf("Unmount failed: %vnUnmounting arguments: %snOutput: %sn", err, target, string(output)) return err </del> <ins> return fmt.Errorf("Unmount failed: %vnUnmounting arguments: %snOutput: %sn", err, target, string(output)) </ins> } return nil }


Question:
Heapster pod: And logs: cc p0 for triage
Did the required node scope change?
Its either the API on the console or monitoring write scopes on the node. , Mike Danese wrote:
I think this affects only GCE
I will fix this issue since is on vacation.
Need to cherry pick so i'll leave open
is out for review.
has been closed and the cherrypick here () has been merged so I'm closing this issue. Thanks everyone for the debugging.

#   google         - Heapster, Google Cloud Monitoring, and Google Cloud Logging #   googleinfluxdb - Enable influxdb and google (except GCM) #   standalone     - Heapster only. Metrics available via Heapster REST API. <del> ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-googleinfluxdb}" </del> <ins> ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-influxdb}" </ins> # Optional: Enable node logging. ENABLE_NODE_LOGGING="${KUBE_ENABLE_NODE_LOGGING:-true}"


Question:
Heapster pod: And logs: cc p0 for triage
Did the required node scope change?
Its either the API on the console or monitoring write scopes on the node. , Mike Danese wrote:
I think this affects only GCE
I will fix this issue since is on vacation.
Need to cherry pick so i'll leave open
is out for review.
has been closed and the cherrypick here () has been merged so I'm closing this issue. Thanks everyone for the debugging.

AUTOSCALER_MIN_NODES="${KUBE_AUTOSCALER_MIN_NODES:-1}" AUTOSCALER_MAX_NODES="${KUBE_AUTOSCALER_MAX_NODES:-${NUM_MINIONS}}" TARGET_NODE_UTILIZATION="${KUBE_TARGET_NODE_UTILIZATION:-0.7}" <ins> ENABLE_CLUSTER_MONITORING=googleinfluxdb </ins> fi # Optional: Enable deployment experimental feature, not ready for production use.


Question: has TODOs for how to bring up cluster addons like monitoring and logging. We should fill these in. requested by user:

# Kubernetes Cluster Admin Guide: Cluster Components <ins> **Table of Contents** <!-- BEGIN MUNGE: GENERATED_TOC --> - [Kubernetes Cluster Admin Guide: Cluster Components](#kubernetes-cluster-admin-guide-cluster-components) - [Master Components](#master-components) - [kube-apiserver](#kube-apiserver) - [etcd](#etcd) - [kube-controller-manager](#kube-controller-manager) - [kube-scheduler](#kube-scheduler) - [addons](#addons) - [DNS](#dns) - [User interface](#user-interface) - [Container Resource Monitoring](#container-resource-monitoring) - [Cluster-level Logging](#cluster-level-logging) - [Node components](#node-components) - [kubelet](#kubelet) - [kube-proxy](#kube-proxy) - [docker](#docker) - [rkt](#rkt) - [monit](#monit) - [fluentd](#fluentd) <!-- END MUNGE: GENERATED_TOC --> </ins> This document outlines the various binary components that need to run to deliver a functioning Kubernetes cluster.


Question: has TODOs for how to bring up cluster addons like monitoring and logging. We should fill these in. requested by user:

Addon objects are created in the "kube-system" namespace. <del> Example addons are: * [DNS](http://releases.k8s.io/HEAD/cluster/addons/dns/) provides cluster local DNS. * [kube-ui](http://releases.k8s.io/HEAD/cluster/addons/kube-ui/) provides a graphical UI for the cluster. * [fluentd-elasticsearch](http://releases.k8s.io/HEAD/cluster/addons/fluentd-elasticsearch/) provides log storage. Also see the [gcp version](http://releases.k8s.io/HEAD/cluster/addons/fluentd-gcp/). * [cluster-monitoring](http://releases.k8s.io/HEAD/cluster/addons/cluster-monitoring/) provides monitoring for the cluster. </del> <ins> #### DNS While the other addons are not strictly required, all Kubernetes clusters should have [cluster DNS](dns.md), as many examples rely on it. Cluster DNS is a DNS server, in addition to the other DNS server(s) in your environment, which serves DNS records for Kubernetes services. Containers started by Kubernetes automatically include this DNS server in their DNS searches. #### User interface The kube-ui provides a read-only overview of the cluster state.  Access [the UI using kubectl proxy](../user-guide/connecting-to-applications-proxy.md#connecting-to-the-kube-ui-service-from-your-local-workstation) #### Container Resource Monitoring [Container Resource Monitoring](../user-guide/monitoring.md) records generic time-series metrics about containers in a central database, and provides a UI for browsing that data. #### Cluster-level Logging [Container Logging](../user-guide/monitoring.md) saves container logs to a central log store with search/browsing interface.  There are two implementations: * [Cluster-level logging to Google Cloud Logging]( docs/user-guide/logging.md#cluster-level-logging-to-google-cloud-logging) * [Cluster-level Logging with Elasticsearch and Kibana]( docs/user-guide/logging.md#cluster-level-logging-with-elasticsearch-and-kibana) </ins> ## Node components


Question: has TODOs for how to bring up cluster addons like monitoring and logging. We should fill these in. requested by user:

`monit` is a lightweight process babysitting system for keeping kubelet and docker running. <ins> ### fluentd `fluentd` is a daemon which helps provide [cluster-level logging](#cluster-level-logging). </ins> <!-- BEGIN MUNGE: GENERATED_ANALYTICS --> [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/admin/cluster-components.md?pixel)]()


Question: has TODOs for how to bring up cluster addons like monitoring and logging. We should fill these in. requested by user:

- [Scheduler pod template](#scheduler-pod-template) - [Controller Manager Template](#controller-manager-template) - [Starting and Verifying Apiserver, Scheduler, and Controller Manager](#starting-and-verifying-apiserver-scheduler-and-controller-manager) <del> - [Logging](#logging) - [Monitoring](#monitoring) - [DNS](#dns) </del> <ins> - [Starting Cluster Services](#starting-cluster-services) </ins> - [Troubleshooting](#troubleshooting) - [Running validate-cluster](#running-validate-cluster) - [Inspect pods and services](#inspect-pods-and-services)


Question: has TODOs for how to bring up cluster addons like monitoring and logging. We should fill these in. requested by user:

- Otherwise, if taking the firewall-based security approach - `--api-servers=http://$MASTER_IP` - `--config=/etc/kubernetes/manifests` <del> - `--cluster-dns=` to the address of the DNS server you will setup (see [Starting Addons](#starting-addons).) </del> <ins> - `--cluster-dns=` to the address of the DNS server you will setup (see [Starting Cluster Services](#starting-cluster-services).) </ins> - `--cluster-domain=` to the dns domain prefix to use for cluster DNS addresses. - `--docker-root=` - `--root-dir=`


Question: has TODOs for how to bring up cluster addons like monitoring and logging. We should fill these in. requested by user:

You should soon be able to see all your nodes by running the `kubectl get nodes` command. Otherwise, you will need to manually create node objects. <del> ### Logging **TODO** talk about starting Logging. ### Monitoring **TODO** talk about starting Monitoring. ### DNS **TODO** talk about starting DNS. </del> <ins> ### Starting Cluster Services You will want to complete your Kubernetes clusters by adding cluster-wide services.  These are sometimes called *addons*, and [an overview of their purpose is in the admin guide]( ../../docs/admin/cluster-components.md#addons). Notes for setting up each cluster service are given below: * Cluster DNS: * required for many kubernetes examples * [Setup instructions](http://releases.k8s.io/HEAD/cluster/addons/dns/) * [Admin Guide](../admin/dns.md) * Cluster-level Logging * Multiple implementations with different storage backends and UIs. * [Elasticsearch Backend Setup Instructions](http://releases.k8s.io/HEAD/cluster/addons/fluentd-elasticsearch/) * [Google Cloud Logging Backend Setup Instructions](http://releases.k8s.io/HEAD/cluster/addons/fluentd-gcp/). * Both require running fluentd on each node. * [User Guide](../user-guide/logging.md) * Container Resource Monitoring * [Setup instructions](http://releases.k8s.io/HEAD/cluster/addons/cluster-monitoring/) * GUI * [Setup instructions](http://releases.k8s.io/HEAD/cluster/addons/kube-ui/) cluster. </ins> ## Troubleshooting


Question: has TODOs for how to bring up cluster addons like monitoring and logging. We should fill these in. requested by user:

# Connecting to applications: kubectl proxy and apiserver proxy <ins> <!-- BEGIN MUNGE: GENERATED_TOC --> - [Connecting to applications: kubectl proxy and apiserver proxy](#connecting-to-applications-kubectl-proxy-and-apiserver-proxy) - [Getting the apiserver proxy URL of kube-ui](#getting-the-apiserver-proxy-url-of-kube-ui) - [Connecting to the kube-ui service from your local workstation](#connecting-to-the-kube-ui-service-from-your-local-workstation) <!-- END MUNGE: GENERATED_TOC --> </ins> You have seen the [basics](accessing-the-cluster.md) about `kubectl proxy` and `apiserver proxy`. This guide shows how to use them together to access a service([kube-ui](ui.md)) running on the Kubernetes cluster from your workstation.


Question: has TODOs for how to bring up cluster addons like monitoring and logging. We should fill these in. requested by user:

# Logging <ins> **Table of Contents** <!-- BEGIN MUNGE: GENERATED_TOC --> - [Logging](#logging) - [Logging by Kubernetes Components](#logging-by-kubernetes-components) - [Examining the logs of running containers](#examining-the-logs-of-running-containers) - [Cluster level logging to Google Cloud Logging](#cluster-level-logging-to-google-cloud-logging) - [Cluster level logging with Elasticsearch and Kibana](#cluster-level-logging-with-elasticsearch-and-kibana) - [Ingesting Application Log Files](#ingesting-application-log-files) - [Known issues](#known-issues) <!-- END MUNGE: GENERATED_TOC --> </ins> ## Logging by Kubernetes Components Kubernetes components, such as kubelet and apiserver, use the [glog](https://godoc.org/github.com/golang/glog) logging library.  Developer conventions for logging severity are described in [docs/devel/logging.md](../devel/logging.md).


Question: Currently watches only log when they complete, which can make analyzing kube-apiserver log files a bit tricky. (did component x never start watching after that list or did apiserver shut down?)

} if (opts.Watch || forceWatch) && rw != nil { <ins> glog.Infof("Started to log from %v for %v", ctx, req.Request.URL.RequestURI()) </ins> watcher, err := rw.Watch(ctx, &opts) if err != nil { scope.err(err, res.ResponseWriter, req.Request)


Question:
We have found that on 1k nodes cluster scheduling takes 60-200 ms and becomes the bottleneck. We want to have a standard way to reproduce scheduling latency metrics result and benchmark scheduler as simple and fast as possible. Only with that can we: 1. figure out underlying behaviors and potential bottlenecks quickly; 2. publish results in a standard way and prevent future performance degradation. We have the following goals: Goal 1: Save time on testing. More accurate scheduling latency. We mock up kubelets and only measure scheduling latency and do not care about the latency to start the pod. This is different from Kubemark -- scheduled pods won’t be run by mocked kubelets. We only need to set up the master components. Goal 2: Profiling runtime metrics and find out performance bottleneck. Write scheduler integration test. Take advantage of go test and profiling tools, like cpu-profiling, memory-profiling and block-profiling. This is very fine-grained metrics. Goal 3: Reproduce test result easily. We want to have a known place to do the performance related test for scheduler. Developers should just run one script to collect all the information they need. We have the following plans for writing scheduler tests: density test (by adding a new Go test) schedule 30k pods on 1000 (fake) nodes and 3k pods on 100 (fake) nodes print out scheduling rate every second let you learn the rate changes vs number of scheduled pods benchmark (by modifying the existing benchmark in integration or simply add a new one) make use of and report nanosecond/op. schedule b.N pods when the cluster has N nodes and P scheduled pods. Since it takes relatively long time to finish one round, b.N is small -- level of 10. We are new to k8s codebase. We also want to know where is the best place (directory) to commit in these tests? Any suggestions are welcomed!
/cc
This extends
So density already does this... the one option it looks like you may want is custom control , real or fake is irrelevant as the e2e's just need to point to an api-server. re: benchmarks, seems like we could simply add to the unit-tests.
After chatting with lets add this as an agenda item for the SIG meeting tomorrow.
+1 for adding benchmark (howeveer this should be integration test, not unit-test in my opinion)
cc The next SIG on scheduler meeting is on 12.07, isn't it? If there is one tomorrow, can I have the information please?
retract, +1 to . I was adding this to the list given the context, but it crosses over into both.
Slides from today's meeting:
Here are are the action item notes from the SIG meeting: Everyone agrees this is a good idea and long overdue, but the question is where would this live: New category of tests? (isolated/sandbox’d/component) The current testing structure allows to easily append. create new directory (component) + hack/script to execute Future "we could" gate on known good data, and fail performance regressions
/cc
/cc

-o -path './test/e2e/*'  -o -path './test/e2e_node/*'  -o -path './test/integration/*'  <ins> -o -path './test/component/scheduler/perf/*'  </ins> ) -prune  ) -name '*_test.go' -print0 | xargs -0n1 dirname | sed 's|^./||' | sort -u )


Question:
We have found that on 1k nodes cluster scheduling takes 60-200 ms and becomes the bottleneck. We want to have a standard way to reproduce scheduling latency metrics result and benchmark scheduler as simple and fast as possible. Only with that can we: 1. figure out underlying behaviors and potential bottlenecks quickly; 2. publish results in a standard way and prevent future performance degradation. We have the following goals: Goal 1: Save time on testing. More accurate scheduling latency. We mock up kubelets and only measure scheduling latency and do not care about the latency to start the pod. This is different from Kubemark -- scheduled pods won’t be run by mocked kubelets. We only need to set up the master components. Goal 2: Profiling runtime metrics and find out performance bottleneck. Write scheduler integration test. Take advantage of go test and profiling tools, like cpu-profiling, memory-profiling and block-profiling. This is very fine-grained metrics. Goal 3: Reproduce test result easily. We want to have a known place to do the performance related test for scheduler. Developers should just run one script to collect all the information they need. We have the following plans for writing scheduler tests: density test (by adding a new Go test) schedule 30k pods on 1000 (fake) nodes and 3k pods on 100 (fake) nodes print out scheduling rate every second let you learn the rate changes vs number of scheduled pods benchmark (by modifying the existing benchmark in integration or simply add a new one) make use of and report nanosecond/op. schedule b.N pods when the cluster has N nodes and P scheduled pods. Since it takes relatively long time to finish one round, b.N is small -- level of 10. We are new to k8s codebase. We also want to know where is the best place (directory) to commit in these tests? Any suggestions are welcomed!
/cc
This extends
So density already does this... the one option it looks like you may want is custom control , real or fake is irrelevant as the e2e's just need to point to an api-server. re: benchmarks, seems like we could simply add to the unit-tests.
After chatting with lets add this as an agenda item for the SIG meeting tomorrow.
+1 for adding benchmark (howeveer this should be integration test, not unit-test in my opinion)
cc The next SIG on scheduler meeting is on 12.07, isn't it? If there is one tomorrow, can I have the information please?
retract, +1 to . I was adding this to the list given the context, but it crosses over into both.
Slides from today's meeting:
Here are are the action item notes from the SIG meeting: Everyone agrees this is a good idea and long overdue, but the question is where would this live: New category of tests? (isolated/sandbox’d/component) The current testing structure allows to easily append. create new directory (component) + hack/script to execute Future "we could" gate on known good data, and fail performance regressions
/cc
/cc

PLEASE NOTE: This document applies to the HEAD of the source tree
 If you are using a released version of Kubernetes, you should refer to the docs that go with that version.  The latest release of this document can be found [here](http://releases.k8s.io/release-1.1/docs/proposals/choosing-scheduler.md). Documentation for other releases can be found at [releases.k8s.io](http://releases.k8s.io).  --   Scheduler Performance Test ====== Motivation ------ We already have a performance testing system -- Kubemark. However, Kubemark requires setting up and bootstrapping a whole cluster, which takes a lot of time. We want to have a standard way to reproduce scheduling latency metrics result and benchmark scheduler as simple and fast as possible. We have the following goals: - Save time on testing - The test and benchmark can be run in a single box. We only set up components necessary to scheduling without booting up a cluster. - Profiling runtime metrics to find out bottleneck - Write scheduler integration test but focus on performance measurement. Take advantage of go profiling tools and collect fine-grained metrics, like cpu-profiling, memory-profiling and block-profiling. - Reproduce test result easily - We want to have a known place to do the performance related test for scheduler. Developers should just run one script to collect all the information they need. Currently the test suite has the following: - density test (by adding a new Go test) - schedule 30k pods on 1000 (fake) nodes and 3k pods on 100 (fake) nodes - print out scheduling rate every second - let you learn the rate changes vs number of scheduled pods - benchmark - make use of `go test -bench` and report nanosecond/op. - schedule b.N pods when the cluster has N nodes and P scheduled pods. Since it takes relatively long time to finish one round, b.N is small: 10 - 100. How To Run ------ ``` cd kubernetes/test/component/scheduler/perf ./test-performance.sh ```  [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/test/component/scheduler/perf/README.md?pixel)]()


Question:
We have found that on 1k nodes cluster scheduling takes 60-200 ms and becomes the bottleneck. We want to have a standard way to reproduce scheduling latency metrics result and benchmark scheduler as simple and fast as possible. Only with that can we: 1. figure out underlying behaviors and potential bottlenecks quickly; 2. publish results in a standard way and prevent future performance degradation. We have the following goals: Goal 1: Save time on testing. More accurate scheduling latency. We mock up kubelets and only measure scheduling latency and do not care about the latency to start the pod. This is different from Kubemark -- scheduled pods won’t be run by mocked kubelets. We only need to set up the master components. Goal 2: Profiling runtime metrics and find out performance bottleneck. Write scheduler integration test. Take advantage of go test and profiling tools, like cpu-profiling, memory-profiling and block-profiling. This is very fine-grained metrics. Goal 3: Reproduce test result easily. We want to have a known place to do the performance related test for scheduler. Developers should just run one script to collect all the information they need. We have the following plans for writing scheduler tests: density test (by adding a new Go test) schedule 30k pods on 1000 (fake) nodes and 3k pods on 100 (fake) nodes print out scheduling rate every second let you learn the rate changes vs number of scheduled pods benchmark (by modifying the existing benchmark in integration or simply add a new one) make use of and report nanosecond/op. schedule b.N pods when the cluster has N nodes and P scheduled pods. Since it takes relatively long time to finish one round, b.N is small -- level of 10. We are new to k8s codebase. We also want to know where is the best place (directory) to commit in these tests? Any suggestions are welcomed!
/cc
This extends
So density already does this... the one option it looks like you may want is custom control , real or fake is irrelevant as the e2e's just need to point to an api-server. re: benchmarks, seems like we could simply add to the unit-tests.
After chatting with lets add this as an agenda item for the SIG meeting tomorrow.
+1 for adding benchmark (howeveer this should be integration test, not unit-test in my opinion)
cc The next SIG on scheduler meeting is on 12.07, isn't it? If there is one tomorrow, can I have the information please?
retract, +1 to . I was adding this to the list given the context, but it crosses over into both.
Slides from today's meeting:
Here are are the action item notes from the SIG meeting: Everyone agrees this is a good idea and long overdue, but the question is where would this live: New category of tests? (isolated/sandbox’d/component) The current testing structure allows to easily append. create new directory (component) + hack/script to execute Future "we could" gate on known good data, and fail performance regressions
/cc
/cc

<ins> /* Copyright 2015 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package benchmark import ( "testing" "time" ) // BenchmarkScheduling100Nodes0Pods benchmarks the scheduling rate // when the cluster has 100 nodes and 0 scheduled pods func BenchmarkScheduling100Nodes0Pods(b *testing.B) { benchmarkScheduling(100, 0, b) } // BenchmarkScheduling100Nodes1000Pods benchmarks the scheduling rate // when the cluster has 100 nodes and 1000 scheduled pods func BenchmarkScheduling100Nodes1000Pods(b *testing.B) { benchmarkScheduling(100, 1000, b) } // BenchmarkScheduling1000Nodes0Pods benchmarks the scheduling rate // when the cluster has 1000 nodes and 0 scheduled pods func BenchmarkScheduling1000Nodes0Pods(b *testing.B) { benchmarkScheduling(1000, 0, b) } // BenchmarkScheduling1000Nodes1000Pods benchmarks the scheduling rate // when the cluster has 1000 nodes and 1000 scheduled pods func BenchmarkScheduling1000Nodes1000Pods(b *testing.B) { benchmarkScheduling(1000, 1000, b) } // benchmarkScheduling benchmarks scheduling rate with specific number of nodes // and specific number of pods already scheduled. Since an operation takes relatively // long time, b.N should be small: 10 - 100. func benchmarkScheduling(numNodes, numScheduledPods int, b *testing.B) { schedulerConfigFactory, finalFunc := mustSetupScheduler() defer finalFunc() c := schedulerConfigFactory.Client makeNodes(c, numNodes) makePods(c, numScheduledPods) for { scheduled := schedulerConfigFactory.ScheduledPodLister.Store.List() if len(scheduled) >= numScheduledPods { break } time.Sleep(1 * time.Second) } // start benchmark b.ResetTimer() makePods(c, b.N) for { // This can potentially affect performance of scheduler, since List() is done under mutex. // TODO: Setup watch on apiserver and wait until all pods scheduled. scheduled := schedulerConfigFactory.ScheduledPodLister.Store.List() if len(scheduled) >= numScheduledPods+b.N { break } // Note: This might introduce slight deviation in accuracy of benchmark results. // Since the total amount of time is relatively large, it might not be a concern. time.Sleep(100 * time.Millisecond) } } </ins>


Question:
We have found that on 1k nodes cluster scheduling takes 60-200 ms and becomes the bottleneck. We want to have a standard way to reproduce scheduling latency metrics result and benchmark scheduler as simple and fast as possible. Only with that can we: 1. figure out underlying behaviors and potential bottlenecks quickly; 2. publish results in a standard way and prevent future performance degradation. We have the following goals: Goal 1: Save time on testing. More accurate scheduling latency. We mock up kubelets and only measure scheduling latency and do not care about the latency to start the pod. This is different from Kubemark -- scheduled pods won’t be run by mocked kubelets. We only need to set up the master components. Goal 2: Profiling runtime metrics and find out performance bottleneck. Write scheduler integration test. Take advantage of go test and profiling tools, like cpu-profiling, memory-profiling and block-profiling. This is very fine-grained metrics. Goal 3: Reproduce test result easily. We want to have a known place to do the performance related test for scheduler. Developers should just run one script to collect all the information they need. We have the following plans for writing scheduler tests: density test (by adding a new Go test) schedule 30k pods on 1000 (fake) nodes and 3k pods on 100 (fake) nodes print out scheduling rate every second let you learn the rate changes vs number of scheduled pods benchmark (by modifying the existing benchmark in integration or simply add a new one) make use of and report nanosecond/op. schedule b.N pods when the cluster has N nodes and P scheduled pods. Since it takes relatively long time to finish one round, b.N is small -- level of 10. We are new to k8s codebase. We also want to know where is the best place (directory) to commit in these tests? Any suggestions are welcomed!
/cc
This extends
So density already does this... the one option it looks like you may want is custom control , real or fake is irrelevant as the e2e's just need to point to an api-server. re: benchmarks, seems like we could simply add to the unit-tests.
After chatting with lets add this as an agenda item for the SIG meeting tomorrow.
+1 for adding benchmark (howeveer this should be integration test, not unit-test in my opinion)
cc The next SIG on scheduler meeting is on 12.07, isn't it? If there is one tomorrow, can I have the information please?
retract, +1 to . I was adding this to the list given the context, but it crosses over into both.
Slides from today's meeting:
Here are are the action item notes from the SIG meeting: Everyone agrees this is a good idea and long overdue, but the question is where would this live: New category of tests? (isolated/sandbox’d/component) The current testing structure allows to easily append. create new directory (component) + hack/script to execute Future "we could" gate on known good data, and fail performance regressions
/cc
/cc

<ins> /* Copyright 2015 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package benchmark import ( "fmt" "testing" "time" ) // TestSchedule100Node3KPods schedules 3k pods on 100 nodes. func TestSchedule100Node3KPods(t *testing.T) { schedulePods(100, 3000) } // TestSchedule1000Node30KPods schedules 30k pods on 1000 nodes. func TestSchedule1000Node30KPods(t *testing.T) { schedulePods(1000, 30000) } // schedulePods schedules specific number of pods on specific number of nodes. // This is used to learn the scheduling throughput on various // sizes of cluster and changes as more and more pods are scheduled. // It won't stop until all pods are scheduled. func schedulePods(numNodes, numPods int) { schedulerConfigFactory, destroyFunc := mustSetupScheduler() defer destroyFunc() c := schedulerConfigFactory.Client makeNodes(c, numNodes) makePods(c, numPods) prev := 0 start := time.Now() for { // This can potentially affect performance of scheduler, since List() is done under mutex. // Listing 10000 pods is an expensive operation, so running it frequently may impact scheduler. // TODO: Setup watch on apiserver and wait until all pods scheduled. scheduled := schedulerConfigFactory.ScheduledPodLister.Store.List() fmt.Printf("%dstrate: %dttotal: %dn", time.Since(start)/time.Second, len(scheduled)-prev, len(scheduled)) if len(scheduled) >= numPods { return } prev = len(scheduled) time.Sleep(1 * time.Second) } } </ins>


Question:
We have found that on 1k nodes cluster scheduling takes 60-200 ms and becomes the bottleneck. We want to have a standard way to reproduce scheduling latency metrics result and benchmark scheduler as simple and fast as possible. Only with that can we: 1. figure out underlying behaviors and potential bottlenecks quickly; 2. publish results in a standard way and prevent future performance degradation. We have the following goals: Goal 1: Save time on testing. More accurate scheduling latency. We mock up kubelets and only measure scheduling latency and do not care about the latency to start the pod. This is different from Kubemark -- scheduled pods won’t be run by mocked kubelets. We only need to set up the master components. Goal 2: Profiling runtime metrics and find out performance bottleneck. Write scheduler integration test. Take advantage of go test and profiling tools, like cpu-profiling, memory-profiling and block-profiling. This is very fine-grained metrics. Goal 3: Reproduce test result easily. We want to have a known place to do the performance related test for scheduler. Developers should just run one script to collect all the information they need. We have the following plans for writing scheduler tests: density test (by adding a new Go test) schedule 30k pods on 1000 (fake) nodes and 3k pods on 100 (fake) nodes print out scheduling rate every second let you learn the rate changes vs number of scheduled pods benchmark (by modifying the existing benchmark in integration or simply add a new one) make use of and report nanosecond/op. schedule b.N pods when the cluster has N nodes and P scheduled pods. Since it takes relatively long time to finish one round, b.N is small -- level of 10. We are new to k8s codebase. We also want to know where is the best place (directory) to commit in these tests? Any suggestions are welcomed!
/cc
This extends
So density already does this... the one option it looks like you may want is custom control , real or fake is irrelevant as the e2e's just need to point to an api-server. re: benchmarks, seems like we could simply add to the unit-tests.
After chatting with lets add this as an agenda item for the SIG meeting tomorrow.
+1 for adding benchmark (howeveer this should be integration test, not unit-test in my opinion)
cc The next SIG on scheduler meeting is on 12.07, isn't it? If there is one tomorrow, can I have the information please?
retract, +1 to . I was adding this to the list given the context, but it crosses over into both.
Slides from today's meeting:
Here are are the action item notes from the SIG meeting: Everyone agrees this is a good idea and long overdue, but the question is where would this live: New category of tests? (isolated/sandbox’d/component) The current testing structure allows to easily append. create new directory (component) + hack/script to execute Future "we could" gate on known good data, and fail performance regressions
/cc
/cc

<ins> #!/usr/bin/env bash # Copyright 2014 The Kubernetes Authors All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -o errexit set -o nounset set -o pipefail pushd "../../../.." source "./hack/lib/util.sh" source "./hack/lib/logging.sh" source "./hack/lib/etcd.sh" popd cleanup() { kube::etcd::cleanup kube::log::status "performance test cleanup complete" } trap cleanup EXIT kube::etcd::start kube::log::status "performance test start" # TODO: set log-dir and prof output dir. DIR_BASENAME=$(basename `pwd`) go test -c -o "${DIR_BASENAME}.test" # We are using the benchmark suite to do profiling. Because it only runs a few pods and # theoretically it has less variance. "./${DIR_BASENAME}.test" -test.bench=. -test.run=xxxx -test.cpuprofile=prof.out -logtostderr=false kube::log::status "benchmark tests finished" # Running density tests. It might take a long time. "./${DIR_BASENAME}.test" -test.run=. -test.timeout=60m kube::log::status "density tests finished" </ins>


Question:
We have found that on 1k nodes cluster scheduling takes 60-200 ms and becomes the bottleneck. We want to have a standard way to reproduce scheduling latency metrics result and benchmark scheduler as simple and fast as possible. Only with that can we: 1. figure out underlying behaviors and potential bottlenecks quickly; 2. publish results in a standard way and prevent future performance degradation. We have the following goals: Goal 1: Save time on testing. More accurate scheduling latency. We mock up kubelets and only measure scheduling latency and do not care about the latency to start the pod. This is different from Kubemark -- scheduled pods won’t be run by mocked kubelets. We only need to set up the master components. Goal 2: Profiling runtime metrics and find out performance bottleneck. Write scheduler integration test. Take advantage of go test and profiling tools, like cpu-profiling, memory-profiling and block-profiling. This is very fine-grained metrics. Goal 3: Reproduce test result easily. We want to have a known place to do the performance related test for scheduler. Developers should just run one script to collect all the information they need. We have the following plans for writing scheduler tests: density test (by adding a new Go test) schedule 30k pods on 1000 (fake) nodes and 3k pods on 100 (fake) nodes print out scheduling rate every second let you learn the rate changes vs number of scheduled pods benchmark (by modifying the existing benchmark in integration or simply add a new one) make use of and report nanosecond/op. schedule b.N pods when the cluster has N nodes and P scheduled pods. Since it takes relatively long time to finish one round, b.N is small -- level of 10. We are new to k8s codebase. We also want to know where is the best place (directory) to commit in these tests? Any suggestions are welcomed!
/cc
This extends
So density already does this... the one option it looks like you may want is custom control , real or fake is irrelevant as the e2e's just need to point to an api-server. re: benchmarks, seems like we could simply add to the unit-tests.
After chatting with lets add this as an agenda item for the SIG meeting tomorrow.
+1 for adding benchmark (howeveer this should be integration test, not unit-test in my opinion)
cc The next SIG on scheduler meeting is on 12.07, isn't it? If there is one tomorrow, can I have the information please?
retract, +1 to . I was adding this to the list given the context, but it crosses over into both.
Slides from today's meeting:
Here are are the action item notes from the SIG meeting: Everyone agrees this is a good idea and long overdue, but the question is where would this live: New category of tests? (isolated/sandbox’d/component) The current testing structure allows to easily append. create new directory (component) + hack/script to execute Future "we could" gate on known good data, and fail performance regressions
/cc
/cc

<ins> /* Copyright 2015 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package benchmark import ( "net/http" "net/http/httptest" "github.com/golang/glog" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/resource" "k8s.io/kubernetes/pkg/api/testapi" "k8s.io/kubernetes/pkg/client/record" client "k8s.io/kubernetes/pkg/client/unversioned" "k8s.io/kubernetes/pkg/master" "k8s.io/kubernetes/plugin/pkg/scheduler" _ "k8s.io/kubernetes/plugin/pkg/scheduler/algorithmprovider" "k8s.io/kubernetes/plugin/pkg/scheduler/factory" "k8s.io/kubernetes/test/integration/framework" ) // mustSetupScheduler starts the following components: // - k8s api server (a.k.a. master) // - scheduler // It returns scheduler config factory and destroyFunc which should be used to // remove resources after finished. // Notes on rate limiter: //   - The BindPodsRateLimiter is nil, meaning no rate limits. //   - client rate limit is set to 5000. func mustSetupScheduler() (schedulerConfigFactory *factory.ConfigFactory, destroyFunc func()) { framework.DeleteAllEtcdKeys() var m *master.Master masterConfig := framework.NewIntegrationTestMasterConfig() m = master.New(masterConfig) s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { m.Handler.ServeHTTP(w, req) })) c := client.NewOrDie(&client.Config{ Host:         s.URL, GroupVersion: testapi.Default.GroupVersion(), QPS:          5000.0, Burst:        5000, }) schedulerConfigFactory = factory.NewConfigFactory(c, nil) schedulerConfig, err := schedulerConfigFactory.Create() if err != nil { panic("Couldn't create scheduler config") } eventBroadcaster := record.NewBroadcaster() schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: "scheduler"}) eventBroadcaster.StartRecordingToSink(c.Events("")) scheduler.New(schedulerConfig).Run() destroyFunc = func() { glog.Infof("destroying") close(schedulerConfig.StopEverything) s.Close() glog.Infof("destroyed") } return } func makeNodes(c client.Interface, nodeCount int) { glog.Infof("making %d nodes", nodeCount) baseNode := &api.Node{ ObjectMeta: api.ObjectMeta{ GenerateName: "scheduler-test-node-", }, Spec: api.NodeSpec{ ExternalID: "foobar", }, Status: api.NodeStatus{ Capacity: api.ResourceList{ api.ResourcePods:   *resource.NewQuantity(32, resource.DecimalSI), api.ResourceCPU:    resource.MustParse("4"), api.ResourceMemory: resource.MustParse("32Gi"), }, Phase: api.NodeRunning, Conditions: []api.NodeCondition{ {Type: api.NodeReady, Status: api.ConditionTrue}, }, }, } for i := 0; i < nodeCount; i++ { if _, err := c.Nodes().Create(baseNode); err != nil { panic("error creating node: " + err.Error()) } } } // makePods will setup specified number of scheduled pods. // Currently it goes through scheduling path and it's very slow to setup large number of pods. // TODO: Setup pods evenly on all nodes and quickly/non-linearly. func makePods(c client.Interface, podCount int) { glog.Infof("making %d pods", podCount) basePod := &api.Pod{ ObjectMeta: api.ObjectMeta{ GenerateName: "scheduler-test-pod-", }, Spec: api.PodSpec{ Containers: []api.Container{{ Name:  "pause", Image: "gcr.io/google_containers/pause:1.0", Resources: api.ResourceRequirements{ Limits: api.ResourceList{ api.ResourceCPU:    resource.MustParse("100m"), api.ResourceMemory: resource.MustParse("500Mi"), }, Requests: api.ResourceList{ api.ResourceCPU:    resource.MustParse("100m"), api.ResourceMemory: resource.MustParse("500Mi"), }, }, }}, }, } threads := 30 remaining := make(chan int, 1000) go func() { for i := 0; i < podCount; i++ { remaining <- i } close(remaining) }() for i := 0; i < threads; i++ { go func() { for { _, ok := <-remaining if !ok { return } for { _, err := c.Pods("default").Create(basePod) if err == nil { break } } } }() } } </ins>


Question:
We have found that on 1k nodes cluster scheduling takes 60-200 ms and becomes the bottleneck. We want to have a standard way to reproduce scheduling latency metrics result and benchmark scheduler as simple and fast as possible. Only with that can we: 1. figure out underlying behaviors and potential bottlenecks quickly; 2. publish results in a standard way and prevent future performance degradation. We have the following goals: Goal 1: Save time on testing. More accurate scheduling latency. We mock up kubelets and only measure scheduling latency and do not care about the latency to start the pod. This is different from Kubemark -- scheduled pods won’t be run by mocked kubelets. We only need to set up the master components. Goal 2: Profiling runtime metrics and find out performance bottleneck. Write scheduler integration test. Take advantage of go test and profiling tools, like cpu-profiling, memory-profiling and block-profiling. This is very fine-grained metrics. Goal 3: Reproduce test result easily. We want to have a known place to do the performance related test for scheduler. Developers should just run one script to collect all the information they need. We have the following plans for writing scheduler tests: density test (by adding a new Go test) schedule 30k pods on 1000 (fake) nodes and 3k pods on 100 (fake) nodes print out scheduling rate every second let you learn the rate changes vs number of scheduled pods benchmark (by modifying the existing benchmark in integration or simply add a new one) make use of and report nanosecond/op. schedule b.N pods when the cluster has N nodes and P scheduled pods. Since it takes relatively long time to finish one round, b.N is small -- level of 10. We are new to k8s codebase. We also want to know where is the best place (directory) to commit in these tests? Any suggestions are welcomed!
/cc
This extends
So density already does this... the one option it looks like you may want is custom control , real or fake is irrelevant as the e2e's just need to point to an api-server. re: benchmarks, seems like we could simply add to the unit-tests.
After chatting with lets add this as an agenda item for the SIG meeting tomorrow.
+1 for adding benchmark (howeveer this should be integration test, not unit-test in my opinion)
cc The next SIG on scheduler meeting is on 12.07, isn't it? If there is one tomorrow, can I have the information please?
retract, +1 to . I was adding this to the list given the context, but it crosses over into both.
Slides from today's meeting:
Here are are the action item notes from the SIG meeting: Everyone agrees this is a good idea and long overdue, but the question is where would this live: New category of tests? (isolated/sandbox’d/component) The current testing structure allows to easily append. create new directory (component) + hack/script to execute Future "we could" gate on known good data, and fail performance regressions
/cc
/cc

"fmt" "net/http" "net/http/httptest" <del> "sync" </del> "testing" "time"


Question:
We have found that on 1k nodes cluster scheduling takes 60-200 ms and becomes the bottleneck. We want to have a standard way to reproduce scheduling latency metrics result and benchmark scheduler as simple and fast as possible. Only with that can we: 1. figure out underlying behaviors and potential bottlenecks quickly; 2. publish results in a standard way and prevent future performance degradation. We have the following goals: Goal 1: Save time on testing. More accurate scheduling latency. We mock up kubelets and only measure scheduling latency and do not care about the latency to start the pod. This is different from Kubemark -- scheduled pods won’t be run by mocked kubelets. We only need to set up the master components. Goal 2: Profiling runtime metrics and find out performance bottleneck. Write scheduler integration test. Take advantage of go test and profiling tools, like cpu-profiling, memory-profiling and block-profiling. This is very fine-grained metrics. Goal 3: Reproduce test result easily. We want to have a known place to do the performance related test for scheduler. Developers should just run one script to collect all the information they need. We have the following plans for writing scheduler tests: density test (by adding a new Go test) schedule 30k pods on 1000 (fake) nodes and 3k pods on 100 (fake) nodes print out scheduling rate every second let you learn the rate changes vs number of scheduled pods benchmark (by modifying the existing benchmark in integration or simply add a new one) make use of and report nanosecond/op. schedule b.N pods when the cluster has N nodes and P scheduled pods. Since it takes relatively long time to finish one round, b.N is small -- level of 10. We are new to k8s codebase. We also want to know where is the best place (directory) to commit in these tests? Any suggestions are welcomed!
/cc
This extends
So density already does this... the one option it looks like you may want is custom control , real or fake is irrelevant as the e2e's just need to point to an api-server. re: benchmarks, seems like we could simply add to the unit-tests.
After chatting with lets add this as an agenda item for the SIG meeting tomorrow.
+1 for adding benchmark (howeveer this should be integration test, not unit-test in my opinion)
cc The next SIG on scheduler meeting is on 12.07, isn't it? If there is one tomorrow, can I have the information please?
retract, +1 to . I was adding this to the list given the context, but it crosses over into both.
Slides from today's meeting:
Here are are the action item notes from the SIG meeting: Everyone agrees this is a good idea and long overdue, but the question is where would this live: New category of tests? (isolated/sandbox’d/component) The current testing structure allows to easily append. create new directory (component) + hack/script to execute Future "we could" gate on known good data, and fail performance regressions
/cc
/cc

} } } <del> func BenchmarkScheduling(b *testing.B) { framework.DeleteAllEtcdKeys() var m *master.Master s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { m.Handler.ServeHTTP(w, req) })) defer s.Close() masterConfig := framework.NewIntegrationTestMasterConfig() m = master.New(masterConfig) c := client.NewOrDie(&client.Config{ Host:         s.URL, GroupVersion: testapi.Default.GroupVersion(), QPS:          5000.0, Burst:        5000, }) schedulerConfigFactory := factory.NewConfigFactory(c, nil) schedulerConfig, err := schedulerConfigFactory.Create() if err != nil { b.Fatalf("Couldn't create scheduler config: %v", err) } eventBroadcaster := record.NewBroadcaster() schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: "scheduler"}) eventBroadcaster.StartRecordingToSink(c.Events("")) scheduler.New(schedulerConfig).Run() defer close(schedulerConfig.StopEverything) makeNNodes(c, 1000) N := b.N b.ResetTimer() makeNPods(c, N) for { objs := schedulerConfigFactory.ScheduledPodLister.Store.List() if len(objs) >= N { fmt.Printf("%v pods scheduled.n", len(objs)) /* // To prove that this actually works: for _, o := range objs { fmt.Printf("%sn", o.(*api.Pod).Spec.NodeName) } */ break } time.Sleep(time.Millisecond) } b.StopTimer() } func makeNNodes(c client.Interface, N int) { baseNode := &api.Node{ ObjectMeta: api.ObjectMeta{ GenerateName: "scheduler-test-node-", }, Spec: api.NodeSpec{ ExternalID: "foobar", }, Status: api.NodeStatus{ Capacity: api.ResourceList{ api.ResourcePods:   *resource.NewQuantity(32, resource.DecimalSI), api.ResourceCPU:    resource.MustParse("4"), api.ResourceMemory: resource.MustParse("32Gi"), }, Phase: api.NodeRunning, Conditions: []api.NodeCondition{ {Type: api.NodeReady, Status: api.ConditionTrue}, }, }, } for i := 0; i < N; i++ { if _, err := c.Nodes().Create(baseNode); err != nil { panic("error creating node: " + err.Error()) } } } func makeNPods(c client.Interface, N int) { basePod := &api.Pod{ ObjectMeta: api.ObjectMeta{ GenerateName: "scheduler-test-pod-", }, Spec: api.PodSpec{ Containers: []api.Container{{ Name:  "pause", Image: "gcr.io/google_containers/pause:1.0", Resources: api.ResourceRequirements{ Limits: api.ResourceList{ api.ResourceCPU:    resource.MustParse("100m"), api.ResourceMemory: resource.MustParse("500Mi"), }, Requests: api.ResourceList{ api.ResourceCPU:    resource.MustParse("100m"), api.ResourceMemory: resource.MustParse("500Mi"), }, }, }}, }, } wg := sync.WaitGroup{} threads := 30 wg.Add(threads) remaining := make(chan int, N) go func() { for i := 0; i < N; i++ { remaining <- i } close(remaining) }() for i := 0; i < threads; i++ { go func() { defer wg.Done() for { _, ok := <-remaining if !ok { return } for { _, err := c.Pods("default").Create(basePod) if err == nil { break } } } }() } wg.Wait() } </del>


Question:
I'm running application with several env variables. One of them has a key ending with ==. Another variable has a comma separated list of email addresses. While perfectly valid on *nix and windows, we can't pass these values into Kubernetes run command. It fails with "error: invalid env:" Example command: The parsing in should be more robust
I volunteer myself to fix this.
Maybe they can be passed from file? I mean s'rsly.. you need to lits 20 variables by hand?? I've specified them in yaml file already, but kubectl-run have no way to specify it.. without this, deployment is totally impossible when you need to run migrations first!

func parseEnvs(envArray []string) ([]api.EnvVar, error) { envs := []api.EnvVar{} for _, env := range envArray { <del> parts := strings.Split(env, "=") if len(parts) != 2 || !validation.IsCIdentifier(parts[0]) || len(parts[1]) == 0 { </del> <ins> pos := strings.Index(env, "=") if pos == -1 { </ins> return nil, fmt.Errorf("invalid env: %v", env) } <del> envVar := api.EnvVar{Name: parts[0], Value: parts[1]} </del> <ins> name := env[:pos] value := env[pos+1:] if len(name) == 0 || !validation.IsCIdentifier(name) || len(value) == 0 { return nil, fmt.Errorf("invalid env: %v", env) } envVar := api.EnvVar{Name: name, Value: value} </ins> envs = append(envs, envVar) } return envs, nil


Question:
I'm running application with several env variables. One of them has a key ending with ==. Another variable has a comma separated list of email addresses. While perfectly valid on *nix and windows, we can't pass these values into Kubernetes run command. It fails with "error: invalid env:" Example command: The parsing in should be more robust
I volunteer myself to fix this.
Maybe they can be passed from file? I mean s'rsly.. you need to lits 20 variables by hand?? I've specified them in yaml file already, but kubectl-run have no way to specify it.. without this, deployment is totally impossible when you need to run migrations first!

} } } <ins> func TestParseEnv(t *testing.T) { tests := []struct { envArray  []string expected  []api.EnvVar expectErr bool test      string }{ { envArray: []string{ "THIS_ENV=isOK", "HAS_COMMAS=foo,bar", "HAS_EQUALS=jJnro54iUu75xNy==", }, expected: []api.EnvVar{ { Name:  "THIS_ENV", Value: "isOK", }, { Name:  "HAS_COMMAS", Value: "foo,bar", }, { Name:  "HAS_EQUALS", Value: "jJnro54iUu75xNy==", }, }, expectErr: false, test:      "test case 1", }, { envArray: []string{ "WITH_OUT_EQUALS", }, expected:  []api.EnvVar{}, expectErr: true, test:      "test case 2", }, { envArray: []string{ "WITH_OUT_VALUES=", }, expected:  []api.EnvVar{}, expectErr: true, test:      "test case 3", }, { envArray: []string{ "=WITH_OUT_NAME", }, expected:  []api.EnvVar{}, expectErr: true, test:      "test case 4", }, } for _, test := range tests { envs, err := parseEnvs(test.envArray) if !test.expectErr && err != nil { t.Errorf("unexpected error: %v (%s)", err, test.test) } if test.expectErr && err != nil { continue } if !reflect.DeepEqual(envs, test.expected) { t.Errorf("nexpected:n%#vnsaw:n%#v (%s)", test.expected, envs, test.test) } } } </ins>


Question:
Have not reproduced locally even after a few tens of runs.
which version of Go are you using locally? This probably only repros on 1.5.1.
This was from Jenkins runs.
This is caused because an attempts counter is being read/written to without synchronization. I think this is caused by the event creation order not being guaranteed. I am looking a little deeper to track this down.
Got a fix with 130k+ runs and no failures. Will send out PR shortly.

eventCorrelator := NewEventCorrelator(util.RealClock{}) return eventBroadcaster.StartEventWatcher( func(event *api.Event) { <del> // Make a copy before modification, because there could be multiple listeners. // Events are safe to copy like this. eventCopy := *event event = &eventCopy result, err := eventCorrelator.EventCorrelate(event) if err != nil { util.HandleError(err) } if result.Skip { return } tries := 0 for { if recordEvent(sink, result.Event, result.Patch, result.Event.Count > 1, eventCorrelator) { break } tries++ if tries >= maxTriesPerEvent { glog.Errorf("Unable to write event '%#v' (retry limit exceeded!)", event) break } // Randomize the first sleep so that various clients won't all be // synced up if the master goes down. if tries == 1 { time.Sleep(time.Duration(float64(sleepDuration) * randGen.Float64())) } else { time.Sleep(sleepDuration) } } </del> <ins> recordToSink(sink, event, eventCorrelator, randGen) </ins> }) } <ins> func recordToSink(sink EventSink, event *api.Event, eventCorrelator *EventCorrelator, randGen *rand.Rand) { // Make a copy before modification, because there could be multiple listeners. // Events are safe to copy like this. eventCopy := *event event = &eventCopy result, err := eventCorrelator.EventCorrelate(event) if err != nil { util.HandleError(err) } if result.Skip { return } tries := 0 for { if recordEvent(sink, result.Event, result.Patch, result.Event.Count > 1, eventCorrelator) { break } tries++ if tries >= maxTriesPerEvent { glog.Errorf("Unable to write event '%#v' (retry limit exceeded!)", event) break } // Randomize the first sleep so that various clients won't all be // synced up if the master goes down. if tries == 1 { time.Sleep(time.Duration(float64(sleepDuration) * randGen.Float64())) } else { time.Sleep(sleepDuration) } } } </ins> func isKeyNotFoundError(err error) bool { statusErr, _ := err.(*errors.StatusError) // At the moment the server is returning 500 instead of a more specific


Question:
Have not reproduced locally even after a few tens of runs.
which version of Go are you using locally? This probably only repros on 1.5.1.
This was from Jenkins runs.
This is caused because an attempts counter is being read/written to without synchronization. I think this is caused by the event creation order not being guaranteed. I am looking a little deeper to track this down.
Got a fix with 130k+ runs and no failures. Will send out PR shortly.

import ( "encoding/json" "fmt" <del> "runtime" </del> <ins> "math/rand" </ins> "strconv" "testing" "time"


Question:
Have not reproduced locally even after a few tens of runs.
which version of Go are you using locally? This probably only repros on 1.5.1.
This was from Jenkins runs.
This is caused because an attempts counter is being read/written to without synchronization. I think this is caused by the event creation order not being guaranteed. I am looking a little deeper to track this down.
Got a fix with 130k+ runs and no failures. Will send out PR shortly.

} func TestWriteEventError(t *testing.T) { <del> ref := &api.ObjectReference{ Kind:       "Pod", Name:       "foo", Namespace:  "baz", UID:        "bar", APIVersion: "version", } </del> type entry struct { timesToSendError int <del> attemptsMade     int </del> attemptsWanted   int err              error }


Question:
Have not reproduced locally even after a few tens of runs.
which version of Go are you using locally? This probably only repros on 1.5.1.
This was from Jenkins runs.
This is caused because an attempts counter is being read/written to without synchronization. I think this is caused by the event creation order not being guaranteed. I am looking a little deeper to track this down.
Got a fix with 130k+ runs and no failures. Will send out PR shortly.

err:              fmt.Errorf("A weird error"), }, } <del> done := make(chan struct{}) eventBroadcaster := NewBroadcaster() defer eventBroadcaster.StartRecordingToSink( &testEventSink{ </del> <ins> eventCorrelator := NewEventCorrelator(util.RealClock{}) randGen := rand.New(rand.NewSource(time.Now().UnixNano())) for caseName, ent := range table { attempts := 0 sink := &testEventSink{ </ins> OnCreate: func(event *api.Event) (*api.Event, error) { <del> if event.Message == "finished" { close(done) return event, nil } item, ok := table[event.Message] if !ok { t.Errorf("Unexpected event: %#v", event) return event, nil } item.attemptsMade++ if item.attemptsMade < item.timesToSendError { return nil, item.err </del> <ins> attempts++ if attempts < ent.timesToSendError { return nil, ent.err </ins> } return event, nil }, <del> }, ).Stop() clock := &util.FakeClock{time.Now()} recorder := recorderWithFakeClock(api.EventSource{Component: "eventTest"}, eventBroadcaster, clock) for caseName := range table { clock.Step(1 * time.Second) recorder.Event(ref, api.EventTypeNormal, "Reason", caseName) runtime.Gosched() } recorder.Event(ref, api.EventTypeNormal, "Reason", "finished") <-done for caseName, item := range table { if e, a := item.attemptsWanted, item.attemptsMade; e != a { t.Errorf("case %v: wanted %v, got %v attempts", caseName, e, a) </del> <ins> } ev := &api.Event{} recordToSink(sink, ev, eventCorrelator, randGen) if attempts != ent.attemptsWanted { t.Errorf("case %v: wanted %d, got %d attempts", caseName, ent.attemptsWanted, attempts) </ins> } } }


Question:
Currently scheduler latency are observed regardless of failure. For example, binding latency is like: This could be a potential confusion if latency spikes happen because of failure. We should better handle it like separating error case from normal
/cc
I completely agree - are you going to change this?
Yes. I will do it.

var BindingSaturationReportInterval = 1 * time.Second var ( <del> E2eSchedulingLatency = prometheus.NewSummary( prometheus.SummaryOpts{ </del> <ins> E2eSchedulingLatency = prometheus.NewHistogram( prometheus.HistogramOpts{ </ins> Subsystem: schedulerSubsystem, Name:      "e2e_scheduling_latency_microseconds", Help:      "E2e scheduling latency (scheduling algorithm + binding)", <del> MaxAge:    time.Hour, </del> <ins> Buckets:   prometheus.ExponentialBuckets(1000, 2, 15), </ins> }, ) <del> SchedulingAlgorithmLatency = prometheus.NewSummary( prometheus.SummaryOpts{ </del> <ins> SchedulingAlgorithmLatency = prometheus.NewHistogram( prometheus.HistogramOpts{ </ins> Subsystem: schedulerSubsystem, Name:      "scheduling_algorithm_latency_microseconds", Help:      "Scheduling algorithm latency", <del> MaxAge:    time.Hour, </del> <ins> Buckets:   prometheus.ExponentialBuckets(1000, 2, 15), </ins> }, ) <del> BindingLatency = prometheus.NewSummary( prometheus.SummaryOpts{ </del> <ins> BindingLatency = prometheus.NewHistogram( prometheus.HistogramOpts{ </ins> Subsystem: schedulerSubsystem, Name:      "binding_latency_microseconds", Help:      "Binding latency", <del> MaxAge:    time.Hour, </del> <ins> Buckets:   prometheus.ExponentialBuckets(1000, 2, 15), </ins> }, ) BindingRateLimiterSaturation = prometheus.NewGauge(


Question:
Currently scheduler latency are observed regardless of failure. For example, binding latency is like: This could be a potential confusion if latency spikes happen because of failure. We should better handle it like separating error case from normal
/cc
I completely agree - are you going to change this?
Yes. I will do it.

glog.V(3).Infof("Attempting to schedule: %+v", pod) start := time.Now() <del> defer func() { metrics.E2eSchedulingLatency.Observe(metrics.SinceInMicroseconds(start)) }() </del> dest, err := s.config.Algorithm.Schedule(pod, s.config.NodeLister) <del> metrics.SchedulingAlgorithmLatency.Observe(metrics.SinceInMicroseconds(start)) </del> if err != nil { glog.V(1).Infof("Failed to schedule: %+v", pod) s.config.Recorder.Eventf(pod, api.EventTypeWarning, "FailedScheduling", "%v", err) s.config.Error(pod, err) return } <ins> metrics.SchedulingAlgorithmLatency.Observe(metrics.SinceInMicroseconds(start)) </ins> b := &api.Binding{ ObjectMeta: api.ObjectMeta{Namespace: pod.Namespace, Name: pod.Name}, Target: api.ObjectReference{


Question:
Currently scheduler latency are observed regardless of failure. For example, binding latency is like: This could be a potential confusion if latency spikes happen because of failure. We should better handle it like separating error case from normal
/cc
I completely agree - are you going to change this?
Yes. I will do it.

s.config.Modeler.LockedAction(func() { bindingStart := time.Now() err := s.config.Binder.Bind(b) <del> metrics.BindingLatency.Observe(metrics.SinceInMicroseconds(bindingStart)) </del> if err != nil { glog.V(1).Infof("Failed to bind pod: %+v", err) s.config.Recorder.Eventf(pod, api.EventTypeNormal, "FailedScheduling", "Binding rejected: %v", err) s.config.Error(pod, err) return } <ins> metrics.BindingLatency.Observe(metrics.SinceInMicroseconds(bindingStart)) </ins> s.config.Recorder.Eventf(pod, api.EventTypeNormal, "Scheduled", "Successfully assigned %v to %v", pod.Name, dest) // tell the model to assume that this binding took effect. assumed := *pod assumed.Spec.NodeName = dest s.config.Modeler.AssumePod(&assumed) }) <ins> metrics.E2eSchedulingLatency.Observe(metrics.SinceInMicroseconds(start)) </ins> }


Question:
I'll take a look at the failed builds and triage.
I went through the recent reboot failures in kubernetes-e2e-gke-ci-reboot and here is a summary. [6481, 6486, 6488] Kubelet never came back from the reboot, and stopped posting node status. Throughout the test suite, the failed node stayed silent: It's hard to dig further due to the lack of clarity. Gathering log would at least help us rule out some possible causes . [6485] Heapster seems to crash loop quite frequently, and sometimes it failed the test (due to being not ready).. cc [6487] kube-proxy pod is running but not ready before the reboot test even started: This is not really a reboot test failure since the cluster was already in a bad state. We should make sure all the pods are ready after each test (e.g. "should work after restarting kube-proxy" ran before the failed reboot test). Another thing that would help is to increase the log level of the gke cluster. The log level of a gce cluster is 4, but I think gke is at the regular 2. Due to the low log level, it's unclear when kube-proxy started failing the readiness probe.
Filed for (3).
The reboot tests basically haven't worked since 9pm last night: Interestingly, this lines up with the GKE equivalent for , but in that same window, has had no trouble. cc
Digging into logs from the runs linked.
All of the gke-ci failures in are heapster crash-looping after a reboot.
Changes to the heapster config in the last day: , . Nothing sticks out to me in those PRs.
Heapster could be in crashloop if it cannot reach influxdb. But that is a bad design, and should fix it in today's heapster with image 18. I double checked them, it is image version 18.
From , could be the num_nodes is unset, we give heapster no resource to run? If that is true, it could be point to the same version screw issue of our test infra.
Heapster v0.18.4 will crashloop if InfluxDB isn't available. However we can update HEAD to v0.19.0, which will not crash if the backend goes down. I will send a PR to update InfluxDB version. , Dawn Chen wrote:
GKE doesn't use InfluxDB, right?
Heapster is the only kube-system pod that's not ready in the tests. If it relies on influxdb, shouldn't influxdb be not ready too?
I like theory. My salt suggestion might've been horrible for unset ? I thought I did that correctly, but maybe my Python/Jinja was bad.
Yes, GKE doesn't use influxDB. My point is that if the backend storage is not ready, heapster is in crashloop. Vishnu made some changes to make heapster can run standalone, but looks like that fix is not included in heapster 18 based on his comment.
Okay. I'm still trying to catch up. Why is heapster on GKE waiting on backend storage? And if is unset (e.g. we haven't finished running configure-), it looks like we just give the
Catching others up: is seeing nodes that are actually hung with CPU pegged on reboot, so the heapster conversation may be a red herring.
There are plenty of tests that failed because kubelet didn't come back (and post status) after the reboot. There are a few that failed only because heapster was crash looping. They may be related or not...
I spun up a GKE test cluster: and the heapster manifest looks fine, so I think I ruled out anything regarding
I think is the winner. The reboot tests are a red herring.
I'm actually just seeing heapster flap on a fresh cluster, no reboots.
Heapster crashlooping is one problem. There are other cases, however, where kubelet just stops posting status all together. If heapster crashlooping can render kubelet unresponsive, that's a bug.
is the serial console output from the node that got pegged (in ). I rebooted the node to see if it would repro, and it came back up fine. EDIT: the link to the jenkins run.
I guess the serial console you posted is from build , the very recent one. I saw a similar one and the node never successfully come back. At this moment, I ignore those failures. The reboot failure caused by heapster crashloop is similar to what I observed 2 days ago for influxdb in gce cluster. The comment is at Can we retitle this one for (1)? Reboot test flakiness is caused by many reasons: version screw which is handled by test-infra, heapster and influxdb crashloop is handled by , kube-proxy not ready is handled by cluster team.
Ok. So we think we have the heapster flakes, versioning stuff, and kube-proxy stuff all taken care of elsewhere? I'm fine with targeting this issue at the "node doesn't come back after reboot" case. Is that what you were recommending?
Yes. 1) heapster in crashloop if backend is not ready is a known issue for a while, and fix is merged to heapster repositry for a while. We just need to upgrade our heapster after more testing (PR: ) 2) influxdb one requires more information, but we can handle them through 3) version screw is handled by test-infra through . already test/flake label for it 4) kube-proxy readiness issue is handled through
where else should I be looking for hints on why that VM didn't come back to life until I kicked it? Or did I destroy any evidence we might have had?
Fun fact: I thought this might be related the timing of when I made the ContainerVM change on the GKE side, but I made that change for all releases 1.1.4. That picks up this test suite, too, because it's at , and it's green: So this is a divergence from master to 1.1, or between projects.
That suite does not run any reboot tests...
Dammit, I thought I scanned for that. Good call. We need to fix that.
Here's the set of suites that run the reboot tests (thanks kubernetes-e2e-gce kubernetes-e2e-gce-reboot kubernetes-e2e-gke-ci kubernetes-e2e-gke-ci-reboot
Shouldn't we decided only X-reboot test suites run those reboot tests since they are disruptive?
Hm. builds 6478-6480 are all green and all after the (GKE-side) ContainerVM change, so we're looking for something else.
The container-vm change was merged at , which lines up with kubernetes-e2e-gke-ci-reboot 6481 (the beginning of the failures).
6481 is the one with the new containervm change. It includes kubernetes-build/7822, which has the change.
/ That's the OSS side change, which is irrelevant. GKE flipped at 19:45:55.
Ahhh. Fair enough.
Actually, and I just determined that the GKE-side change didn't make it out until 01:45:44, it looks like, so indeed nothing to do with ContainerVM.
I took a more careful look at the serial output pasted, and this is interesting: We're actually in the middle of a clean shutdown, trying to unmount and failing. This same stacktrace repeats every 120s.
I spun up an e2e cluster and with minimal testing have a node in the state of It's the same kernel hang on shutdown. ... and then I accidentally deleted it when I went to bisect, but ohwell. I'm sure I'll be able to get one back if I need.
Okay, so I've confirmed that at the build level at , I can spin up a GKE cluster and reboot tests fail. I'm going to peel back to and see if they pass (because that matches ). ETA: For reference, I'm able to tickle this with just:
7819 had a kernel hang on the second run as well, though slightly different signature (still aufs in the end):
For my next test, I'm peeling back to v1.1.4. We have no test coverage on reboots on the release branch of GKE right now, and I just want to guarantee this isn't some weird GCE hypervisor issue. And v1.1.4 is clean, at least for two runs worth.
This failed quickly on v1.1.5. Hrm. I'm wondering if my v1.1.4 test was a fluke, peeling back to try it there again. v1.1.5 only differs by ContainerVM (and go 1.4.3 vs 1.4.2), but all of our other timelines show that not being directly related. (Alternately, it's a lot easier on the newer kernel.) (Still no idea why the tests aren't triggered, but is.)
was very solid. I'm now trying on GKE test. This is a GKE unique thing, because after Friday morning at 1am, anything = will get deployed using the new ContainerVM. So I'm ret-conning a way to effectively try with the new ContainerVM, just to see if it's a contributing factor. It doesn't look like it should be. One of the other commits in the stream is a heapster version bump (from / ). If there's additional flapping or something caused by that (unique to GKE), it may be causing the hang. But that timeline doesn't match the line (relevant to , etc.)
Bad news bears: failed reboot tests. I'm having trouble rectifying this information with what we know about the timeline. But it does suggest there's a problem with ContainerVM at the very least, so there might be two problems here, so I'm rolling one back on GKE and we'll see what happens.
I've manually confirmed that by reverting deployment of on GKE, I can run reboot tests at and a latest build. Using a special build of latest that will still trigger deployment of , reboot tests fail. Well, damn.
GKE has been repushed and should, in theory, go back green soon. There's a lot I don't understand about this bug, and it's effectively going to stop us shipping and until we can figure it out.
Okay, I think that reverting was not all that's necessary. We unfortunately haven't managed to collect a green on yet. That gives me some comfort, because it means that the timeline-discomfort I had previously was founded.
Heh, I lied and right right writing that we collected our first one. Woo. I suspect will follow soon.
Haven't we updated go to 1.5 on Friday as well? I don't know if we use the same version of go for GKE as we do in GKE, but in general we know it causes trouble. cc for Heapster thing.
The timeline there is off, too. The golang1.5.3 PR went in at , which is well after the trouble started.
Morning status: Looks like we made a little greener, and didn't touch , so the patient had two viruses.
Okay, this is my suspicion, but I can't quite piece together what's going on: For some reason, something in (? but that busted the Heapster config, so it's confusing me as to why), caused this. I'm basing this on: the timeline of the timeline of If you suppose that we injected a bug in build 7820 that caused about a 60-70% cointoss as to whether we regress (which is similar to the rate we're seeing now that has been rolled back on GKE), it's not unreasonable that was green for , and that's the only one you have to explain away on this theory. After that, it goes kind of red, and then at about 1am, comes in and wrecks completely as well. Pulling it back much further, to, say, build 7819, and you have to explain the entire range from 6477-6480 as green, which is a much lower probability. So my money is on build 7820. (7821 is revision-equivalent, so boring to talk about.)
Oh. It is Heapster. I spun up a 7819 cluster and a 7820 cluster, and the difference is that even though still had slightly busted Salt syntax, it was functional enough to create the replication controller, whereas on 7819, there's no Heapster replication controller at all. So I think the Heapster flapping is entirely the cause of the remaining issues, so we really need to get in ASAP.
So yesterday afternoon I was wondering why build 7820 triggered such a large regression when, say, the bump to heapster in didn't regress us at all. Build 7820 finally let the the fix in , which was actually important because GKE at HEAD wasn't deploying (until last night), it was deploying only , so the pillar wasn't configured. I'm still not exactly sure what's going on here, and I had intentionally left blank for a while during this investigation because it didn't seem like it should matter, but it's obvious that if 7820 is the culprit that it's interesting. I pushed a fix last night to GKE that adds on 1.2 clusters as well, and we've collected our first green dot. I don't know why is still flakey, though.
Er, correction to my last comment. My change didn't make it out last night and won't be relevant until . The green is a fluke. Sad panda.
PR introduced syntax error, PR fixed the error although it introduced another syntax issue which was finally fixed in . if you are testing anything make sure you are using version with merged. I agree with that the problem is not with Heapster accessing InfluxDB. Also the same problem takes places in autoscaling suite (described here ). We started observing such problems around Jan 19th. AFAIK once the container crashes kubelet invoke docker inspect on the failing container. How about invoking also docker logs then? It would be very useful for debugging purposes.
Yes, I understand that fixed the issue fully, but at build 7820 the cluster that's created actually spins up a cluster with a replication controller .yaml with resources. At build 7819, it doesn't. The syntax in is busted, but it's apparently not so busted. Previous to , in build 7819 (and builds previous), there was no controller .yaml at all, so there was a window where GKE wasn't spinning up Heapster at all. Hence my conclusion.
The syntax in is correct from Salt point of view, although the generated file has empty line in the begging and is not accepted by Api Server, so the RC is not created.
That's inconsistent with the fact that Heapster is starting on a 7820 (GKE) cluster.
I have a new theory, which is that Heapster hasn't been running on GKE in 1.2 since , until .
It's not clear to me that upgrading heapster will solve this issue, since heapster doesn't rely on any backend on gke. Instead of asking kubelet to do that, it makes more sense for the test itself to get the logs through the apiserver. I think we should add that to the e2e test.
I spun up a cluster from just prior to and this is the set of RCs I see: So basically, when went in, Heapster was enabled for the first time in GKE (master, not the release branches) in months and we started flipping out. This is a serious test coverage issue, obviously.
I edited the above comment, but we would obviously catch this in the release branches because we have monitoring tied to Heapster related things. We need to be better here for CI.
I made some investigation. The problem with crashlooping Heapster is due to: The issue was fixed in I'll create a patch release and bump the version.
It looks like is green. I'm hoping that , which is the first build past , will go green as well. (The cycle time on that build is waaaaay too long.)
It looks like we're good! is green now!
for node problem detector.

apiVersion: v1 kind: ReplicationController metadata: <del> name: heapster-v11 </del> <ins> name: heapster-v12 </ins> namespace: kube-system labels: k8s-app: heapster <del> version: v11 </del> <ins> version: v12 </ins> kubernetes.io/cluster-service: "true" spec: replicas: 1 selector: k8s-app: heapster <del> version: v11 </del> <ins> version: v12 </ins> template: metadata: labels: k8s-app: heapster <del> version: v11 </del> <ins> version: v12 </ins> kubernetes.io/cluster-service: "true" spec: containers: <del> - image: gcr.io/google_containers/heapster:v0.18.4 </del> <ins> - image: gcr.io/google_containers/heapster:v0.18.5 </ins> name: heapster resources: # keep request = limit to keep this container in guaranteed class


Question: I noticed this flake by looking at the submit queue. The test is simple, but when it fails, we have no good way to debug it. Below is the test in question. When the test fails, we should print the lines we got in each case.

By("restricting to a time range") time.Sleep(1500 * time.Millisecond) // ensure that startup logs on the node are seen as older than 1s <del> out = runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--since=1s") recent := len(strings.Split(out, "n")) out = runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--since=24h") older := len(strings.Split(out, "n")) Expect(recent).To(BeNumerically("<", older)) </del> <ins> recent_out := runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--since=1s") recent := len(strings.Split(recent_out, "n")) older_out := runKubectlOrDie("log", pod.Name, containerName, nsFlag, "--since=24h") older := len(strings.Split(older_out, "n")) Expect(recent).To(BeNumerically("<", older), "expected recent(%v) to be less than older(%v)nrecent lines:n%vnolder lines:n%vn", recent, older, recent_out, older_out) </ins> }) }) })


Question:
hack/build-cross sets it without looking at the current value. And the value has to be an array, which is a bit awkward to use.
This could be improved indeed. Maybe something like and Haven't tested it though... But why would one want to change the value?
I just wanted to build one, so I did the "obvious" thing: hack/dev-build-and-` It failed for multiple reasons :) , Lucas Käldström wrote:
This issue has been a major downer for me. It's much harder right now than it ought to be to articulate the few platforms for which you'd like to build each of server binaries, client binaries, test binaries, etc. Every time I've tried to do this "the right way," I've spent hours looking at scripts and feeling frustrated that build options aren't documented better. Every time, I end up cutting my losses and resorting to temporary build script hacks instead, just to get the job done.
Doesn't fix this issue exactly, but I've found that you can easily build only if you set .
Taking a look at this today, I also thought that setting seems to be the most reasonable.
/sig release

readonly KUBE_NODE_BINARIES=("${KUBE_NODE_TARGETS[@]##*/}") readonly KUBE_NODE_BINARIES_WIN=("${KUBE_NODE_BINARIES[@]/%/.exe}") <del> if [[ "${KUBE_FASTBUILD:-}" == "true" ]]; then </del> <ins> if [[ -n "${KUBE_BUILD_PLATFORMS:-}" ]]; then readonly KUBE_SERVER_PLATFORMS=(${KUBE_BUILD_PLATFORMS}) readonly KUBE_NODE_PLATFORMS=(${KUBE_BUILD_PLATFORMS}) readonly KUBE_TEST_PLATFORMS=(${KUBE_BUILD_PLATFORMS}) readonly KUBE_CLIENT_PLATFORMS=(${KUBE_BUILD_PLATFORMS}) elif [[ "${KUBE_FASTBUILD:-}" == "true" ]]; then </ins> readonly KUBE_SERVER_PLATFORMS=(linux/amd64) readonly KUBE_NODE_PLATFORMS=(linux/amd64) if [[ "${KUBE_BUILDER_OS:-}" == "darwin"* ]]; then


Question:
The delete command for a replicaset hangs forever. This then manifests as phantom RSs when I delete and recreate a namespace.
- delete is the gift that keeps on giving
cc
Had you already deleted the corresponding Deployment?
I am still seeing this problem. Deployments and replica sets are sticking.
From what I can tell, calling DELETE_COLLECTION on Deployments is returning: And since the controller assumes not found errors are equivalent to already deleted, it thinks the content is deleted. When I manually inspect, I still see the deployment and replica set laying around... //cc
holding on this while i debug some more...
This was a false alarm, apologies.

pendingActionSet.Insert( strings.Join([]string{"delete-collection", "daemonsets", ""}, "-"), strings.Join([]string{"delete-collection", "deployments", ""}, "-"), <ins> strings.Join([]string{"delete-collection", "replicasets", ""}, "-"), </ins> strings.Join([]string{"delete-collection", "jobs", ""}, "-"), strings.Join([]string{"delete-collection", "horizontalpodautoscalers", ""}, "-"), strings.Join([]string{"delete-collection", "ingresses", ""}, "-"),


Question:
The delete command for a replicaset hangs forever. This then manifests as phantom RSs when I delete and recreate a namespace.
- delete is the gift that keeps on giving
cc
Had you already deleted the corresponding Deployment?
I am still seeing this problem. Deployments and replica sets are sticking.
From what I can tell, calling DELETE_COLLECTION on Deployments is returning: And since the controller assumes not found errors are equivalent to already deleted, it thinks the content is deleted. When I manually inspect, I still see the deployment and replica set laying around... //cc
holding on this while i debug some more...
This was a false alarm, apologies.

mockClient := fake.NewSimpleClientset(testInput.testNamespace) if containsVersion(versions, "extensions/v1beta1") { resources := []unversioned.APIResource{} <del> for _, resource := range []string{"daemonsets", "deployments", "jobs", "horizontalpodautoscalers", "ingresses"} { </del> <ins> for _, resource := range []string{"daemonsets", "deployments", "replicasets", "jobs", "horizontalpodautoscalers", "ingresses"} { </ins> resources = append(resources, unversioned.APIResource{Name: resource}) } mockClient.Resources = map[string]*unversioned.APIResourceList{


Question:
The delete command for a replicaset hangs forever. This then manifests as phantom RSs when I delete and recreate a namespace.
- delete is the gift that keeps on giving
cc
Had you already deleted the corresponding Deployment?
I am still seeing this problem. Deployments and replica sets are sticking.
From what I can tell, calling DELETE_COLLECTION on Deployments is returning: And since the controller assumes not found errors are equivalent to already deleted, it thinks the content is deleted. When I manually inspect, I still see the deployment and replica set laying around... //cc
holding on this while i debug some more...
This was a false alarm, apologies.

return estimate, err } } <ins> if containsResource(resources, "replicasets") { err = deleteReplicaSets(kubeClient.Extensions(), namespace) if err != nil { return estimate, err } } </ins> } return estimate, nil }


Question:
The delete command for a replicaset hangs forever. This then manifests as phantom RSs when I delete and recreate a namespace.
- delete is the gift that keeps on giving
cc
Had you already deleted the corresponding Deployment?
I am still seeing this problem. Deployments and replica sets are sticking.
From what I can tell, calling DELETE_COLLECTION on Deployments is returning: And since the controller assumes not found errors are equivalent to already deleted, it thinks the content is deleted. When I manually inspect, I still see the deployment and replica set laying around... //cc
holding on this while i debug some more...
This was a false alarm, apologies.

return expClient.Deployments(ns).DeleteCollection(nil, api.ListOptions{}) } <ins> func deleteReplicaSets(expClient extensions_unversioned.ExtensionsInterface, ns string) error { return expClient.ReplicaSets(ns).DeleteCollection(nil, api.ListOptions{}) } </ins> func deleteIngress(expClient extensions_unversioned.ExtensionsInterface, ns string) error { return expClient.Ingresses(ns).DeleteCollection(nil, api.ListOptions{}) }


Question:
http://pr- is another failure.
Also seen in http://pr-
Again. Not assigned to anyone, so I am sending to for routing
Again: Is there anyone working on this? I will spend some time looking into this.
I think that the name of this issue is misleading - the issue seems to be unrelated to etcd_watcher Basically, if you look into this part of log: So it seems that, the WaitGroup that is problematic is in httptest.Server. So my feeling is that we are simply talking to a server that is already being close and this is ~roughly duplicate of We should just find where exactly the problem is and try to work-around it.
I think this is exactly what we had before. I will send a PR with the fix today.

func TestClient(t *testing.T) { _, s := framework.RunAMaster(t) <del> defer s.Close() </del> <ins> // TODO: Uncomment when fix #19254 // defer s.Close() </ins> ns := api.NamespaceDefault framework.DeleteAllEtcdKeys()


Question:
http://pr- is another failure.
Also seen in http://pr-
Again. Not assigned to anyone, so I am sending to for routing
Again: Is there anyone working on this? I will spend some time looking into this.
I think that the name of this issue is misleading - the issue seems to be unrelated to etcd_watcher Basically, if you look into this part of log: So it seems that, the WaitGroup that is problematic is in httptest.Server. So my feeling is that we are simply talking to a server that is already being close and this is ~roughly duplicate of We should just find where exactly the problem is and try to work-around it.
I think this is exactly what we had before. I will send a PR with the fix today.

func TestSingleWatch(t *testing.T) { _, s := framework.RunAMaster(t) <del> defer s.Close() </del> <ins> // TODO: Uncomment when fix #19254 // defer s.Close() </ins> ns := "blargh" deleteAllEtcdKeys()


Question:
http://pr- is another failure.
Also seen in http://pr-
Again. Not assigned to anyone, so I am sending to for routing
Again: Is there anyone working on this? I will spend some time looking into this.
I think that the name of this issue is misleading - the issue seems to be unrelated to etcd_watcher Basically, if you look into this part of log: So it seems that, the WaitGroup that is problematic is in httptest.Server. So my feeling is that we are simply talking to a server that is already being close and this is ~roughly duplicate of We should just find where exactly the problem is and try to work-around it.
I think this is exactly what we had before. I will send a PR with the fix today.

framework.DeleteAllEtcdKeys() defer framework.DeleteAllEtcdKeys() _, s := framework.RunAMaster(t) <del> defer s.Close() </del> <ins> // TODO: Uncomment when fix #19254 // defer s.Close() </ins> ns := api.NamespaceDefault client := client.NewOrDie(&client.Config{Host: s.URL, ContentConfig: client.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}})


Question:
http://pr- is another failure.
Also seen in http://pr-
Again. Not assigned to anyone, so I am sending to for routing
Again: Is there anyone working on this? I will spend some time looking into this.
I think that the name of this issue is misleading - the issue seems to be unrelated to etcd_watcher Basically, if you look into this part of log: So it seems that, the WaitGroup that is problematic is in httptest.Server. So my feeling is that we are simply talking to a server that is already being close and this is ~roughly duplicate of We should just find where exactly the problem is and try to work-around it.
I think this is exactly what we had before. I will send a PR with the fix today.

func TestExperimentalPrefix(t *testing.T) { _, s := framework.RunAMaster(t) <del> defer s.Close() </del> <ins> // TODO: Uncomment when fix #19254 // defer s.Close() </ins> resp, err := http.Get(s.URL + "/apis/extensions/") if err != nil {