Question: <ins> /* Copyright 2014 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package e2e import ( "encoding/json" "fmt" "time" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/client" "github.com/GoogleCloudPlatform/kubernetes/pkg/util/wait" . "github.com/onsi/ginkgo" ) // partially cloned from webserver.go type State struct { Received map[string]int } func testPreStop(c *client.Client, ns string) { // This is the server that will receive the preStop notification podDescr := &api.Pod{ ObjectMeta: api.ObjectMeta{ Name: "server", }, Spec: api.PodSpec{ Containers: []api.Container{ { Name:  "server", Image: "gcr.io/google_containers/nettest:1.3", Ports: []api.ContainerPort{{ContainerPort: 8080}}, }, }, }, } By(fmt.Sprintf("Creating server pod %s in namespace %s", podDescr.Name, ns)) _, err := c.Pods(ns).Create(podDescr) expectNoError(err, fmt.Sprintf("creating pod %s", podDescr.Name)) // At the end of the test, clean up by removing the pod. defer func() { By("Deleting the server pod") c.Pods(ns).Delete(podDescr.Name, nil) }() By("Waiting for pods to come up.") err = waitForPodRunningInNamespace(c, podDescr.Name, ns) expectNoError(err, "waiting for server pod to start") val := "{"Source": "prestop"}" podOut, err := c.Pods(ns).Get(podDescr.Name) expectNoError(err, "getting pod info") preStopDescr := &api.Pod{ ObjectMeta: api.ObjectMeta{ Name: "tester", }, Spec: api.PodSpec{ Containers: []api.Container{ { Name:    "tester", Image:   "busybox", Command: []string{"sleep", "600"}, Lifecycle: &api.Lifecycle{ PreStop: &api.Handler{ Exec: &api.ExecAction{ Command: []string{ "wget", "-O-", "--post-data=" + val, fmt.Sprintf("http://%s:8080/write", podOut.Status.PodIP), }, }, }, }, }, }, }, } By(fmt.Sprintf("Creating tester pod %s in namespace %s", podDescr.Name, ns)) _, err = c.Pods(ns).Create(preStopDescr) expectNoError(err, fmt.Sprintf("creating pod %s", preStopDescr.Name)) deletePreStop := true // At the end of the test, clean up by removing the pod. defer func() { if deletePreStop { By("Deleting the tester pod") c.Pods(ns).Delete(preStopDescr.Name, nil) } }() err = waitForPodRunningInNamespace(c, preStopDescr.Name, ns) expectNoError(err, "waiting for tester pod to start") // Delete the pod with the preStop handler. By("Deleting pre-stop pod") if err := c.Pods(ns).Delete(preStopDescr.Name, nil); err == nil { deletePreStop = false } expectNoError(err, fmt.Sprintf("deleting pod: %s", preStopDescr.Name)) // Validate that the server received the web poke. err = wait.Poll(time.Second*5, time.Second*60, func() (bool, error) { if body, err := c.Get(). Namespace(ns).Prefix("proxy"). Resource("pods"). Name(podDescr.Name). Suffix("read"). DoRaw(); err != nil { By(fmt.Sprintf("Error validating prestop: %v", err)) } else { state := State{} err := json.Unmarshal(body, &state) if err != nil { Logf("Error parsing: %v", err) return false, nil } if state.Received["prestop"] != 0 { return true, nil } Logf("Saw: %s", string(body)) } return false, nil }) expectNoError(err, "validating pre-stop.") } var _ = Describe("PreStop", func() { f := NewFramework("prestop") It("should call prestop when killing a pod", func() { testPreStop(f.Client, f.Namespace.Name) }) }) </ins>

PreStop in Lifecycle is a very excited feature that I want it so much. With it, I can do some clean work before killing the Pod/Container, for example, I need this container stop receiving requests first, then complete processing its requests already received before it is killed. However It does not work as PostStop now. And I have checked the latest code and found it is still not implemented. Could anyone give a time point on it?
Sorry, prestop hasn't been implemented yet. See discussion in .
I'll implement it once is finished
thanks, I will also subscribe
/subscribe
Moved to post-1.0
PR is in flight.
was merged. We should add an e2e test, though.
Tried by E2E, it works! And it can be closed. Well done.


Question: } func TestDeleteOrEvict(t *testing.T) { <del> for _, evictionSupported := range []bool{true, false} { evictionSupported := evictionSupported t.Run(fmt.Sprintf("evictionSupported=%v", evictionSupported), func(t *testing.T) { h := &Helper{ Out:                os.Stdout, GracePeriodSeconds: 10, } // Create 4 pods, and try to remove the first 2 var expectedEvictions []policyv1beta1.Eviction var create []runtime.Object deletePods := []corev1.Pod{} for i := 1; i <= 4; i++ { pod := &corev1.Pod{} pod.Name = fmt.Sprintf("mypod-%d", i) pod.Namespace = "default" </del> <ins> tests := []struct { description       string evictionSupported bool disableEviction   bool }{ { description:       "eviction supported/enabled", evictionSupported: true, disableEviction:   false, }, { description:       "eviction unsupported/disabled", evictionSupported: false, disableEviction:   false, }, { description:       "eviction supported/disabled", evictionSupported: true, disableEviction:   true, }, { description:       "eviction unsupported/disabled", evictionSupported: false, disableEviction:   false, }, } for _, tc := range tests { t.Run(tc.description, func(t *testing.T) { h := &Helper{ Out:                os.Stdout, GracePeriodSeconds: 10, } </ins> <del> create = append(create, pod) if i <= 2 { deletePods = append(deletePods, *pod) </del> <ins> // Create 4 pods, and try to remove the first 2 var expectedEvictions []policyv1beta1.Eviction var create []runtime.Object deletePods := []corev1.Pod{} for i := 1; i <= 4; i++ { pod := &corev1.Pod{} pod.Name = fmt.Sprintf("mypod-%d", i) pod.Namespace = "default" </ins> <del> if evictionSupported { eviction := policyv1beta1.Eviction{} eviction.Kind = "Eviction" eviction.APIVersion = "policy/v1" eviction.Namespace = pod.Namespace eviction.Name = pod.Name </del> <ins> create = append(create, pod) if i <= 2 { deletePods = append(deletePods, *pod) </ins> <del> gracePeriodSeconds := int64(h.GracePeriodSeconds) eviction.DeleteOptions = &metav1.DeleteOptions{ GracePeriodSeconds: &gracePeriodSeconds, } </del> <ins> if tc.evictionSupported && !tc.disableEviction { eviction := policyv1beta1.Eviction{} eviction.Kind = "Eviction" eviction.APIVersion = "policy/v1" eviction.Namespace = pod.Namespace eviction.Name = pod.Name </ins> <del> expectedEvictions = append(expectedEvictions, eviction) </del> <ins> gracePeriodSeconds := int64(h.GracePeriodSeconds) eviction.DeleteOptions = &metav1.DeleteOptions{ GracePeriodSeconds: &gracePeriodSeconds, </ins> } <ins> expectedEvictions = append(expectedEvictions, eviction) </ins> } } <ins> } </ins> <del> // Build the fake client k := fake.NewSimpleClientset(create...) if evictionSupported { addEvictionSupport(t, k) } h.Client = k </del> <ins> // Build the fake client k := fake.NewSimpleClientset(create...) if tc.evictionSupported { addEvictionSupport(t, k) } h.Client = k h.DisableEviction = tc.disableEviction // Do the eviction if err := h.DeleteOrEvictPods(deletePods); err != nil { t.Fatalf("error from DeleteOrEvictPods: %v", err) } </ins> <del> // Do the eviction if err := h.DeleteOrEvictPods(deletePods); err != nil { t.Fatalf("error from DeleteOrEvictPods: %v", err) </del> <ins> // Test that other pods are still there var remainingPods []string { podList, err := k.CoreV1().Pods("").List(metav1.ListOptions{}) if err != nil { t.Fatalf("error listing pods: %v", err) </ins> } <del> // Test that other pods are still there var remainingPods []string { podList, err := k.CoreV1().Pods("").List(metav1.ListOptions{}) if err != nil { t.Fatalf("error listing pods: %v", err) } for _, pod := range podList.Items { remainingPods = append(remainingPods, pod.Namespace+"/"+pod.Name) } sort.Strings(remainingPods) } expected := []string{"default/mypod-3", "default/mypod-4"} if !reflect.DeepEqual(remainingPods, expected) { t.Errorf("unexpected remaining pods after DeleteOrEvictPods; actual %v; expected %v", remainingPods, expected) </del> <ins> for _, pod := range podList.Items { remainingPods = append(remainingPods, pod.Namespace+"/"+pod.Name) </ins> } <ins> sort.Strings(remainingPods) } expected := []string{"default/mypod-3", "default/mypod-4"} if !reflect.DeepEqual(remainingPods, expected) { t.Errorf("%s: unexpected remaining pods after DeleteOrEvictPods; actual %v; expected %v", tc.description, remainingPods, expected) } </ins> <del> // Test that pods were evicted as expected var actualEvictions []policyv1beta1.Eviction for _, action := range k.Actions() { if action.GetVerb() != "create" || action.GetResource().Resource != "pods" || action.GetSubresource() != "eviction" { continue } eviction := *action.(ktest.CreateAction).GetObject().(*policyv1beta1.Eviction) actualEvictions = append(actualEvictions, eviction) } sort.Slice(actualEvictions, func(i, j int) bool { return actualEvictions[i].Name < actualEvictions[j].Name }) if !reflect.DeepEqual(actualEvictions, expectedEvictions) { t.Errorf("unexpected evictions; actual %v; expected %v", actualEvictions, expectedEvictions) </del> <ins> // Test that pods were evicted as expected var actualEvictions []policyv1beta1.Eviction for _, action := range k.Actions() { if action.GetVerb() != "create" || action.GetResource().Resource != "pods" || action.GetSubresource() != "eviction" { continue </ins> } <ins> eviction := *action.(ktest.CreateAction).GetObject().(*policyv1beta1.Eviction) actualEvictions = append(actualEvictions, eviction) } sort.Slice(actualEvictions, func(i, j int) bool { return actualEvictions[i].Name < actualEvictions[j].Name </ins> }) <ins> if !reflect.DeepEqual(actualEvictions, expectedEvictions) { t.Errorf("%s: unexpected evictions; actual %v; expected %v", tc.description, actualEvictions, expectedEvictions) } }) </ins> } }

<!-- Please only use this template for submitting enhancement requests --What would you like to be : Allow to have an additional flag (e.g. or ) to allow a to drain pods off a node, even if they violate PodDisruptionBudgets Why is this needed: There are times where you want to drain pods off a node even if violating a PDB. For example, if you are running a one-worker node kubernetes cluster (not minikube), and you want to upgrade it. Or, like me, you are running a test cluster and want to scale down to zero nodes when you're not going to be using the cluster for a while in order to save costs. If that worker node has a deployment with a PDB, then will get stuck as it cannot get rid of the pod while respecting the PDB, which is expected; but in this case, it is a hindrance. Adding the switch will allow users to explicitly acknowledge that they want to ignore the PDBs (and accept the corresponding downtime for an application/service)
/sig node
+1
+1 Our company offers Kubernetes as PaaS. We manage Kubernetes, including IaaS OS updates and if a customer does not specify the PDB correctly, we are not able to drain the node for the OS updates. A flag would be werry helpfull for our use case.
+1
/active
I have created to address this. This is a pretty simple implementation, but I think it achieves the desired results. A suitable workflow would be: 1) Configure you component to use drain as normal 2) After some timeout, run drain with --disable-eviction option.
I'll wait until the PR is merged and released before testing, but if I understand right, it would require me do this, right? If so, I think it would be better if we did it a different way and have one call, something like this: Where implies that if the drain is still running after this time, switch to the mode without having to call kubectl again
I think that would require a sizeable refactor or add a great deal of complexity to the code, the same could be achieved with In practical terms, I'm focused on automatic node remediation via destroying the node and getting a new one. The timeout would be very large, the drain library is blocking, so I want it to fail quickly and requeue.


Question: } func TestDeleteOrEvict(t *testing.T) { <del> for _, evictionSupported := range []bool{true, false} { evictionSupported := evictionSupported t.Run(fmt.Sprintf("evictionSupported=%v", evictionSupported), func(t *testing.T) { h := &Helper{ Out:                os.Stdout, GracePeriodSeconds: 10, } // Create 4 pods, and try to remove the first 2 var expectedEvictions []policyv1beta1.Eviction var create []runtime.Object deletePods := []corev1.Pod{} for i := 1; i <= 4; i++ { pod := &corev1.Pod{} pod.Name = fmt.Sprintf("mypod-%d", i) pod.Namespace = "default" </del> <ins> tests := []struct { description       string evictionSupported bool disableEviction   bool }{ { description:       "eviction supported/enabled", evictionSupported: true, disableEviction:   false, }, { description:       "eviction unsupported/disabled", evictionSupported: false, disableEviction:   false, }, { description:       "eviction supported/disabled", evictionSupported: true, disableEviction:   true, }, { description:       "eviction unsupported/disabled", evictionSupported: false, disableEviction:   false, }, } for _, tc := range tests { t.Run(tc.description, func(t *testing.T) { h := &Helper{ Out:                os.Stdout, GracePeriodSeconds: 10, } </ins> <del> create = append(create, pod) if i <= 2 { deletePods = append(deletePods, *pod) </del> <ins> // Create 4 pods, and try to remove the first 2 var expectedEvictions []policyv1beta1.Eviction var create []runtime.Object deletePods := []corev1.Pod{} for i := 1; i <= 4; i++ { pod := &corev1.Pod{} pod.Name = fmt.Sprintf("mypod-%d", i) pod.Namespace = "default" </ins> <del> if evictionSupported { eviction := policyv1beta1.Eviction{} eviction.Kind = "Eviction" eviction.APIVersion = "policy/v1" eviction.Namespace = pod.Namespace eviction.Name = pod.Name </del> <ins> create = append(create, pod) if i <= 2 { deletePods = append(deletePods, *pod) </ins> <del> gracePeriodSeconds := int64(h.GracePeriodSeconds) eviction.DeleteOptions = &metav1.DeleteOptions{ GracePeriodSeconds: &gracePeriodSeconds, } </del> <ins> if tc.evictionSupported && !tc.disableEviction { eviction := policyv1beta1.Eviction{} eviction.Kind = "Eviction" eviction.APIVersion = "policy/v1" eviction.Namespace = pod.Namespace eviction.Name = pod.Name </ins> <del> expectedEvictions = append(expectedEvictions, eviction) </del> <ins> gracePeriodSeconds := int64(h.GracePeriodSeconds) eviction.DeleteOptions = &metav1.DeleteOptions{ GracePeriodSeconds: &gracePeriodSeconds, </ins> } <ins> expectedEvictions = append(expectedEvictions, eviction) </ins> } } <ins> } </ins> <del> // Build the fake client k := fake.NewSimpleClientset(create...) if evictionSupported { addEvictionSupport(t, k) } h.Client = k </del> <ins> // Build the fake client k := fake.NewSimpleClientset(create...) if tc.evictionSupported { addEvictionSupport(t, k) } h.Client = k h.DisableEviction = tc.disableEviction // Do the eviction if err := h.DeleteOrEvictPods(deletePods); err != nil { t.Fatalf("error from DeleteOrEvictPods: %v", err) } </ins> <del> // Do the eviction if err := h.DeleteOrEvictPods(deletePods); err != nil { t.Fatalf("error from DeleteOrEvictPods: %v", err) </del> <ins> // Test that other pods are still there var remainingPods []string { podList, err := k.CoreV1().Pods("").List(metav1.ListOptions{}) if err != nil { t.Fatalf("error listing pods: %v", err) </ins> } <del> // Test that other pods are still there var remainingPods []string { podList, err := k.CoreV1().Pods("").List(metav1.ListOptions{}) if err != nil { t.Fatalf("error listing pods: %v", err) } for _, pod := range podList.Items { remainingPods = append(remainingPods, pod.Namespace+"/"+pod.Name) } sort.Strings(remainingPods) } expected := []string{"default/mypod-3", "default/mypod-4"} if !reflect.DeepEqual(remainingPods, expected) { t.Errorf("unexpected remaining pods after DeleteOrEvictPods; actual %v; expected %v", remainingPods, expected) </del> <ins> for _, pod := range podList.Items { remainingPods = append(remainingPods, pod.Namespace+"/"+pod.Name) </ins> } <ins> sort.Strings(remainingPods) } expected := []string{"default/mypod-3", "default/mypod-4"} if !reflect.DeepEqual(remainingPods, expected) { t.Errorf("%s: unexpected remaining pods after DeleteOrEvictPods; actual %v; expected %v", tc.description, remainingPods, expected) } </ins> <del> // Test that pods were evicted as expected var actualEvictions []policyv1beta1.Eviction for _, action := range k.Actions() { if action.GetVerb() != "create" || action.GetResource().Resource != "pods" || action.GetSubresource() != "eviction" { continue } eviction := *action.(ktest.CreateAction).GetObject().(*policyv1beta1.Eviction) actualEvictions = append(actualEvictions, eviction) } sort.Slice(actualEvictions, func(i, j int) bool { return actualEvictions[i].Name < actualEvictions[j].Name }) if !reflect.DeepEqual(actualEvictions, expectedEvictions) { t.Errorf("unexpected evictions; actual %v; expected %v", actualEvictions, expectedEvictions) </del> <ins> // Test that pods were evicted as expected var actualEvictions []policyv1beta1.Eviction for _, action := range k.Actions() { if action.GetVerb() != "create" || action.GetResource().Resource != "pods" || action.GetSubresource() != "eviction" { continue </ins> } <ins> eviction := *action.(ktest.CreateAction).GetObject().(*policyv1beta1.Eviction) actualEvictions = append(actualEvictions, eviction) } sort.Slice(actualEvictions, func(i, j int) bool { return actualEvictions[i].Name < actualEvictions[j].Name </ins> }) <ins> if !reflect.DeepEqual(actualEvictions, expectedEvictions) { t.Errorf("%s: unexpected evictions; actual %v; expected %v", tc.description, actualEvictions, expectedEvictions) } }) </ins> } }

I'm going to be performing the calculation of when it's appropriate to evict/delete outside of the drain library, and I suspect most will be as well, so there's not really a great incentive to do it in the drain library itself IMO.
That's all good, I have no issue using the flag as long as does error out and return a non-zero code as expected (there's been a few times where it seems to error out, but still return 0)


Question: # TODO update to dns role once we have one. ${KUBECTL} --kubeconfig="${CERT_DIR}/admin.kubeconfig" create clusterrolebinding system:kube-dns --clusterrole=cluster-admin --serviceaccount=kube-system:default <del> # use kubectl to create kubedns rc and service </del> <ins> # use kubectl to create kubedns deployment and service </ins> ${KUBECTL} --kubeconfig="${CERT_DIR}/admin.kubeconfig" --namespace=kube-system create -f kubedns-deployment.yaml ${KUBECTL} --kubeconfig="${CERT_DIR}/admin.kubeconfig" --namespace=kube-system create -f kubedns-svc.yaml <del> echo "Kube-dns rc and service successfully deployed." </del> <ins> echo "Kube-dns deployment and service successfully deployed." </ins> rm  kubedns-deployment.yaml kubedns-svc.yaml fi }

Kubernetes version (use ): Environment: Cloud provider or hardware configuration: Vagrant OS (e.g. from /etc/os-release): CoreOS Kernel (e.g. ):Install tools: Vagrant & Others: What happened: What you expected to happen: Success How to reproduce it (as minimally and precisely as possible): Followed instructions at Tried to run Received: Anything else do we need to know: From googling, it seems several of the fields in the config are for when using OpenShift origin. But creating the replication controller is a prerequisite to launching OpenShift Origin. Chicken & egg problem.


Question: } } } <ins> func TestApiserverMetricsPods(t *testing.T) { callOrDie := func(_ interface{}, err error) { if err != nil { t.Fatalf("unexpected error: %v", err) } } makePod := func(labelValue string) *v1.Pod { return &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:   "foo", Labels: map[string]string{"foo": labelValue}, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { Name:  "container", Image: "image", }, }, }, } } _, server, closeFn := framework.RunAMaster(framework.NewMasterConfig()) defer closeFn() client, err := clientset.NewForConfig(&restclient.Config{Host: server.URL, QPS: -1}) if err != nil { t.Fatalf("Error in create clientset: %v", err) } c := client.CoreV1().Pods(metav1.NamespaceDefault) for _, tc := range []struct { name     string executor func() want string }{ { name: "create pod", executor: func() { callOrDie(c.Create(context.TODO(), makePod("foo"), metav1.CreateOptions{})) }, want: `apiserver_request_total{code="201", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="resource", subresource="", verb="POST", version="v1"}`, }, { name: "update pod", executor: func() { callOrDie(c.Update(context.TODO(), makePod("bar"), metav1.UpdateOptions{})) }, want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="resource", subresource="", verb="PUT", version="v1"}`, }, { name: "update pod status", executor: func() { callOrDie(c.UpdateStatus(context.TODO(), makePod("bar"), metav1.UpdateOptions{})) }, want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="resource", subresource="status", verb="PUT", version="v1"}`, }, { name: "get pod", executor: func() { callOrDie(c.Get(context.TODO(), "foo", metav1.GetOptions{})) }, want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="resource", subresource="", verb="GET", version="v1"}`, }, { name: "list pod", executor: func() { callOrDie(c.List(context.TODO(), metav1.ListOptions{})) }, want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="namespace", subresource="", verb="LIST", version="v1"}`, }, { name: "delete pod", executor: func() { callOrDie(nil, c.Delete(context.TODO(), "foo", metav1.DeleteOptions{})) }, want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="pods", scope="resource", subresource="", verb="DELETE", version="v1"}`, }, } { t.Run(tc.name, func(t *testing.T) { baseSamples, err := getSamples(server) if err != nil { t.Fatal(err) } tc.executor() updatedSamples, err := getSamples(server) if err != nil { t.Fatal(err) } newSamples := diffMetrics(updatedSamples, baseSamples) found := false for _, sample := range newSamples { if sample.Metric.String() == tc.want { found = true break } } if !found { t.Fatalf("could not find metric for API call >%s< among samples >%+v<", tc.name, newSamples) } }) } } func TestApiserverMetricsNamespaces(t *testing.T) { callOrDie := func(_ interface{}, err error) { if err != nil { t.Fatalf("unexpected error: %v", err) } } makeNamespace := func(labelValue string) *v1.Namespace { return &v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name:   "foo", Labels: map[string]string{"foo": labelValue}, }, } } _, server, closeFn := framework.RunAMaster(framework.NewMasterConfig()) defer closeFn() client, err := clientset.NewForConfig(&restclient.Config{Host: server.URL, QPS: -1}) if err != nil { t.Fatalf("Error in create clientset: %v", err) } c := client.CoreV1().Namespaces() for _, tc := range []struct { name     string executor func() want string }{ { name: "create namespace", executor: func() { callOrDie(c.Create(context.TODO(), makeNamespace("foo"), metav1.CreateOptions{})) }, want: `apiserver_request_total{code="201", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="resource", subresource="", verb="POST", version="v1"}`, }, { name: "update namespace", executor: func() { callOrDie(c.Update(context.TODO(), makeNamespace("bar"), metav1.UpdateOptions{})) }, want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="resource", subresource="", verb="PUT", version="v1"}`, }, { name: "update namespace status", executor: func() { callOrDie(c.UpdateStatus(context.TODO(), makeNamespace("bar"), metav1.UpdateOptions{})) }, want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="resource", subresource="status", verb="PUT", version="v1"}`, }, { name: "get namespace", executor: func() { callOrDie(c.Get(context.TODO(), "foo", metav1.GetOptions{})) }, want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="resource", subresource="", verb="GET", version="v1"}`, }, { name: "list namespace", executor: func() { callOrDie(c.List(context.TODO(), metav1.ListOptions{})) }, want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="cluster", subresource="", verb="LIST", version="v1"}`, }, { name: "delete namespace", executor: func() { callOrDie(nil, c.Delete(context.TODO(), "foo", metav1.DeleteOptions{})) }, want: `apiserver_request_total{code="200", component="apiserver", contentType="application/json", dry_run="", group="", resource="namespaces", scope="resource", subresource="", verb="DELETE", version="v1"}`, }, } { t.Run(tc.name, func(t *testing.T) { baseSamples, err := getSamples(server) if err != nil { t.Fatal(err) } tc.executor() updatedSamples, err := getSamples(server) if err != nil { t.Fatal(err) } newSamples := diffMetrics(updatedSamples, baseSamples) found := false for _, sample := range newSamples { if sample.Metric.String() == tc.want { found = true break } } if !found { t.Fatalf("could not find metric for API call >%s< among samples >%+v<", tc.name, newSamples) } }) } } func getSamples(s *httptest.Server) (model.Samples, error) { metrics, err := scrapeMetrics(s) if err != nil { return nil, err } samples, ok := metrics["apiserver_request_total"] if !ok { return nil, errors.New("apiserver_request_total doesn't exist") } return samples, nil } func diffMetrics(newSamples model.Samples, oldSamples model.Samples) model.Samples { samplesDiff := model.Samples{} for _, sample := range newSamples { if !sampleExistsInSamples(sample, oldSamples) { samplesDiff = append(samplesDiff, sample) } } return samplesDiff } func sampleExistsInSamples(s *model.Sample, samples model.Samples) bool { for _, sample := range samples { if sample.Equal(s) { return true } } return false } </ins>

In we discovered a bug where kube-apiserver metrics were being incorrectly reported (GET vs LIST problem). In this was addressed together with test exercising that proper metrics are exposed. However, this test isn't ideal because if params wouldn't be swapped, it wouldn't catch them. We should ideally develop a lower-level unit test for mapping url paths to exact values of labels/parameters. /sig apimachinery
The label(s) cannot be applied, because the repository doesn't have them <detailsIn response to : In we discovered a bug where kube-apiserver metrics were being incorrectly reported (GET vs LIST problem). In this was addressed together with test exercising that proper metrics are exposed. We should ideally develop a lower-level unit test for mapping url paths to exact values of labels/parameters. Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig api-machinery
are you planning on sending a fix for this one too? Thanks!
Not very soon - if someone else could help with it, it would be great.
/assign
/triage accepted
cc


Question: manager.nodeStore.Add(c.oldNode) c.ds.Spec.UpdateStrategy = *strategy manager.dsStore.Add(c.ds) <del> syncAndValidateDaemonSets(t, manager, c.ds, podControl, 0, 0, 0) </del> <ins> expectedEvents := 0 if c.expectedEventsFunc != nil { expectedEvents = c.expectedEventsFunc(strategy.Type) } syncAndValidateDaemonSets(t, manager, c.ds, podControl, 0, 0, expectedEvents) </ins> manager.enqueueDaemonSet = func(ds *apps.DaemonSet) { if ds.Name == "ds" {

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: a cluster, And node1 with resource 1 DaemonSet, request resource 4 run , There is no pod created. resource to the node: run , There is still no pod created. What you expected to happen: In step 5, There is one pod created. How to reproduce it (as minimally and precisely as possible): see above description Anything else we need to know?: Environment: Kubernetes version (use ):1.10.5 Cloud provider or hardware configuration: OS (e.g. from /etc/os-release):CentOS7 Kernel (e.g. ): Install tools: Others: /sig app
xref
Thanks! is not work here! Because: DaemonSetsController didn't watch the changes.(we use to updateNode.) we restart kubelet, The DaemonSetsController watch the changes, but it seems that the node info is not updated, and still failed to create pod:
I found it is a bug here, ignore the changes. I made a fix , PTAL! Thanks!
Any workarounds for old cluster versions? Up: looks like in my case helped but after some delay.
, which version of k8s are you using? If it's sunset, you have to merge the pr yourself :). I think restart kubelet trigger delete/add to make sure it sync .
Sorry, I didn't say
I think we still support 1.12 for now :) let's cherry pick it if not and wait for next 1.12 release.


Question: return fmt.Errorf("size of file %s is %d, expected %d", fpath, size, expectSize) } <del> By("verifying file content") // use `grep ... -f` rather than the expected content in a variable to reduce logging rtnstr, err = podExec(pod, fmt.Sprintf("grep -c -m1 -f %s %s", dd_input, fpath)) </del> <ins> By("verifying file hash") rtnstr, err = podExec(pod, fmt.Sprintf("md5sum %s | cut -d' ' -f1", fpath)) </ins> if err != nil { <del> return fmt.Errorf("unable to test file content via `grep %s`: %v", fpath, err) </del> <ins> return fmt.Errorf("unable to test file hash via `md5sum %s`: %v", fpath, err) </ins> } <del> foundCnt, err := strconv.Atoi(strings.TrimSuffix(rtnstr, "n")) if err != nil { return fmt.Errorf("unable to convert string %q to int: %v", rtnstr, err) </del> <ins> actualHash := strings.TrimSuffix(rtnstr, "n") expectedHash, ok := md5hashes[expectSize] if !ok { return fmt.Errorf("File hash is unknown for file size %d. Was a new file size added to the test suite?", expectSize) </ins> } <del> if foundCnt == 0 { rtnstr, err = podExec(pod, fmt.Sprintf("cat %s", dd_input)) if err != nil || len(rtnstr) == 0 { return fmt.Errorf("string not found in file %s and unable to read dd's input file %s: %v", fpath, dd_input, err) } return fmt.Errorf("string %q not found in file %s", rtnstr, fpath) </del> <ins> if actualHash != expectedHash { return fmt.Errorf("MD5 hash is incorrect for file %s with size %d. Expected: `%s`; Actual: `%s`", fpath, expectSize, expectedHash, actualHash) </ins> } return nil

The "Volume plugin streaming [Slow] NFS should write files of various sizes, verify size, validate content" e2e test is failing on our 2k-node gce clusters (https://k8s-) with the following error: cc
Could you take a look into it or reassign as apt? This is part of the correctness suite and large cluster tests are release-blocking.
/assign
is this run using the kubemark environment? I'll also take a look. It could be that the commands this test needs to execute may not work with kubemark. /assign
cc
No, this is not kubemark. This is on a real 2000-node cluster.
Also all other volume-related e2e tests are passing, except this one.
Looking at the log I see that the nfs test fails trying to verify the test file's content. The test exec's into the nfs client pod and executes: The return code from above is 127. The test then deletes the nfs client pod calling which works. The nfs-server pod is deleted in the AfterEach. For some reason the reported error is 137 but the returned err 127. The same command is exec'd for all of the tested storage plugins so not sure why it's failing just for nfs. Error 127 might be "command not found".
The other strange thing is that it's only failing in this suite and not the others.
When I look at the kubelet log, I see the exec command coming through: I also see these suspicious kubelet soon afterwards: So I think the grep command is causing the node to run out of memory and get evicted. I think we can fix this either by: Making the grep command more memory efficient. I'm guessing it's reading the whole file into memory or, Make this test Serial so that it doesn't run in parallel with other pods, since it consumes a lot of memory.
An unrelated issue, is that the node allocatable feature didn't actually evict the pod. Instead this pod got a system OOM kill. /cc
related to the memcg notifications, not node allocatable. Thanks!
any theories why we see this only for the nfs-io tests, and not the glusterfs-io test for instance?
the gluster test didn't run on this in this suite.


Question: return fmt.Errorf("size of file %s is %d, expected %d", fpath, size, expectSize) } <del> By("verifying file content") // use `grep ... -f` rather than the expected content in a variable to reduce logging rtnstr, err = podExec(pod, fmt.Sprintf("grep -c -m1 -f %s %s", dd_input, fpath)) </del> <ins> By("verifying file hash") rtnstr, err = podExec(pod, fmt.Sprintf("md5sum %s | cut -d' ' -f1", fpath)) </ins> if err != nil { <del> return fmt.Errorf("unable to test file content via `grep %s`: %v", fpath, err) </del> <ins> return fmt.Errorf("unable to test file hash via `md5sum %s`: %v", fpath, err) </ins> } <del> foundCnt, err := strconv.Atoi(strings.TrimSuffix(rtnstr, "n")) if err != nil { return fmt.Errorf("unable to convert string %q to int: %v", rtnstr, err) </del> <ins> actualHash := strings.TrimSuffix(rtnstr, "n") expectedHash, ok := md5hashes[expectSize] if !ok { return fmt.Errorf("File hash is unknown for file size %d. Was a new file size added to the test suite?", expectSize) </ins> } <del> if foundCnt == 0 { rtnstr, err = podExec(pod, fmt.Sprintf("cat %s", dd_input)) if err != nil || len(rtnstr) == 0 { return fmt.Errorf("string not found in file %s and unable to read dd's input file %s: %v", fpath, dd_input, err) } return fmt.Errorf("string %q not found in file %s", rtnstr, fpath) </del> <ins> if actualHash != expectedHash { return fmt.Errorf("MD5 hash is incorrect for file %s with size %d. Expected: `%s`; Actual: `%s`", fpath, expectSize, expectedHash, actualHash) </ins> } return nil

I'm suspecting that the reason why we didn't see a problem with this test on other suites is that the load could be higher here. I've asked to see if we can reduce the memory usage of the grep call.
pointed out that there are no newlines in the generated file, so when grep is called it could buffer the entire file in memory. Adding a newline at the end of the dd input file should do it.
/priority critical-urgent
[MILESTONENOTIFIER] Milestone Labels Complete Issue label settings: sig/scalability sig/storage: Issue will be escalated to these SIGs if needed. priority/critical-urgent: Never automatically move out of a release milestone; continually escalate to contributor and SIG through all available channels. kind/bug: Fixes a bug discovered during the current release. <detailsAdditional instructions available <a href=""here</a</details
It seems like even if the generated file is broken up by newlines, grep could still use on the order of hundreds of MB of memory for a 100MiB file (even with the -m1 and -F options). We could check the file line by line, but it would take a long time. A better solution would be to check the MD5 file hash. It's fast and uses little memory. The expected hashes will be hard-coded into the test, and if different file sizes are in the future, a hash needs to be computed and as well.
Thanks a lot for fixing this!
Ref


Question: "crypto/x509" "crypto/x509/pkix" "fmt" <ins> "net" </ins> "strings" "testing" "time"

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: I'm testing the new option in kubernetes 1.12 and it seems that kubelet always requests the signing of a new CSR and tries to rotate the certificates when it is started. This is happening even though there's a valid (brand new) certificate on disk issued by kube-controller-manager. From what I can tell, this is happening because kubelet implements and is set after a successful certificate rotation, but is never persisted to disk. If kubelet is restarted, then the last requested CSR is nil. What you expected to happen: Ideally, kubelet should only request the signing of a new CSR if it doesn't already have a serving certificate or it's about to expire. How to reproduce it (as minimally and precisely as possible): Restart a kubelet instance that had it's serving certificate issued by kube-controller-manager Check and see that it requested the signing of a new CSR Anything else we need to know?: kubelet is being run with . , , and are all omitted to use the default values. The TLS bootstrapping for client certificates is currently not being used--just the bootstrapping for serving certificates. Environment: Kubernetes version: v1.12.0 - built from source at the tag () Cloud provider or hardware configuration: Bare metal OS: CentOS 7.5.1804 Kernel: Install tools: Custom RPMs built from this git repo with no additional patches Others:
/sig node
/sig auth
Not sure if this is the ideal fix, but I've patched it so that it compares the current CSR template to the last requested template only if it doesn't already match the current cert.
cc
thanks for the proposed fix. I think it looks good. Could you send a PR? We should also remove entirely and only look at current cert and template. does that sound reasonable?
Sure thing, I'd be happy to do that. EDIT: I will submit a PR after approval from my employer.
I think that's probably ok, as long as it checks everything we include in the template and we consider a certificate missing anything we request to be cause for re-requesting another cert (an issuer couldn't choose to issue a cert that drops some SANs and us not request another cert) left some comments in
This regression is a frequent flyer.


Question: "crypto/x509" "crypto/x509/pkix" "fmt" <ins> "net" </ins> "strings" "testing" "time"

I'd love to see an integration test that would catch this (but that shouldn't block a fix).
Thanks for the comments! Now that I have the CLA signed, I'll address those and submit a PR.
Just submitted the PR:


Question: unmanagedNodes:         sets.NewString(), routeCIDRs:             map[string]string{}, resourceRequestBackoff: resourceRequestBackoff, <del> DisksClient:                     newAzDisksClient(azClientConfig), SnapshotsClient:                 newSnapshotsClient(azClientConfig), RoutesClient:                    newAzRoutesClient(azClientConfig), SubnetsClient:                   newAzSubnetsClient(azClientConfig), InterfacesClient:                newAzInterfacesClient(azClientConfig), RouteTablesClient:               newAzRouteTablesClient(azClientConfig), LoadBalancerClient:              newAzLoadBalancersClient(azClientConfig), SecurityGroupsClient:            newAzSecurityGroupsClient(azClientConfig), StorageAccountClient:            newAzStorageAccountClient(azClientConfig), VirtualMachinesClient:           newAzVirtualMachinesClient(azClientConfig), PublicIPAddressesClient:         newAzPublicIPAddressesClient(azClientConfig), VirtualMachineSizesClient:       newAzVirtualMachineSizesClient(azClientConfig), VirtualMachineScaleSetsClient:   newAzVirtualMachineScaleSetsClient(azClientConfig), VirtualMachineScaleSetVMsClient: newAzVirtualMachineScaleSetVMsClient(azClientConfig), FileClient:                      &azureFileClient{env: *env}, </del> } az.metadata, err = NewInstanceMetadataService(metadataURL) if err != nil { return nil, err } <ins> // No credentials provided, InstanceMetadataService would be used for getting Azure resources. // Note that this only applies to Kubelet, controller-manager should configure credentials for managing Azure resources. if servicePrincipalToken == nil { return &az, nil } // Initialize Azure clients. azClientConfig := &azClientConfig{ subscriptionID:                 config.SubscriptionID, resourceManagerEndpoint:        env.ResourceManagerEndpoint, servicePrincipalToken:          servicePrincipalToken, rateLimiterReader:              operationPollRateLimiter, rateLimiterWriter:              operationPollRateLimiterWrite, CloudProviderBackoffRetries:    config.CloudProviderBackoffRetries, CloudProviderBackoffDuration:   config.CloudProviderBackoffDuration, ShouldOmitCloudProviderBackoff: config.shouldOmitCloudProviderBackoff(), } az.DisksClient = newAzDisksClient(azClientConfig) az.SnapshotsClient = newSnapshotsClient(azClientConfig) az.RoutesClient = newAzRoutesClient(azClientConfig) az.SubnetsClient = newAzSubnetsClient(azClientConfig) az.InterfacesClient = newAzInterfacesClient(azClientConfig) az.RouteTablesClient = newAzRouteTablesClient(azClientConfig) az.LoadBalancerClient = newAzLoadBalancersClient(azClientConfig) az.SecurityGroupsClient = newAzSecurityGroupsClient(azClientConfig) az.StorageAccountClient = newAzStorageAccountClient(azClientConfig) az.VirtualMachinesClient = newAzVirtualMachinesClient(azClientConfig) az.PublicIPAddressesClient = newAzPublicIPAddressesClient(azClientConfig) az.VirtualMachineSizesClient = newAzVirtualMachineSizesClient(azClientConfig) az.VirtualMachineScaleSetsClient = newAzVirtualMachineScaleSetsClient(azClientConfig) az.VirtualMachineScaleSetVMsClient = newAzVirtualMachineScaleSetVMsClient(azClientConfig) az.FileClient = &azureFileClient{env: *env} </ins> if az.MaximumLoadBalancerRuleCount == 0 { az.MaximumLoadBalancerRuleCount = maximumLoadBalancerRuleCount }

Currently there is no way to run without any identity (be it service principal or E/MSI). In reality code running on a worker node can use via - and only via - without the need to access ARM. Here is the proposed change a new mode authN = this mode, is basically telling the cloud provider to work any identity, this means only IMDS calls is allowed. This mode can be detected from if no SPN set and both MSI settings are set to . to use a config map within the cluster. This config map can be loaded on the cluster statically. or created - if not existent - in that then only is used in The net new result of the above is users bootstrapping clusters can configure the worker node to use without any identity on said worker nodes.
/sig azure These SIGs are my best guesses for this issue. Please comment if I am incorrect about one. I am a bot run by
In-tree AzureDisk volume plugin is also depending on Azure credential to fetch lun id. we should figure out how to solve it without credential
fetch disk lun id is also a dependency in kubelet code now, after my investigation, it could be removed on Linux node, while it cannot be removed on Windows node, need to change the k/k upstream code, there is a discussion on going here: , let' wait for the discussion result.
For Linux: 1- use IMDS my understanding is new api versions will have the disk/lun pairs. 2- don't use arm for the return of is the second parameter passed to . For Windows: I don't understand the dep on ARM for this. The above still apply, we just need to find a way to encode the value as ut return from that allow us to find the disk by on a windows machine. might be a good source for this. Ideally speaking. The value should be the same. But i will leave it for you to decide.
could be a good idea since currently it's a LUN num returned by Attach() and then it could be changed to the windows disk num, I will make the code change on v1.15 later, thanks for discussion.
Update: PR would remove the VM API calls when getting the LUN number for AzureDisk.


Question: {utiliptables.TableNAT, kubePostroutingChain, utiliptables.ChainPostrouting, "kubernetes postrouting rules", nil}, } <ins> var iptablesEnsureChains = []struct { table utiliptables.Table chain utiliptables.Chain }{ {utiliptables.TableNAT, KubeMarkDropChain}, } </ins> var iptablesCleanupOnlyChains = []iptablesJumpChain{} // CleanupLeftovers removes all iptables rules and chains created by the Proxier

<!-- Please only use this template for submitting reports about continuously failing tests or jobs in Kubernetes CI --Which jobs are failing: ci-kubernetes-e2e-gci-gce-ipvs Which test(s) are failing: [sig-network] Services should only allow access from service loadbalancer source ranges [Slow] Testgrid link: https://k8s- Reason for failure: Anything else we need to know: The test is failing consistently, however, there is also test that is failing a lot but not always, may it be related?
/sig-network /area ipvs /cc /help wanted
/sig network
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
/assign
/remove-triage unresolved
Hi, , turnout this is because mark drop rule in be deleted, I raied a pr to fix this issue, PTAL. : )
/close as duplicate of
Closing this issue. <detailsIn response to : /close as duplicate of Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/reopen this was affecting ipvs only before , let's keep it open until we verify is not failing anymore
Reopened this issue. <detailsIn response to : /reopen this was affecting ipvs only before , let's keep it open until we verify is not failing anymore Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign Ok I tested this out with kind in ipvs-mode with metallb. In iptables mode I had zero failures but ipvs does not pass at all during the timeout window. This is likely due to


Question: # This file creates a standard build environment for building cross # platform go binary for the architecture kubernetes cares about. <del> FROM golang:1.8.3 </del> <ins> FROM golang:1.9.1 </ins> ENV GOARM 7 ENV KUBE_DYNAMIC_CROSSPLATFORMS

Is this a BUG REPORT or FEATURE REQUEST?: /kind feature This is a followup on , but for Go 1.9 We should do the same kind of testing now to possibly catch bugs in either Go or Kubernetes when doing this kind of dependency version bump. Ideally we'd have automated tests against Go tip (), but we don't have that at the moment, so we have to do it kind of manually. We should also decide which k8s version we should target. k8s v1.8 or v1.9? What Go versions have been used so far? Kubernetes v1.7.0 - go1.8.3 Kubernetes v1.6.0 - go1.7.5 Kubernetes v1.5.0 - go1.7.4 Kubernetes v1.4.0 - go1.6.3 Kubernetes v1.3.0 - go1.6.2 Kubernetes v1.2.0 - go1.4.2 Seems like the trend has been to stay on the same Go version for two releases and then bump. That'd mean we'd switch to Go1.9 in k8s v1.9 Everyone happy with that? Or is there any reason for us to go start using go1.9 in k8s v1.8? In any case, we should indeed start testing go1.9 out now... cc
Let me know if you find any bugs
FWIW pretty tight to code-freeze I would prefer punting out a cycle fwiw.
I don't think we're that close to code freeze given that we have 5 (of 8) weeks left in this coding cycle But I'm happy to defer this to v1.9 if others think that. I guess the main reason for a bump would be scalability goals, but I'm not sure we have the need to expand above 5000 (and that Go would be the bottleneck in that case) for v1.8
made the Go benchmarks 9% faster with 15% larger binaries. I'm interested to see how that affects our numbers.
mid-stack inlining was not included in Go 1.9. It made the compiler too slow. It's being targeted to Go 1.10 now instead, after some compiler optimization work.
Oh, thanks for the correction!


Question: # This file creates a standard build environment for building cross # platform go binary for the architecture kubernetes cares about. <del> FROM golang:1.8.3 </del> <ins> FROM golang:1.9.1 </ins> ENV GOARM 7 ENV KUBE_DYNAMIC_CROSSPLATFORMS

Regardless, we need to run the scalability tests against the RC so we can report issues earlier-- the last upgrade was particularly painful because of an HTTP regression that we caught way too late.
yeah. I filed for that now
yeah, my recollection is that the Go releases usually occur too late in our release cadence and end up getting bumped to the next Kubernetes release. (Also we inevitably find something wrong, though testing RCs would help with that. Though I'm not sure anyone has cycles to test RCs and fix all of the issues that do come up.)
Based on our current time frame, it looks like go1.9 support will have to slip to k8s 1.9. is open to dig into and start things testing, but there's a number of things that have to move.. I'll try to document those this time around to help with future go version bumps. So far: bazel build rules needed to be updated to support go1.9 () bazel version possibly needs to be updated? some of our custom bazel rules broke
When done, we should update this:
/priority important-soon
Updating the dependency is blocked pending bazel 0.6.0 - .
Can we upgrade to 0.6.0 now? Does bazel 0.6.0 have multiarch support btw?
We can't update to depend on bazel 0.6.0+ until we can figure out how to support different versions of the bazelbuild and kubekins-e2e images in PR jobs in prow based on what branch is being tested. cc
Noting that we're working on branching jobs, is testing one option here:
[MILESTONENOTIFIER] Milestone Issue Current <details open<summaryIssue Labels</summary- : Issue will be escalated to these SIGs if needed. : Escalate to the issue owners and SIG owner; move out of milestone after several unsuccessful escalation attempts. : New functionality.


Question: # This file creates a standard build environment for building cross # platform go binary for the architecture kubernetes cares about. <del> FROM golang:1.8.3 </del> <ins> FROM golang:1.9.1 </ins> ENV GOARM 7 ENV KUBE_DYNAMIC_CROSSPLATFORMS

</details<details<summaryHelp</summary<ul<li<a href=""Additional instructions</a</li<li<a href=""Commands for setting labels</a</li</ul</details
I approved this for v1.9, this is something we definitely want in order to keep up with Go (and for improved performance)
we're starting to use the forked images now /cc this should be more feasible very soon Edit: test-infra jobs have also been using bazel 0.6 (now 0.6.1) / a recent rules_go to soak things a bit
We've identified a failing test as a result of the WIP PR to upgrade the codebase to go1.9: The admission webhook test fails with certificate errors under go1.9.1, but runs successfully under go1.8.3. cc: who wanted to be alerted to potential bugs
- please see for a possible fix


What would you like to be : Support dynamically set log level for kube-scheduler, is blocked on node controller approval, but we can move forward with a scheduler only change. /sig scheduling
/help
/assign /remove-help
do you have a PR for this one?
I submitted the PR.


Question: "//pkg/kubelet/types:go_default_library", "//pkg/volume:go_default_library", "//vendor/github.com/golang/glog:go_default_library", <del> "//vendor/github.com/golang/protobuf/proto:go_default_library", </del> "//vendor/github.com/google/cadvisor/fs:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/github.com/google/cadvisor/info/v2:go_default_library",

In , we merged a work around for heapster panic We should get rid of the work around after heapster version is updated. /cc /cc
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten /remove-lifecycle stale
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close


Question: // NewNonInteractiveDeferredLoadingClientConfig creates a ConfigClientClientConfig using the passed context name func NewNonInteractiveDeferredLoadingClientConfig(loader ClientConfigLoader, overrides *ConfigOverrides) ClientConfig { <del> return &DeferredLoadingClientConfig{loader: loader, overrides: overrides, icc: inClusterClientConfig{}} </del> <ins> return &DeferredLoadingClientConfig{loader: loader, overrides: overrides, icc: &inClusterClientConfig{overrides: overrides}} </ins> } // NewInteractiveDeferredLoadingClientConfig creates a ConfigClientClientConfig using the passed context name and the fallback auth reader func NewInteractiveDeferredLoadingClientConfig(loader ClientConfigLoader, overrides *ConfigOverrides, fallbackReader io.Reader) ClientConfig { <del> return &DeferredLoadingClientConfig{loader: loader, overrides: overrides, icc: inClusterClientConfig{}, fallbackReader: fallbackReader} </del> <ins> return &DeferredLoadingClientConfig{loader: loader, overrides: overrides, icc: &inClusterClientConfig{overrides: overrides}, fallbackReader: fallbackReader} </ins> } func (config *DeferredLoadingClientConfig) createClientConfig() (ClientConfig, error) {

Found in openshift, but I'm pretty sure the codepaths are the same. That means the token is getting ignored and in-cluster config is being used in preference to flags.
Is a similar issue?
Yep, kind of regression.
Sigh.
Yes, most likely the same cause.
I'm on it.
Unit test :)
Probably will want to cherrypick this into 1.5.
WIP in


Question: } func (sched *Scheduler) addPodToSchedulingQueue(obj interface{}) { <del> if err := sched.SchedulingQueue.Add(obj.(*v1.Pod)); err != nil { </del> <ins> pod := obj.(*v1.Pod) klog.V(3).Infof("add event for unscheduled pod %s/%s", pod.Namespace, pod.Name) if err := sched.SchedulingQueue.Add(pod); err != nil { </ins> utilruntime.HandleError(fmt.Errorf("unable to queue %T: %v", obj, err)) } }

This helps debugging scheduler's behavior, currently we don't have logs that indicate when the scheduler first saw a pod or a node. The log level should be v=3. /sig scheduling /cc
Do you mean add logging for callbacks in ?
yes, at least for add and delete events
Sure, I can also give this pr, before tomorrow
/assign
We actually have one v=2 log line for a Node being to the cache's tree: But it's probably good to keep this one.


Question: ``` Just call `make` to regenerate the diagrams. <ins> ## Building with Docker If you are on a Mac or your pip install is messed up, you can easily build with docker. ``` make docker ``` The first run will be slow but things should be fast after that. To clean up the docker containers that are created (and other cruft that is left around) you can run `make docker-clean`. If you are using boot2docker and get warnings about clock skew (or if things aren't building for some reason) then you can fix that up with `make fix-clock-skew`. ## Automatically rebuild on file changes If you have the fswatch utility installed, you can have it monitor the file system and automatically rebuild when files have changed.  Just do a `make watch`. </ins>  No newline at end of file

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug /priority failing-test /area test-infra What happened: job is failing. See https://k8s- https://k8s- https://k8s- https://k8s- What you expected to happen: test should pass
There are no sig labels on this issue. Please by: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. You can find the group list and label list . The in the method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals
/assign
Reiterating the mentions to trigger a notification: <detailsIn response to : /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
I'm guessing that something got messed up in the bazel cache when reverting from 0.5.2 to 0.4.5. I'm not sure how best to fix.
How about updating everything to bazel 0.5.3 at once? I know you're trying to do a step-by-step approach to avoid breakages, but I suspect it will be cheaper to just deal with a brief outage while we update? (aka update the bazelbuild image to 0.5.3 and then merge your PR with updated and working BUILD rules?)
there's at least one other issue I've discovered with the latest go rules - the binary no longer works, due to changes introduced in that might be arguably better (at least everything still builds?), though it's frustrating that it makes things more broken in a different way.


Question: "testing" "github.com/prometheus/common/model" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema"


Question: // Nodeports need SNAT, unless they're local. // ipset call <del> var nodePortSet *IPSet </del> <ins> var ( nodePortSet *IPSet entries     []*utilipset.Entry ) </ins> switch protocol { case "tcp": nodePortSet = proxier.ipsetList[kubeNodePortSetTCP] <del> entry = &utilipset.Entry{ </del> <ins> entries = []*utilipset.Entry{{ </ins> // No need to provide ip info Port:     svcInfo.NodePort(), Protocol: protocol, SetType:  utilipset.BitmapPort, <del> } </del> <ins> }} </ins> case "udp": nodePortSet = proxier.ipsetList[kubeNodePortSetUDP] <del> entry = &utilipset.Entry{ </del> <ins> entries = []*utilipset.Entry{{ </ins> // No need to provide ip info Port:     svcInfo.NodePort(), Protocol: protocol, SetType:  utilipset.BitmapPort, <del> } </del> <ins> }} </ins> case "sctp": nodePortSet = proxier.ipsetList[kubeNodePortSetSCTP] <del> entry = &utilipset.Entry{ IP:       proxier.nodeIP.String(), Port:     svcInfo.NodePort(), Protocol: protocol, SetType:  utilipset.HashIPPort, </del> <ins> // Since hash ip:port is used for SCTP, all the nodeIPs to be used in the SCTP ipset entries. entries = []*utilipset.Entry{} for _, nodeIP := range nodeIPs { entries = append(entries, &utilipset.Entry{ IP:       nodeIP.String(), Port:     svcInfo.NodePort(), Protocol: protocol, SetType:  utilipset.HashIPPort, }) </ins> } default: // It should never hit klog.Errorf("Unsupported protocol type: %s", protocol) } if nodePortSet != nil { <del> if valid := nodePortSet.validateEntry(entry); !valid { klog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, nodePortSet.Name)) </del> <ins> entryInvalidErr := false for _, entry := range entries { if valid := nodePortSet.validateEntry(entry); !valid { klog.Errorf("%s", fmt.Sprintf(EntryInvalidErr, entry, nodePortSet.Name)) entryInvalidErr = true break } nodePortSet.activeEntries.Insert(entry.String()) } if entryInvalidErr { </ins> continue } <del> nodePortSet.activeEntries.Insert(entry.String()) </del> } // Add externaltrafficpolicy=local type nodeport entry

What happened: SCTP nodeport service is not working for all IPs present in the node when ipvs is enabled. It is working only for node's InternalIP. What you expected to happen: SCTP nodeport service must work for all ips present in the node when ipvs is enabled. How to reproduce it (as minimally and precisely as possible): Create a kubernetes cluster with at least one node having multiple interfaces with different IPs. Start an SCTP server deployment. Then create a nodeport service for the same. Now try to reach the service using all the IPs present in that node. The service will be accessible only via node's InternalIP not any other IPs. Environment: Kubernetes version: v1.15.2 OS: ubuntu 18.04 Kernel: 4.15.0-54-generic Network plugin and version (if this is a network-related bug): ipset version v6.34
/sig network /area kube-proxy /area ipvs
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
/remove-triage unresolved


Question: package apimachinery import ( <ins> "encoding/json" </ins> "fmt" "strings" "sync"

This issue proposes the writing of a new test for the API endpoint with the intention of promoting it to Conformance. This issue contains a very masic mock test in order to start the conversation about what a good e2e test would look like for this endpoint. The code below is verified to be hitting the intended endpoint, show as per the queries in APIsnoop's live view of the cluster's audit logs. According to this query, there is only one API endpoint which is for Namespace that isn't being covered. We selected patchCoreV1Namespace to write a test for. Dependencies: The mock test: If a test with these calls is build and merged, conformance will go up by 1 endpoint.
/sig testing /sig architecture /area conformance


Question: glog.V(4).Infof("Deleting %q", castObj.Name) c.enqueue(castObj) } <del> // there aren't very many apiservices, just check them all. func (c *APIServiceRegistrationController) getAPIServicesFor(service *v1.Service) []*apiregistration.APIService { var ret []*apiregistration.APIService apiServiceList, _ := c.apiServiceLister.List(labels.Everything()) for _, apiService := range apiServiceList { if apiService.Spec.Service == nil { continue } if apiService.Spec.Service.Namespace == service.Namespace && apiService.Spec.Service.Name == service.Name { ret = append(ret, apiService) } } return ret } // TODO, think of a way to avoid checking on every service manipulation func (c *APIServiceRegistrationController) addService(obj interface{}) { for _, apiService := range c.getAPIServicesFor(obj.(*v1.Service)) { c.enqueue(apiService) } } func (c *APIServiceRegistrationController) updateService(obj, _ interface{}) { for _, apiService := range c.getAPIServicesFor(obj.(*v1.Service)) { c.enqueue(apiService) } } func (c *APIServiceRegistrationController) deleteService(obj interface{}) { castObj, ok := obj.(*v1.Service) if !ok { tombstone, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { glog.Errorf("Couldn't get object from tombstone %#v", obj) return } castObj, ok = tombstone.Obj.(*v1.Service) if !ok { glog.Errorf("Tombstone contained object that is not expected %#v", obj) return } } for _, apiService := range c.getAPIServicesFor(castObj) { c.enqueue(apiService) } } </del>

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: Currently APIServiceRegistrationController watch service change and find the related APIServices related if any. Then it AddAPIService/RemoveAPIService for APIAggregator. I dig into the kube-aggregator and find it no need totally, because update APIServices status when related service or endpoint change. So if does not watch services, it will not miss any event for APIServices. What you expected to happen: How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others:
/cc
/kind enhancement /sig api-machinery
/assign
that does not follow. AvailableConditionController modifies the APIService object. APIServiceRegistrationController must still watch the APIService objects to observe those changes.
oh, I misread the description. You're saying APIServiceRegistrationController doesn't need to watch v1.Service objects


Question: // GetNewReplicaSet returns a replica set that matches the intent of the given deployment; get ReplicaSetList from client interface. // Returns nil if the new replica set doesn't exist yet. func GetNewReplicaSet(deployment *extensions.Deployment, c clientset.Interface) (*extensions.ReplicaSet, error) { <del> rsList, err := ListReplicaSets(deployment, rsListFromClient(c)) </del> <ins> rsList, err := ListReplicaSets(deployment, RsListFromClient(c)) </ins> if err != nil { return nil, err } return FindNewReplicaSet(deployment, rsList) } <del> // rsListFromClient returns an rsListFunc that wraps the given client. func rsListFromClient(c clientset.Interface) rsListFunc { </del> <ins> // RsListFromClient returns an rsListFunc that wraps the given client. func RsListFromClient(c clientset.Interface) RsListFunc { </ins> return func(namespace string, options metav1.ListOptions) ([]*extensions.ReplicaSet, error) { rsList, err := c.Extensions().ReplicaSets(namespace).List(options) if err != nil {

IMO it feels like a bad practice to have mutating resource handlers. The deletePod handler is used for queueing a Recreate deployment once it has no pods running. It should merely look at the ReplicaSets that have owner refs as opposed to using dc.getReplicaSetsForDeployment. The orphaning/adoption dance is already happening inside the reconcilation loop.
I'd like to work on this
feel free to grab it. I can't assign the issue to you so I'll add myself; ping me on the PR when you have one.


Question: <del> apiVersion: extensions/v1beta1 </del> <ins> apiVersion: networking.k8s.io/v1beta1 </ins> kind: Ingress metadata: name: hostname spec: backend: serviceName: hostname <del> servicePort: 80 </del>  No newline at end of file <ins> servicePort: 80 </ins>

<!-- Please only use this template for submitting reports about continuously failing tests or jobs in Kubernetes CI --Which jobs are failing: Which test(s) are failing: Since when has it been failing: Testgrid link: https://k8s- Reason for failure: Anything else we need to know: /cc /milestone v1.19 /priority important-soon /sig network
Ref:
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
my fu points me to this commit [] Fix multiple conversion tests that belongs to this PR , that seems related can you confirm if that PR is causing this test to fail? (should we gate on this test?)
Yes, that PR is related to the failure. The test utility is trying to convert extensions/v1beta1 data read from various files directly to a go type. This was never a valid thing to do, and happened to silently succeed because the two structs had an identical structure. This is exactly the type of bug was designed to flush out
the correct fix is to update the test/e2e/testing- files to
/assign
It was my PR that broke this - I can also fix that.
Thanks, o it was the commit that was good but the test that was wrong :upsidedownface: , At least I got right the commit :sweat_smile:
is out for review (talked with offline, he was aware that I'll send that out)


Question: for k, v := range pod.Annotations { copyPod.Annotations[k] = v } <del> copyPod.Annotations[kubetypes.ConfigMirrorAnnotationKey] = kubetypes.MirrorType </del> <ins> copyPod.Annotations[kubetypes.ConfigMirrorAnnotationKey] = getPodHash(pod) </ins> _, err := mc.apiserverClient.Pods(copyPod.Namespace).Create(&copyPod) return err

See for more context. Currently, kubelet compares the spec of a mirror pod (created via apiserver) with that of a static pod (config stored as a file) to decide whether the two are considered identical. If they differ, kubelet would delete the mirror pod and re-create it with the correct spec. Due to the fact that apiserver may set default values for some fields in the spec of the mirror pod, the specs could deviate undesirably. To avoid problems like this, we can store the config of a static pod as an annotation in the mirror pod, and compare the static pod with the annotation.
Would a hash of the config be sufficient if we're only going to compare? The whole spec would generally be more useful, but I wonder if that is too much data to put in an annotation.
This seems totally reasonable to me, but I'm still desperately curious why you can't just bypass defaulting when you generate mirror pod from static pod? (i.e. just use the static pod's configuration verbatim)
It's not kubelet who applies the default values here. The admission plugin was only run at the apiserver, and to bypass that, you have to ask apiserver to treat mirror pods differently and potentially at multiple places.
On May 22, 2015 10:38 AM, "Yu-Ju Hong" wrote: Understood, but is it really multiple places? If so then I agree it's a bad idea. Otherwise it might be worth considering as an alternative.
It's untenable for Kubelet to assume to know/circumvent all fields that will be set by other entities: admission control, defaulting, late initialization, auto-something, whatever. It needs to obey the same rules as other clients.
I don't understand why you would ever want to apply any of the things you mentioned to static pods. Static pods should be copied verbatim into the mirror pods, right? That's what I'm proposing.
What if the master is updated before the kubelet, and the master adds a new API field with a default value?
That case should be discussed in
I agree that mirror pod should reflect the true status of the static pod running on the node, and no default values should be set in this case.


Question: for k, v := range pod.Annotations { copyPod.Annotations[k] = v } <del> copyPod.Annotations[kubetypes.ConfigMirrorAnnotationKey] = kubetypes.MirrorType </del> <ins> copyPod.Annotations[kubetypes.ConfigMirrorAnnotationKey] = getPodHash(pod) </ins> _, err := mc.apiserverClient.Pods(copyPod.Namespace).Create(&copyPod) return err

However, I think it'd be harder to implement this way since, as stated above, that there are many things to circumvent in the apiserver itself. It could also be harder to keep this special case from being broken by a yet another new feature/plugin. Keeping the hash of the config in the annotation seems like a good compromise even though it may not reflect the true status 100% accurately. I think this is acceptable. What do you think?
Sure, if you think the annotation approach is easier, I would go with that. I guess in the case Brian mentoined (master adds a new API field with a default value), you would not update the mirror pod, because the static pod definition has not changed?
Yes, a hash would be sufficient.
It shouldn't be necessary to update the mirror pod in the case that a new default field was updated, because that field should be populated automatically by the apiserver on decode from etcd.
We should fix this so we don't have this problem again in 1.2.


Question: node, victims, nominatedPodsToClear, err := sched.config.Algorithm.Preempt(preemptor, sched.config.NodeLister, scheduleErr) if err != nil { <del> klog.Errorf("Error preempting victims to make room for %v/%v.", preemptor.Namespace, preemptor.Name) </del> <ins> klog.Errorf("Error preempting victims to make room for %v/%v: %v", preemptor.Namespace, preemptor.Name, err) </ins> return "", err } var nodeName = ""

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Default scheduler crash if scheduler extender filter returns a not found node What you expected to happen: Default scheduler just should consider the not found node as not an candidate, instead of crash itself (panic). How to reproduce it (as minimally and precisely as possible): cluster only have 1 node: 0.0.0.1 scheduler extender (with nodeCacheCapable enabled) filter accidentally returns a candidate node: 0.0.0.2 scheduler recieved the candidate nodes: 0.0.0.2 from scheduler extender, but got below error: The is SuggestedHost: filteredNodes[0].Name: Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig scheduling


Question: }) } <ins> // PodSecurityPolicyDescriber generates information about a PodSecurityPolicy. type PodSecurityPolicyDescriber struct { clientset.Interface } func (d *PodSecurityPolicyDescriber) Describe(namespace, name string, describerSettings printers.DescriberSettings) (string, error) { psp, err := d.Extensions().PodSecurityPolicies().Get(name, metav1.GetOptions{}) if err != nil { return "", err } return describePodSecurityPolicy(psp) } func describePodSecurityPolicy(psp *extensions.PodSecurityPolicy) (string, error) { return tabbedString(func(out io.Writer) error { w := NewPrefixWriter(out) w.Write(LEVEL_0, "Name:t%sn", psp.Name) w.Write(LEVEL_0, "nSettings:n") w.Write(LEVEL_1, "Allow Privileged:t%tn", psp.Spec.Privileged) w.Write(LEVEL_1, "Default Add Capabilities:t%vn", capsToString(psp.Spec.DefaultAddCapabilities)) w.Write(LEVEL_1, "Required Drop Capabilities:t%sn", capsToString(psp.Spec.RequiredDropCapabilities)) w.Write(LEVEL_1, "Allowed Capabilities:t%sn", capsToString(psp.Spec.AllowedCapabilities)) w.Write(LEVEL_1, "Allowed Volume Types:t%sn", fsTypeToString(psp.Spec.Volumes)) w.Write(LEVEL_1, "Allow Host Network:t%tn", psp.Spec.HostNetwork) w.Write(LEVEL_1, "Allow Host Ports:t%sn", hostPortRangeToString(psp.Spec.HostPorts)) w.Write(LEVEL_1, "Allow Host PID:t%tn", psp.Spec.HostPID) w.Write(LEVEL_1, "Allow Host IPC:t%tn", psp.Spec.HostIPC) w.Write(LEVEL_1, "Read Only Root Filesystem:t%vn", psp.Spec.ReadOnlyRootFilesystem) w.Write(LEVEL_1, "SELinux Context Strategy: %stn", string(psp.Spec.SELinux.Rule)) var user, role, seLinuxType, level string if psp.Spec.SELinux.SELinuxOptions != nil { user = psp.Spec.SELinux.SELinuxOptions.User role = psp.Spec.SELinux.SELinuxOptions.Role seLinuxType = psp.Spec.SELinux.SELinuxOptions.Type level = psp.Spec.SELinux.SELinuxOptions.Level } w.Write(LEVEL_2, "User:t%sn", stringOrNone(user)) w.Write(LEVEL_2, "Role:t%sn", stringOrNone(role)) w.Write(LEVEL_2, "Type:t%sn", stringOrNone(seLinuxType)) w.Write(LEVEL_2, "Level:t%sn", stringOrNone(level)) w.Write(LEVEL_1, "Run As User Strategy: %stn", string(psp.Spec.RunAsUser.Rule)) w.Write(LEVEL_2, "Ranges:t%sn", userIDRangeToString(psp.Spec.RunAsUser.Ranges)) w.Write(LEVEL_1, "FSGroup Strategy: %stn", string(psp.Spec.FSGroup.Rule)) w.Write(LEVEL_2, "Ranges:t%sn", groupIDRangeToString(psp.Spec.FSGroup.Ranges)) w.Write(LEVEL_1, "Supplemental Groups Strategy: %stn", string(psp.Spec.SupplementalGroups.Rule)) w.Write(LEVEL_2, "Ranges:t%sn", groupIDRangeToString(psp.Spec.SupplementalGroups.Ranges)) return nil }) } func stringOrAll(s string) string { if len(s) > 0 { return s } return "*" } func stringOrNone(s string) string { if len(s) > 0 { return s } return "<none>" } func fsTypeToString(volumes []extensions.FSType) string { strVolumes := []string{} for _, v := range volumes { strVolumes = append(strVolumes, string(v)) } return stringOrNone(strings.Join(strVolumes, ",")) } func hostPortRangeToString(ranges []extensions.HostPortRange) string { formattedString := "" if ranges != nil { strRanges := []string{} for _, r := range ranges { strRanges = append(strRanges, fmt.Sprintf("%d-%d", r.Min, r.Max)) } formattedString = strings.Join(strRanges, ",") } return stringOrNone(formattedString) } func userIDRangeToString(ranges []extensions.UserIDRange) string { formattedString := "" if ranges != nil { strRanges := []string{} for _, r := range ranges { strRanges = append(strRanges, fmt.Sprintf("%d-%d", r.Min, r.Max)) } formattedString = strings.Join(strRanges, ",") } return stringOrNone(formattedString) } func groupIDRangeToString(ranges []extensions.GroupIDRange) string { formattedString := "" if ranges != nil { strRanges := []string{} for _, r := range ranges { strRanges = append(strRanges, fmt.Sprintf("%d-%d", r.Min, r.Max)) } formattedString = strings.Join(strRanges, ",") } return stringOrNone(formattedString) } func capsToString(caps []api.Capability) string { formattedString := "" if caps != nil { strCaps := []string{} for _, c := range caps { strCaps = append(strCaps, string(c)) } formattedString = strings.Join(strCaps, ",") } return stringOrNone(formattedString) } </ins> // newErrNoDescriber creates a new ErrNoDescriber with the names of the provided types. func newErrNoDescriber(types ...reflect.Type) error { names := make([]string, 0, len(types))

Is this a request for help? (If yes, you should use our troubleshooting guide and community support channels, see ): No What keywords did you search in Kubernetes issues before filing this one? (If you have found any duplicates, you should instead reply there.): describe psp Is this a BUG REPORT or FEATURE REQUEST? (choose one): FEATURE REQUEST Kubernetes version (use ): Environment: Doesn't matter. What happened: doesn't show PSP's fields: What you expected to happen: All PSP fields should be shown. How to reproduce it (as minimally and precisely as possible):
If you're agree with that than I (or :) ) could implement it.
yes, improve the describer. Look at the scc describer for formatting.
any ideas which SIG should own PSP?
In this case it should be sig-cli /sig cli
yeah, this is cli related. I'd go with sig-auth for actual behavior.


Question: "description": "ServicePort contains information on service's port.", "properties": { "appProtocol": { <del> "description": "The application protocol for this port. This field follows standard Kubernetes label syntax. Un-prefixed names are reserved for IANA standard service names (as per RFC-6335 and http://www.iana.org/assignments/service-names). Non-standard protocols should use prefixed names such as mycompany.com/my-custom-protocol. This is a beta field that is guarded by the ServiceAppProtocol feature gate and enabled by default.", </del> <ins> "description": "The application protocol for this port. This field follows standard Kubernetes label syntax. Un-prefixed names are reserved for IANA standard service names (as per RFC-6335 and https://www.iana.org/assignments/service-names). Non-standard protocols should use prefixed names such as mycompany.com/my-custom-protocol. This is a beta field that is guarded by the ServiceAppProtocol feature gate and enabled by default.", </ins> "type": "string" }, "name": {

We should make the change in PR API reference links to should use HTTPS, not HTTP Visit h and look at the IANA hyperlinks Once this fix is made, a PR to is needed to update the docs (by autogenerating them). This typically would be deferred until the next Kubernetes release. v1.23 not applicable not applicable not applicable not applicable not applicable
/sig docs
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
I opened this issue so that someone else can open a new PR that is equivalent to .
/retitle Update links to IANA for Service Name and Transport Protocol Port Number Registry
I would like to work on this issue. May I assign it to myself?
Anyone - including you - is welcome to work on this issue! So long as you've you can open a pull request to propose a fix.
Thanks
/assign
I have made the PR. I hope this works. If not then will be happy to do the necessary changes.
Had to close the previous PR because of issues, here is the new one I made.


Question: // Read retrieves data with the given "key" from CycleState. If the key is not // present an error is returned. <del> // This function is not thread safe. In multi-threaded code, lock should be // acquired first. </del> <ins> // This function is thread safe by acquiring an internal lock first. </ins> func (c *CycleState) Read(key StateKey) (StateData, error) { <ins> c.mx.RLock() defer c.mx.RUnlock() </ins> if v, ok := c.storage[key]; ok { return v, nil }

What happened: kube-scheduler () crashes due to a panic while running : The race condition is triggered when an out-of-tree plugin, that writes to during its Filter phase, is registered through . Access to in general happens from a single goroutine and there are no practical concurrency issues, with the exception of the phase, that parallelizes calls for each node across plugins: A quick glance at the tip of the branch seems to indicate the bug is still present there. What you expected to happen: Ideally access to should be synchronized everywhere for correctness, including by all in-tree plugins, since there are no guarantees about how will call multiple plugins concurrently. It seems to me it is something that was considered at some point, given that has methods. Sadly, almost no in-tree plugins are synchronizing access. For now, I will try to work around the problem by doing a single write to during and storing a data bag structure where I will write state into during (with proper synchronization), instead of writing to directly. How to reproduce it (as minimally and precisely as possible): Install a scheduler plugin that writes to during its phase. In-tree (builtin) plugins are not synchronizing their access to . Environment: Kubernetes version (use ): v1.18.8 Cloud provider or hardware configuration: linux-amd64 (AWS) OS (e.g: ): Ubuntu 18.04.5 LTS (Bionic Beaver) Others: custom kube-scheduler binary built with an out-of-tree plugin using
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig scheduling
AFAICT all in-tree plugins write to the cycle state in PreFilter and then only read from cycle state in Filter. So this shouldn't be a problem with in-tree plugins as concurrent read access should be fine during RunFilterPlugins. But since your out-of-tree plugin writes in Filter, I can see why this can be a problem.


Question: // Read retrieves data with the given "key" from CycleState. If the key is not // present an error is returned. <del> // This function is not thread safe. In multi-threaded code, lock should be // acquired first. </del> <ins> // This function is thread safe by acquiring an internal lock first. </ins> func (c *CycleState) Read(key StateKey) (StateData, error) { <ins> c.mx.RLock() defer c.mx.RUnlock() </ins> if v, ok := c.storage[key]; ok { return v, nil }

Aside from adding Lock/Unlock in in-tree plugin, offering mutual exclusion on each key as a built-in feature within CycleState could be an alternative solution. This way, reads from in-tree plugins during Filter wouldn't interfere with writes during Filter from out-of-tree plugins.
cc
should we change the Read/Write CycleState methods to be thread safe?
Adding locks without a proper justification would just increase the latency. could you clarify your use case? Why do you need to write anything during Filter?
yeah, we should measure it, but I don't expect it to have significant effect...
Adding locks without a proper justification would just increase the latency. Without contention (which is the case with in-tree plugins), I don't think this is a concern.
Adapting source from : source: essentially, 11.3 ns/Lock-Unlock pair
I remember locks showing up in profiling for PreScore/Score for certain plugins. Although I have to admit that the accesses were not properly implemented. I'd like to hear an answer for before doing any experiments.
+1 to that I'd also like to understand the motivations / requirements of mutating CycleState in Filter().
My particular use-case is that we have resources being dynamically to/removed from Nodes, and those resources need to be allocated (and tracked) in the scheduling loop. Technically I could pre-compute all the available dynamic resources for all Nodes at the phase, and snapshot it, but that would require me to pre-compute everything for all Nodes on all scheduling cycles. Instead, what I'm currently doing is computing what dynamic resources can be utilized on-demand during the phase (kind of a lazy loading of dynamic resources for each pair), and saving state of what extra resources were considered during for each Node that was evaluated. Then, later in the scheduler plugin can lock-in the exact dynamic resources for the particular Node that was selected. I understand the concern with adding extra synchronization overhead without a real need. Even if my use-case is not good enough to justify it, there is another argument for it: technically, the lack of access synchronization on violates the . Even if in the end we decide synchronizing all access is not worth the overhead (for correctness), I would strongly encourage better documentation saying which plugin phases are allowed to modify , and perhaps even forcing a snapshot-read-only copy of to be used by plugins.
Using the scheduler to track resources is not a good idea.


Question: // Read retrieves data with the given "key" from CycleState. If the key is not // present an error is returned. <del> // This function is not thread safe. In multi-threaded code, lock should be // acquired first. </del> <ins> // This function is thread safe by acquiring an internal lock first. </ins> func (c *CycleState) Read(key StateKey) (StateData, error) { <ins> c.mx.RLock() defer c.mx.RUnlock() </ins> if v, ok := c.storage[key]; ok { return v, nil }

The scheduler should be stateless. Can't you track the resources with an external controller or with a device plugin and making it part of the Node spec?
there may be some misunderstanding in my poor choice of words. What I mean by "tracking" is that the scheduler needs to track usage and availability of these extra per-Node resources, to be able to allocate them during placement/scheduling. Exactly the same it does with volumes, cpus, memory, ephemeral disk, etc Doing computations in a controller outside of the scheduling loop would be similar to "precomputing" these resources. In my particular use case I chose a different way because: introduces another point of failure, if the external controller stops pre-computing extra resources and publishing them somewhere (CRD, Node spec, etc) scheduling halts (at least scheduling on these extra resources) Node resources in my use case are volatile (they are coming and going frequently), and the extra latency by an extra controller (vs doing all computations right in the scheduling loop) may be problematic for my use case. Even if you consider my use case not enough to add proper synchronization to , I'd still encourage making it explicit what scheduling phases are allowed to modify it (and even passing in a snapshot read-only version of to phases that are not expected to update it), to avoid confusion and race conditions.
Ok, if you are not tracking the existence of resources in the scheduler, I wouldn't suggest the external controller. Still, have you considered the device plugin framework? That way, the default kube-scheduler could keep track of resources usage. Otherwise, I still don't see why you would need to update the CycleState during Filter. As its name indicates, CycleState only lives during a scheduling cycle.
Regardless of the correctness of usage of CycleState to do what wants, I still think we need to do at least one of the following: that writing to CycleState during Filter will lead to data races, a read-only CycleState to Filter plugins (whether we do this at the interface level or if we simply drop Write requests during Filter is an implementation detail), or make CycleState concurrency safe CycleState as it exists right now allows plugin authors to shoot themselves in the foot IMO.
I'm leaning towards 1, but it's fair to continue the discussion before deciding.
Can you run the "controller" as a goroutine along with your scheduler plugin?


Question: // Read retrieves data with the given "key" from CycleState. If the key is not // present an error is returned. <del> // This function is not thread safe. In multi-threaded code, lock should be // acquired first. </del> <ins> // This function is thread safe by acquiring an internal lock first. </ins> func (c *CycleState) Read(key StateKey) (StateData, error) { <ins> c.mx.RLock() defer c.mx.RUnlock() </ins> if v, ok := c.storage[key]; ok { return v, nil }

So it's not an external controller binary, and hence don't introduce another point of failure. It's pretty like how the default scheduler manages its internal cache. But note that cache is only needed when you need a more efficient data structure. If a {key:value} map is good enough, spinning up a XYZInformer and obtain the latest XYZ resource on-demand is the most recommended way. I think it quite depends on how you design the internal XYZ cache. For example, it should only tweak the cache in an incremental manner - in other words, unchanged resource will be kept unmodified. You can check out the "doubly-pointer snapshot" of scheduler's cache. With above said, if I understand your case correctly, you may need to build a sophisticated XYZ cache, which runs in a separate goroutine and responds to your XYZ resource changes. And then upon each scheduling cycle, take a snapshot (may needs a specific snapshotting algorithm) in PreFilter() and WRITE to CycleState, and then in Filter() READ from CycleState. My personal option is to suggest all plugin developers to read from CycleState in Filter().
nailed it. What you describe is almost exactly what I’m doing today. There are some details particular to my use case (e.g. just tracking current state with an informer and an incremental cache is not enough, because there’s some logic involved in deciding if currently available resources are usable by each pair, and I currently do that computation in as pairs are evaluated). In any case, I just wanted to share my use case to provide some color, but I don’t want what I’m doing distracting us from the more important questions, mentioned by above. My personal preference would be as a very short term next step, then when someone has cycles to do it. I would be OK with punting on until a more strong need shows up.
Note that limiting to only reading from works for me, as long as I can still read my own data structure from it and do writes on it (i.e. don’t deepclone for the phase).
also adds overhead (from copying the state, which can be very expensive).
Could implement just by changing the interface passed to (e.g. ). That wouldn’t be backwards compatible, but very cheap.


Question: // Read retrieves data with the given "key" from CycleState. If the key is not // present an error is returned. <del> // This function is not thread safe. In multi-threaded code, lock should be // acquired first. </del> <ins> // This function is thread safe by acquiring an internal lock first. </ins> func (c *CycleState) Read(key StateKey) (StateData, error) { <ins> c.mx.RLock() defer c.mx.RUnlock() </ins> if v, ok := c.storage[key]; ok { return v, nil }

I guess that would work in the sense that you cannot change the high level values of the CycleState map, but you could still override what's inside them, if they have pointers. But maybe that's ok.
but you could still override what's inside them, if they have pointers. But maybe that's ok. yeah, if it was pointers and if the underlying data needed to be modified, I'd imagine it would have its own Lock/Unlock semantics
for such implementation, the only remaining concern is backwards compatibility. So I would leave that decision to who leads the scheduling-plugins repo.
I rethink 's original problem a little bit: the original racing issue occurs when reading CycleState in in-tree plugins, this indicates one of the (out-of-tree) plugins is (over)writing the most high-level value of the CycleState map. But if we control the locking inside the out-of-tree plugin, and mutate that particular in-place, that would be good enough? Case 3 in the above program demonstrates the correct way to manipulate the specific out-of-tree state. Let me know your thoughts. If it resolves 's concern, we don't need to make any change on the Filter() interface.
Case 3 is how I am working around the problem today to avoid the race. I write a "data bag" once into top level in , then only modify my own inner structure (with proper locking) in , without doing any further modifications to directly (top level) in . That is acceptable, but I'd encourage you to consider making the interface (and documentation) more explicit for plugins, so they avoid the mistake of writing directly into . Without changing anything (no documentation, no readonly interface), it is pretty easy for a plugin to use in a way that you don't want to support.
Thanks for confirming that. Yes, documenting it, or(and) publishing examples would be quite helpful to clarify the semantics. As a wrap-up: a keyed value can be read from CycleState, literally , in any phase. It's always race-free.


Question: // Read retrieves data with the given "key" from CycleState. If the key is not // present an error is returned. <del> // This function is not thread safe. In multi-threaded code, lock should be // acquired first. </del> <ins> // This function is thread safe by acquiring an internal lock first. </ins> func (c *CycleState) Read(key StateKey) (StateData, error) { <ins> c.mx.RLock() defer c.mx.RUnlock() </ins> if v, ok := c.storage[key]; ok { return v, nil }

if you want to mutate the top-level keyed value in CycleState, if it's a phase that CycleState is accessed consecutively, such as PreFilter, PreScore, it's safe to mutate CycleState in any manner if it's a phase that CycleState is accessed parallelly, to be race-free, (for now) you only have 1 option: keep the keyed value's pointer unchanged, and use an internal lock to mutate that pointer obj [TBD] we may come up with changes to add read locks to all in-tree plugins when reading values from CycleState, so that the out-of-tree plugin can surround its mutating logic between and - th mutating logic thereby can be any style Any thoughts on the TBD item? I would assume read locks wouldn't cost too much.
For now, I think the documentation is correct, it clearly says Read() and Write() are not thread-safe: The key thing here is whether in-tree plugin should use RLock/RUnlock or not. If yes, out-of-tree plugins can be freed to use Lock/Unlock. (see the TBD item in the previous comment)
I like the TBD, and optimizing for correctness in how in-tree plugins are accessing . I understand the concern with synchronization overhead, but I would be really surprised if it is the bottleneck in scheduling code.
It can be, when you have 5k-nodes clusters, or bigger.
I don't think it will be expensive, a rlock should be equivalent to an atomic add operation (basically it increments the number of readers). As a comparison, looking up the state from the map is likely to be more expensive.
Of course we are assuming no contention in the default plugins.
This is my impression as well. A vanilla version of scheduler doesn't impose write lock in Filter() and Score(), so should be good.
We can always see with the benchmarks and revert if its not negligible.
I will run the benchmarks to see if there is perf downgrade. /assign
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle stale
/remove-lifecycle stale This is still relevant, yes?


Question: // Read retrieves data with the given "key" from CycleState. If the key is not // present an error is returned. <del> // This function is not thread safe. In multi-threaded code, lock should be // acquired first. </del> <ins> // This function is thread safe by acquiring an internal lock first. </ins> func (c *CycleState) Read(key StateKey) (StateData, error) { <ins> c.mx.RLock() defer c.mx.RUnlock() </ins> if v, ok := c.storage[key]; ok { return v, nil }

IIUC the volumebinding plugin has been fixed, but external plugins can still shoot themselves in the foot.
/priority backlog
Yes. I'm working on this, late this week I will send out a PR along with perf test result.


Question: // returned. func (adapter *EndpointsAdapter) Create(namespace string, endpoints *corev1.Endpoints) (*corev1.Endpoints, error) { endpoints, err := adapter.endpointClient.Endpoints(namespace).Create(endpoints) <del> if err == nil && adapter.endpointSliceClient != nil { _, err = adapter.ensureEndpointSliceFromEndpoints(namespace, endpoints) </del> <ins> if err == nil { err = adapter.EnsureEndpointSliceFromEndpoints(namespace, endpoints) </ins> } return endpoints, err }

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: EndpointSlice for masters was not created after enabling EndpointSlice on a pre-existing cluster. Only those EndpointSlices managed by kube-controller-manager were created. What you expected to happen: EndpointSlice for masters should be created after enabling EndpointSlice on a pre-existing cluster, the same as the EndpointSlices managed by kube-controller-manager. How to reproduce it (as minimally and precisely as possible): kubeadm to create a cluster: the EndpointSlice feature by editing the kube-apiserver, kube-controller-manager, kube-proxy manifest kube-apiserver and kube-controller-manager is restarted, the EndpointSlice in namespace will be missing. Anything else we need to know?: Environment: Kubernetes version (use ):Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools:Network plugin and version (if this is a network-related bug): Others:
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
/remove-triage unresolved /assign