Question: <ins> /* Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package network import ( "fmt" "strings" "time" "github.com/onsi/ginkgo" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eservice "k8s.io/kubernetes/test/e2e/framework/service" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" ) const ( serviceName = "svc-udp" podClient   = "pod-client" podBackend1 = "pod-server-1" podBackend2 = "pod-server-2" srcPort     = 12345 ) // Linux NAT uses conntrack to perform NAT, everytime a new // flow is seen, a connection is created in the conntrack table, and it // is being used by the NAT module. // Each entry in the conntrack table has associated a timeout, that removes // the connection once it expires. // UDP is a connectionless protocol, so the conntrack module tracking functions // are not very advanced. // It uses a short timeout (30 sec by default) that is renewed if there are new flows // matching the connection. Otherwise it expires the entry. // This behaviour can cause issues in Kubernetes when one entry on the conntrack table // is never expired because the sender does not stop sending traffic, but the pods or // endpoints were deleted, blackholing the traffic // In order to mitigate this problem, Kubernetes delete the stale entries: // - when an endpoint is removed // - when a service goes from no endpoints to new endpoint // Ref: https://api.semanticscholar.org/CorpusID:198903401 // Boye, Magnus. “Netfilter Connection Tracking and NAT Implementation.” (2012). var _ = SIGDescribe("Conntrack", func() { fr := framework.NewDefaultFramework("conntrack") type nodeInfo struct { name   string nodeIP string } var ( cs                             clientset.Interface ns                             string clientNodeInfo, serverNodeInfo nodeInfo ) ginkgo.BeforeEach(func() { cs = fr.ClientSet ns = fr.Namespace.Name nodes, err := e2enode.GetBoundedReadySchedulableNodes(cs, 2) framework.ExpectNoError(err) if len(nodes.Items) < 2 { e2eskipper.Skipf( "Test requires >= 2 Ready nodes, but there are only %v nodes", len(nodes.Items)) } ips := e2enode.CollectAddresses(nodes, v1.NodeInternalIP) clientNodeInfo = nodeInfo{ name:   nodes.Items[0].Name, nodeIP: ips[0], } serverNodeInfo = nodeInfo{ name:   nodes.Items[1].Name, nodeIP: ips[1], } }) ginkgo.It("should be able to preserve UDP traffic when server pod cycles for a NodePort service", func() { // Create a NodePort service udpJig := e2eservice.NewTestJig(cs, ns, serviceName) ginkgo.By("creating a UDP service " + serviceName + " with type=NodePort in " + ns) udpService, err := udpJig.CreateUDPService(func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeNodePort svc.Spec.Ports = []v1.ServicePort{ {Port: 80, Name: "udp", Protocol: v1.ProtocolUDP, TargetPort: intstr.FromInt(80)}, } }) framework.ExpectNoError(err) // Create a pod in one node to create the UDP traffic against the NodePort service every 5 seconds ginkgo.By("creating a client pod for probing the service " + serviceName) clientPod := newAgnhostPod(podClient, "") clientPod.Spec.NodeName = clientNodeInfo.name cmd := fmt.Sprintf(`date; for i in $(seq 1 300); do echo "$(date) Try: ${i}"; echo hostname | nc -u -w 5 -p %d %s %d; echo; done`, srcPort, serverNodeInfo.nodeIP, udpService.Spec.Ports[0].NodePort) clientPod.Spec.Containers[0].Command = []string{"/bin/sh", "-c", cmd} clientPod.Spec.Containers[0].Name = podClient fr.PodClient().CreateSync(clientPod) // Read the client pod logs logs, err := e2epod.GetPodLogs(cs, ns, podClient, podClient) framework.ExpectNoError(err) framework.Logf("Pod client logs: %s", logs) // Add a backend pod to the service in the other node ginkgo.By("creating a backend pod " + podBackend1 + " for the service " + serviceName) serverPod1 := newAgnhostPod(podBackend1, "netexec", fmt.Sprintf("--udp-port=%d", 80)) serverPod1.Labels = udpJig.Labels serverPod1.Spec.NodeName = serverNodeInfo.name fr.PodClient().CreateSync(serverPod1) // Waiting for service to expose endpoint. err = validateEndpointsPorts(cs, ns, serviceName, portsByPodName{podBackend1: {80}}) framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns) // Check that the pod receives the traffic // UDP conntrack entries timeout is 30 sec by default ginkgo.By("checking client pod connected to the backend 1 on Node IP " + serverNodeInfo.nodeIP) time.Sleep(30 * time.Second) logs, err = e2epod.GetPodLogs(cs, ns, podClient, podClient) framework.ExpectNoError(err) framework.Logf("Pod client logs: %s", logs) if !strings.Contains(string(logs), podBackend1) { framework.Failf("Failed to connecto to backend 1") } // Create a second pod ginkgo.By("creating a second backend pod " + podBackend2 + " for the service " + serviceName) serverPod2 := newAgnhostPod(podBackend2, "netexec", fmt.Sprintf("--udp-port=%d", 80)) serverPod2.Labels = udpJig.Labels serverPod2.Spec.NodeName = serverNodeInfo.name fr.PodClient().CreateSync(serverPod2) // and delete the first pod framework.Logf("Cleaning up %s pod", podBackend1) fr.PodClient().DeleteSync(podBackend1, metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout) // Check that the second pod keeps receiving traffic // UDP conntrack entries timeout is 30 sec by default ginkgo.By("checking client pod connected to the backend 2 on Node IP " + serverNodeInfo.nodeIP) time.Sleep(30 * time.Second) logs, err = e2epod.GetPodLogs(cs, ns, podClient, podClient) framework.ExpectNoError(err) framework.Logf("Pod client logs: %s", logs) if !strings.Contains(string(logs), podBackend2) { framework.Failf("Failed to connecto to backend 2") } }) ginkgo.It("should be able to preserve UDP traffic when server pod cycles for a ClusterIP service", func() { // Create a NodePort service udpJig := e2eservice.NewTestJig(cs, ns, serviceName) ginkgo.By("creating a UDP service " + serviceName + " with type=ClusterIP in " + ns) udpService, err := udpJig.CreateUDPService(func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeClusterIP svc.Spec.Ports = []v1.ServicePort{ {Port: 80, Name: "udp", Protocol: v1.ProtocolUDP, TargetPort: intstr.FromInt(80)}, } }) framework.ExpectNoError(err) // Create a pod in one node to create the UDP traffic against the NodePort service every 5 seconds ginkgo.By("creating a client pod for probing the service " + serviceName) clientPod := newAgnhostPod(podClient, "") clientPod.Spec.NodeName = clientNodeInfo.name cmd := fmt.Sprintf(`date; for i in $(seq 1 300); do echo "$(date) Try: ${i}"; echo hostname | nc -u -w 5 -p %d %s %d; echo; done`, srcPort, udpService.Spec.ClusterIP, udpService.Spec.Ports[0].Port) clientPod.Spec.Containers[0].Command = []string{"/bin/sh", "-c", cmd} clientPod.Spec.Containers[0].Name = podClient fr.PodClient().CreateSync(clientPod) // Read the client pod logs logs, err := e2epod.GetPodLogs(cs, ns, podClient, podClient) framework.ExpectNoError(err) framework.Logf("Pod client logs: %s", logs) // Add a backend pod to the service in the other node ginkgo.By("creating a backend pod " + podBackend1 + " for the service " + serviceName) serverPod1 := newAgnhostPod(podBackend1, "netexec", fmt.Sprintf("--udp-port=%d", 80)) serverPod1.Labels = udpJig.Labels serverPod1.Spec.NodeName = serverNodeInfo.name fr.PodClient().CreateSync(serverPod1) // Waiting for service to expose endpoint. err = validateEndpointsPorts(cs, ns, serviceName, portsByPodName{podBackend1: {80}}) framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns) // Check that the pod receives the traffic // UDP conntrack entries timeout is 30 sec by default ginkgo.By("checking client pod connected to the backend 1 on Node IP " + serverNodeInfo.nodeIP) time.Sleep(30 * time.Second) logs, err = e2epod.GetPodLogs(cs, ns, podClient, podClient) framework.ExpectNoError(err) framework.Logf("Pod client logs: %s", logs) if !strings.Contains(string(logs), podBackend1) { framework.Failf("Failed to connecto to backend 1") } // Create a second pod ginkgo.By("creating a second backend pod " + podBackend2 + " for the service " + serviceName) serverPod2 := newAgnhostPod(podBackend2, "netexec", fmt.Sprintf("--udp-port=%d", 80)) serverPod2.Labels = udpJig.Labels serverPod2.Spec.NodeName = serverNodeInfo.name fr.PodClient().CreateSync(serverPod2) // and delete the first pod framework.Logf("Cleaning up %s pod", podBackend1) fr.PodClient().DeleteSync(podBackend1, metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout) // Check that the second pod keeps receiving traffic // UDP conntrack entries timeout is 30 sec by default ginkgo.By("checking client pod connected to the backend 2 on Node IP " + serverNodeInfo.nodeIP) time.Sleep(30 * time.Second) logs, err = e2epod.GetPodLogs(cs, ns, podClient, podClient) framework.ExpectNoError(err) framework.Logf("Pod client logs: %s", logs) if !strings.Contains(string(logs), podBackend2) { framework.Failf("Failed to connecto to backend 2") } }) }) </ins>

Would love to help out in reproing this, I'm at a loss for good next step so any suggestions welcome!
that's the problem with these flakes, they are not easy to repro, and with this low rate of failures it may be caused for combination of running tests, environmental problems (cpu, memory, ...) , ... I always start for checking the current failures and try to find patterns
Let's take a look to one occurrence The tests fails because can't connect to the first pod If we check the code of the tests, we can see that it always uses the port as source port, we can use that to identify the nodePort assigned interesting thing here, we are source nating the pod ip, and we don't see any other conntrack entry The test verifies it has endpoints however, kube-proxy is using endpointlsices since this version but it has no entries so the iptables rules to forward the traffic are not there Seems we have to adapt this function in the e2e framework to use endpointslices cc: EDIT seems that this function is being used by all the network tests that can cause flakes if the endpoint is updated before the endpointslice (we switched recently kube-proxy to use the later)
Great catch. Ideally this function should validate if both Endpoint and appropriate EndpointSlice objects(s) exist.
Actually - let me take it and send a PR for it.
is out for review
Interesting failures here It fails to connect to the first pod
Hmm - I took a quick look at kube-proxy logs and these look fine to me....
I clicked too fast, sorry ... I was looking at the wrong service and I though I've found something
If this issue has been triaged, please comment . If you aren't able to handle this issue, consider unassigning yourself and/or adding the label. I am a bot run by vllry.


Question: <ins> /* Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package network import ( "fmt" "strings" "time" "github.com/onsi/ginkgo" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" e2enode "k8s.io/kubernetes/test/e2e/framework/node" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eservice "k8s.io/kubernetes/test/e2e/framework/service" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" ) const ( serviceName = "svc-udp" podClient   = "pod-client" podBackend1 = "pod-server-1" podBackend2 = "pod-server-2" srcPort     = 12345 ) // Linux NAT uses conntrack to perform NAT, everytime a new // flow is seen, a connection is created in the conntrack table, and it // is being used by the NAT module. // Each entry in the conntrack table has associated a timeout, that removes // the connection once it expires. // UDP is a connectionless protocol, so the conntrack module tracking functions // are not very advanced. // It uses a short timeout (30 sec by default) that is renewed if there are new flows // matching the connection. Otherwise it expires the entry. // This behaviour can cause issues in Kubernetes when one entry on the conntrack table // is never expired because the sender does not stop sending traffic, but the pods or // endpoints were deleted, blackholing the traffic // In order to mitigate this problem, Kubernetes delete the stale entries: // - when an endpoint is removed // - when a service goes from no endpoints to new endpoint // Ref: https://api.semanticscholar.org/CorpusID:198903401 // Boye, Magnus. “Netfilter Connection Tracking and NAT Implementation.” (2012). var _ = SIGDescribe("Conntrack", func() { fr := framework.NewDefaultFramework("conntrack") type nodeInfo struct { name   string nodeIP string } var ( cs                             clientset.Interface ns                             string clientNodeInfo, serverNodeInfo nodeInfo ) ginkgo.BeforeEach(func() { cs = fr.ClientSet ns = fr.Namespace.Name nodes, err := e2enode.GetBoundedReadySchedulableNodes(cs, 2) framework.ExpectNoError(err) if len(nodes.Items) < 2 { e2eskipper.Skipf( "Test requires >= 2 Ready nodes, but there are only %v nodes", len(nodes.Items)) } ips := e2enode.CollectAddresses(nodes, v1.NodeInternalIP) clientNodeInfo = nodeInfo{ name:   nodes.Items[0].Name, nodeIP: ips[0], } serverNodeInfo = nodeInfo{ name:   nodes.Items[1].Name, nodeIP: ips[1], } }) ginkgo.It("should be able to preserve UDP traffic when server pod cycles for a NodePort service", func() { // Create a NodePort service udpJig := e2eservice.NewTestJig(cs, ns, serviceName) ginkgo.By("creating a UDP service " + serviceName + " with type=NodePort in " + ns) udpService, err := udpJig.CreateUDPService(func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeNodePort svc.Spec.Ports = []v1.ServicePort{ {Port: 80, Name: "udp", Protocol: v1.ProtocolUDP, TargetPort: intstr.FromInt(80)}, } }) framework.ExpectNoError(err) // Create a pod in one node to create the UDP traffic against the NodePort service every 5 seconds ginkgo.By("creating a client pod for probing the service " + serviceName) clientPod := newAgnhostPod(podClient, "") clientPod.Spec.NodeName = clientNodeInfo.name cmd := fmt.Sprintf(`date; for i in $(seq 1 300); do echo "$(date) Try: ${i}"; echo hostname | nc -u -w 5 -p %d %s %d; echo; done`, srcPort, serverNodeInfo.nodeIP, udpService.Spec.Ports[0].NodePort) clientPod.Spec.Containers[0].Command = []string{"/bin/sh", "-c", cmd} clientPod.Spec.Containers[0].Name = podClient fr.PodClient().CreateSync(clientPod) // Read the client pod logs logs, err := e2epod.GetPodLogs(cs, ns, podClient, podClient) framework.ExpectNoError(err) framework.Logf("Pod client logs: %s", logs) // Add a backend pod to the service in the other node ginkgo.By("creating a backend pod " + podBackend1 + " for the service " + serviceName) serverPod1 := newAgnhostPod(podBackend1, "netexec", fmt.Sprintf("--udp-port=%d", 80)) serverPod1.Labels = udpJig.Labels serverPod1.Spec.NodeName = serverNodeInfo.name fr.PodClient().CreateSync(serverPod1) // Waiting for service to expose endpoint. err = validateEndpointsPorts(cs, ns, serviceName, portsByPodName{podBackend1: {80}}) framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns) // Check that the pod receives the traffic // UDP conntrack entries timeout is 30 sec by default ginkgo.By("checking client pod connected to the backend 1 on Node IP " + serverNodeInfo.nodeIP) time.Sleep(30 * time.Second) logs, err = e2epod.GetPodLogs(cs, ns, podClient, podClient) framework.ExpectNoError(err) framework.Logf("Pod client logs: %s", logs) if !strings.Contains(string(logs), podBackend1) { framework.Failf("Failed to connecto to backend 1") } // Create a second pod ginkgo.By("creating a second backend pod " + podBackend2 + " for the service " + serviceName) serverPod2 := newAgnhostPod(podBackend2, "netexec", fmt.Sprintf("--udp-port=%d", 80)) serverPod2.Labels = udpJig.Labels serverPod2.Spec.NodeName = serverNodeInfo.name fr.PodClient().CreateSync(serverPod2) // and delete the first pod framework.Logf("Cleaning up %s pod", podBackend1) fr.PodClient().DeleteSync(podBackend1, metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout) // Check that the second pod keeps receiving traffic // UDP conntrack entries timeout is 30 sec by default ginkgo.By("checking client pod connected to the backend 2 on Node IP " + serverNodeInfo.nodeIP) time.Sleep(30 * time.Second) logs, err = e2epod.GetPodLogs(cs, ns, podClient, podClient) framework.ExpectNoError(err) framework.Logf("Pod client logs: %s", logs) if !strings.Contains(string(logs), podBackend2) { framework.Failf("Failed to connecto to backend 2") } }) ginkgo.It("should be able to preserve UDP traffic when server pod cycles for a ClusterIP service", func() { // Create a NodePort service udpJig := e2eservice.NewTestJig(cs, ns, serviceName) ginkgo.By("creating a UDP service " + serviceName + " with type=ClusterIP in " + ns) udpService, err := udpJig.CreateUDPService(func(svc *v1.Service) { svc.Spec.Type = v1.ServiceTypeClusterIP svc.Spec.Ports = []v1.ServicePort{ {Port: 80, Name: "udp", Protocol: v1.ProtocolUDP, TargetPort: intstr.FromInt(80)}, } }) framework.ExpectNoError(err) // Create a pod in one node to create the UDP traffic against the NodePort service every 5 seconds ginkgo.By("creating a client pod for probing the service " + serviceName) clientPod := newAgnhostPod(podClient, "") clientPod.Spec.NodeName = clientNodeInfo.name cmd := fmt.Sprintf(`date; for i in $(seq 1 300); do echo "$(date) Try: ${i}"; echo hostname | nc -u -w 5 -p %d %s %d; echo; done`, srcPort, udpService.Spec.ClusterIP, udpService.Spec.Ports[0].Port) clientPod.Spec.Containers[0].Command = []string{"/bin/sh", "-c", cmd} clientPod.Spec.Containers[0].Name = podClient fr.PodClient().CreateSync(clientPod) // Read the client pod logs logs, err := e2epod.GetPodLogs(cs, ns, podClient, podClient) framework.ExpectNoError(err) framework.Logf("Pod client logs: %s", logs) // Add a backend pod to the service in the other node ginkgo.By("creating a backend pod " + podBackend1 + " for the service " + serviceName) serverPod1 := newAgnhostPod(podBackend1, "netexec", fmt.Sprintf("--udp-port=%d", 80)) serverPod1.Labels = udpJig.Labels serverPod1.Spec.NodeName = serverNodeInfo.name fr.PodClient().CreateSync(serverPod1) // Waiting for service to expose endpoint. err = validateEndpointsPorts(cs, ns, serviceName, portsByPodName{podBackend1: {80}}) framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, ns) // Check that the pod receives the traffic // UDP conntrack entries timeout is 30 sec by default ginkgo.By("checking client pod connected to the backend 1 on Node IP " + serverNodeInfo.nodeIP) time.Sleep(30 * time.Second) logs, err = e2epod.GetPodLogs(cs, ns, podClient, podClient) framework.ExpectNoError(err) framework.Logf("Pod client logs: %s", logs) if !strings.Contains(string(logs), podBackend1) { framework.Failf("Failed to connecto to backend 1") } // Create a second pod ginkgo.By("creating a second backend pod " + podBackend2 + " for the service " + serviceName) serverPod2 := newAgnhostPod(podBackend2, "netexec", fmt.Sprintf("--udp-port=%d", 80)) serverPod2.Labels = udpJig.Labels serverPod2.Spec.NodeName = serverNodeInfo.name fr.PodClient().CreateSync(serverPod2) // and delete the first pod framework.Logf("Cleaning up %s pod", podBackend1) fr.PodClient().DeleteSync(podBackend1, metav1.DeleteOptions{}, framework.DefaultPodDeletionTimeout) // Check that the second pod keeps receiving traffic // UDP conntrack entries timeout is 30 sec by default ginkgo.By("checking client pod connected to the backend 2 on Node IP " + serverNodeInfo.nodeIP) time.Sleep(30 * time.Second) logs, err = e2epod.GetPodLogs(cs, ns, podClient, podClient) framework.ExpectNoError(err) framework.Logf("Pod client logs: %s", logs) if !strings.Contains(string(logs), podBackend2) { framework.Failf("Failed to connecto to backend 2") } }) }) </ins>

just saw it as well
can you please add links to the job with the failures, I just merged new code to improve the logging 2 days ago
ok, here there is a new occurrence and with the new debug logs is easier to follow has unreplied conntrack entries: looking at the other side The iptables rules are there but no conntrack entries related to 10.244.2.32
In I revert PR with debug info as I think that it likely caused significant memory growth in scalability tests, but we have one run that has finished without being killed by OOMKiller (I increased memory limit for that jobs as a mitigation). Maybe logs from this run will be useful for debugging:
If this issue has been triaged, please comment . If you aren't able to handle this issue, consider unassigning yourself and/or adding the label. I am a bot run by vllry.
Two more flakes today:
It seems that the packet never gets DNATed in the receiving host, so it does not reach the pod in the node. There is the same test that is using ClusterIP instead of NodePort that is more reliable. This is being a tough one
/triage accepted
Yesterday's flake - I don't think I'm seeing the log messages from the latest log PR you made ? Or are the prow logs not logging at a high-enough level?
Logs are there, please bear with me: The error on the test is: click on the job failure and you'll see a larger output where you can see the server pod name and get the node where the pod is running: and the client-pod name and its node: and the flow we are looking for: with the node-name we can get the kube-proxy logs there, clicking in artifacts: and you can see the "new" conntrack log entries : and in the client: wow, there is a kernel bug there , can it be related? :thinking:
wohoo - some progress :) I'm not an expert here, but I think it can be? Adding here - they may be able to confirm/reject the hypothesis.
This kernel panic may be a coincidence, maybe this is the reason
Adding some recent failures:
Spotted another flake today:
Another flake in the 5k job:


Question: return "", err } if _, err := os.Stat(filepath.Join(*k8sBinDir, bin)); err != nil { <del> return "", fmt.Errorf("Could not find %s under directory %s.", bin, absPath) </del> <ins> return "", fmt.Errorf("Could not find kube-apiserver under directory %s.", absPath) </ins> } return filepath.Join(absPath, bin), nil }

is failing with: Examples:
This is the failing line of code...
PASS: FAIL: Seems like this disappeared: Also this is flipped: we run the following
my leading theory is this is something in
The issue is that the above PR changed a bunch of to be and thus we no longer wait for getArchive() to complete before starting tests. And unfortunately the previous PR cannot be reverted
After merging, kubelet-gce-e2e-ci still failed with the same issue. reopen the issue. Currently the failure blocked the merge-bot.
did stop the bleeding. Now kubelet-gce-e2e-ci is green: and merge-bot is unblocked.
I think the "go arc" piece was a red herring. I suspect the issue is that the same PR tries to copy the binary into the archive directory, but PR stops building ginkgo, so the copy fails. That is why this line then fails...


Question: - Ingress - Egress ``` <ins> ### Negative caching The `denial` cache TTL has been reduced to the minimum of 5 seconds [here](https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/dns/nodelocaldns/nodelocaldns.yaml#L37). In the unlikely event that this impacts performance, setting this TTL to a higher value make help alleviate issues, but be aware that operations that rely on DNS polling for orchestration may fail (for example operators with StatefulSets). </ins>

What happened: node-local-dns caches negative queries, causing things that do polling for orchestration to fail (like operators with StatefulSets). What you expected to happen: Have a minimal TTL for negative responses (alternatively just disable caching by setting the cache size to 0) that follows what was done for kube-dns: How to reproduce it (as minimally and precisely as possible): With node-local-dns cache deployed, in my cluster it was the that failed to create clusters because the cached entry didn't expire before it had exhausted all of it's retries. Environment: Kubernetes version (use ): 1.11.6 Cloud provider or hardware configuration: GKE OS (e.g. from /etc/os-release): any Kernel (e.g. ): any
/sig network


Question: if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "gci" ]] || [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then cat >>$file <<EOF <del> VOLUME_PLUGIN_DIR: $(yaml-quote ${VOLUME_PLUGIN_DIR:-/home/kubernetes/flexvolume}) </del> <ins> VOLUME_PLUGIN_DIR: $(yaml-quote ${VOLUME_PLUGIN_DIR:-/etc/srv/kubernetes/kubelet-plugins/volume/exec}) </ins> EOF fi

tests are failing atm on GCI https://k8s- because VOLUMEPLUGINDIR=/home/kubernetes/flexvolume set in takes precedence over KUBELETTESTVOLUMEPLUGINDIR="/etc/srv/kubernetes/kubelet-plugins/volume/exec". As far as I can tell /home/kubernetes/flexvolume is not readable yet by containerized kubelet so the test needs to keep using "/etc/srv/kubernetes/kubelet-plugins/volume/exec" and the bash needs to be shuffled around to solve this. cc
/sig storage
This is more an acknowledgment of the issue than anything, can't write a PR until later
cc /kind e2e-test-failure
this is currently blocking alpha.3 release
we are just waiting for approval
I think they’re still failing, not sure what the release process is, but these tests started running only yesterday and so failure indicates something wrong with the test themselves, they shouldn’t block release. (They succeed on gce cluster with Debian nodes but fail on gci)
well, let's see if we can get the kinks worked out before the beta release on 9/27 or we will need to disable it from release-blocking
/assign /reopen
I0824 17:49:28. 8562 ] operationExecutor.VerifyControllerAttachedVolume started for volume "flex-volume-0" (UniqueName: "flexvolume-k8s/dummy-e2e-tests-flexvolume-wf48v/-88f4-11e7-a3f9--flex-volume-0") pod "flex-client" (UID: "-88f4-11e7-a3f9-") edit: VerifyControllerAttachedVolume must succeed since the plugin is non-attachalbe and no error is reported. But the voluem is absent from node.volumesInUse I0824 17:44:06. 4 ] AttachVolume.Attach succeeded for volume "flex-volume-0" (UniqueName: "flexvolume-k8s/dummy-attachable-e2e-tests-flexvolume-j8962/flex-volume-0") from node "bootstrap-e2e-minion-group-9913" W0824 17:44:06.


Question: if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "gci" ]] || [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then cat >>$file <<EOF <del> VOLUME_PLUGIN_DIR: $(yaml-quote ${VOLUME_PLUGIN_DIR:-/home/kubernetes/flexvolume}) </del> <ins> VOLUME_PLUGIN_DIR: $(yaml-quote ${VOLUME_PLUGIN_DIR:-/etc/srv/kubernetes/kubelet-plugins/volume/exec}) </ins> EOF fi

4 plugin-] flexVolume driver k8s/dummy-attachable-e2e-tests-flexvolume-j8962: using default GetVolumeName for volume 0x1c5df60 I0824 17:43:59. 3728 ] operationExecutor.VerifyControllerAttachedVolume started for volume "flex-volume-0" (UniqueName: "flexvolume-k8s/dummy-attachable-e2e-tests-flexvolume-j8962/flex-volume-0") pod "flex-client" (UID: "-88f3-11e7-a3f9-") E0824 17:43:59. 3728 ] Operation for ""flexvolume-k8s/dummy-attachable-e2e-tests-flexvolume-j8962/flex-volume-0"" failed. No retries permitted until 2017-08-24 17:44:00. +0000 UTC (durationBeforeRetry 500ms). Error: Volume has not been to the list of VolumesInUse in the node's volume status for volume "flex-volume-0" (UniqueName: "flexvolume-k8s/dummy-attachable-e2e-tests-flexvolume-j8962/flex-volume-0") pod "flex-client" (UID: "-88f3-11e7-a3f9-") Seems there is something wrong with VerifyControllerAttachedVolume, still at a loss as to why it's showing up in GCI but not Debian edit: get node -o yaml shows: yet it errors because the volume hasn't been to volumesInUse??? Aug 24 20:31:42 mawong-e2e-minion-group-npjh kubelet[9431]: E0824 20:31:42. 9431 ] Operation for ""flexvolume-k8s/dummy-attachable-e2e-tests-flexvolume-z8xc4/flex-volume-0"" failed. No retries permitted until 2017-08-24 20:31:44. +0000 UTC (durationBeforeRetry 2s).


Question: if [[ "${master}" == "true" && "${MASTER_OS_DISTRIBUTION}" == "gci" ]] || [[ "${master}" == "false" && "${NODE_OS_DISTRIBUTION}" == "gci" ]]; then cat >>$file <<EOF <del> VOLUME_PLUGIN_DIR: $(yaml-quote ${VOLUME_PLUGIN_DIR:-/home/kubernetes/flexvolume}) </del> <ins> VOLUME_PLUGIN_DIR: $(yaml-quote ${VOLUME_PLUGIN_DIR:-/etc/srv/kubernetes/kubelet-plugins/volume/exec}) </ins> EOF fi

Error: Volume has not been to the list of VolumesInUse in the node's volume status for volume "flex-volume-0" (UniqueName: "flexvolume-k8s/dummy-attachable-e2e-tests-flexvolume-z8xc4/flex-volume-0") pod "flex-client" (UID: "-890b-11e7-9392-")
Never mind the GCI vs Debian issue, our test fix would have worked but in between and , something else changed and is causing both GCI and Debian tests to fail.
might be the cause, so we might be looking at legitimate failures caught by the test at this point (yay?) and not faulty tests. investigating...


Question: readonly node_logfiles="kube-proxy" readonly aws_logfiles="cloud-init-output" readonly gce_logfiles="startupscript" <del> readonly common_logfiles="kern" </del> <ins> readonly kern_logfile="kern" </ins> readonly initd_logfiles="docker" readonly supervisord_logfiles="kubelet supervisor/supervisord supervisor/kubelet-stdout supervisor/kubelet-stderr supervisor/docker-stdout supervisor/docker-stderr"

Is this a request for help? (If yes, you should use our troubleshooting guide and community support channels, see ): No What keywords did you search in Kubernetes issues before filing this one? (If you have found any duplicates, you should instead reply there.): "LRUExpireCache" Is this a BUG REPORT or FEATURE REQUEST? (choose one): BUG REPORT What happened: requires write lock since needs to manipulate its list to track recently used item. Currently it uses read lock so it may introduce race condition. What you expected to happen: How to reproduce it (as minimally and precisely as possible): Anything else do we need to know:
This was fixed in


Question: } } } <ins> func TestGetRbdImageInfo(t *testing.T) { tmpDir, err := utiltesting.MkTmpdir("rbd_test") if err != nil { t.Fatalf("error creating temp dir: %v", err) } defer os.RemoveAll(tmpDir) for i, c := range []struct { DeviceMountPath    string TargetRbdImageInfo *rbdImageInfo }{ { DeviceMountPath:    fmt.Sprintf("%s/plugins/kubernetes.io/rbd/rbd/pool1-image-image1", tmpDir), TargetRbdImageInfo: &rbdImageInfo{pool: "pool1", name: "image1"}, }, { DeviceMountPath:    fmt.Sprintf("%s/plugins/kubernetes.io/rbd/mounts/pool2-image-image2", tmpDir), TargetRbdImageInfo: &rbdImageInfo{pool: "pool2", name: "image2"}, }, } { rbdImageInfo, err := getRbdImageInfo(c.DeviceMountPath) if err != nil { t.Errorf("Case %d: getRbdImageInfo failed: %v", i, err) continue } if !reflect.DeepEqual(rbdImageInfo, c.TargetRbdImageInfo) { t.Errorf("Case %d: unexpected RbdImageInfo, wanted %v, got %v", i, c.TargetRbdImageInfo, rbdImageInfo) } } } </ins>

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: What happened: Pod was deleted successfully but there is still a lock on the ceph image. I find two logs,one for that lock not removed and another for that lock removed. I find that there are little difference between them. the log that lock not removed: and the log that lock removed: What you expected to happen: How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): v1.6.7 Cloud provider or hardware configuration**: OS (e.g. from /etc/os-release): centos7 Kernel (e.g. ): 3.10.0-327.el7.x86_64 Install tools: Others:
/sig storage
Error 16 is EBUSY. similar to
were you running kubelet with cadvisor on?
/assign
yes, every kubelet run with cadvisor,and i know what code 16 represent for,but i still don't know the reason, is this a bug? How can i avoid this kind of situation from happening,thank you
It is probably related to
We see some other case may cause , kubelet run with nodeexporter and nodeexporter is deployed by k8s daemonset with -v /:/rootfs:ro (mount kubelet host rootfs into nodeexporter container). rbd device is mounted before nodeexporter started, because k8s 1.6.7 not support mount propagation, when the rbd plugin unmount /dev/rbd0 on the host namespace, the mount info will still leave over in node_exporter container namespace, may cause . hope this would help
thanks, does the problem persist if node_exporter stops collecting filesystem metrics?
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with .


Question: } } } <ins> func TestGetRbdImageInfo(t *testing.T) { tmpDir, err := utiltesting.MkTmpdir("rbd_test") if err != nil { t.Fatalf("error creating temp dir: %v", err) } defer os.RemoveAll(tmpDir) for i, c := range []struct { DeviceMountPath    string TargetRbdImageInfo *rbdImageInfo }{ { DeviceMountPath:    fmt.Sprintf("%s/plugins/kubernetes.io/rbd/rbd/pool1-image-image1", tmpDir), TargetRbdImageInfo: &rbdImageInfo{pool: "pool1", name: "image1"}, }, { DeviceMountPath:    fmt.Sprintf("%s/plugins/kubernetes.io/rbd/mounts/pool2-image-image2", tmpDir), TargetRbdImageInfo: &rbdImageInfo{pool: "pool2", name: "image2"}, }, } { rbdImageInfo, err := getRbdImageInfo(c.DeviceMountPath) if err != nil { t.Errorf("Case %d: getRbdImageInfo failed: %v", i, err) continue } if !reflect.DeepEqual(rbdImageInfo, c.TargetRbdImageInfo) { t.Errorf("Case %d: unexpected RbdImageInfo, wanted %v, got %v", i, c.TargetRbdImageInfo, rbdImageInfo) } } } </ins>

Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten /remove-lifecycle stale
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close
I'm experiencing this issue. From what I can read above, if the rbd mount happens before the node_exporter starts; we'll always experience status 16?
We're getting this issue. This is because the mounted volume is mounted on Cadvisor container too. So the /dev/rbdX file is still in use. But still didn't find any solution to fix.
rbd volume will stuck when failed as will always returen
My solution: container use a volumeMount. make a script file to be executed every a minute:


Question: "k8s.io/apimachinery/pkg/util/strategicpatch" core "k8s.io/client-go/testing" <ins> "time" </ins> ) func TestRealHistory_ListControllerRevisions(t *testing.T) {

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: Statefulsets are sometimes continually rolling pods and I don't know why statefulset created: storageclass referenced: kubectl describe statefulset statefulset-test: As you can see from the above describe, the pod in the statefulset has been restarted a lot of times. Observing the created pod, 1) The pod comes up ready 2) immediately transitions to terminating 3) Is terminated 4) A new pod is created 5) Go back to (1) (repeats forever) Despite no updates to the statefulset spec ocurring. Making a change to the statefulset template (for example a trivial change adding a nonsense annotation) stops the rolling and fixes the issue When this statefulset was created, there was a mutating webhook enabled. After the observed pod rolling. I disabled this by making its configuration invalid and making it non required, and the rolling persisted. The status of the statefulset: : You can see that there is a discrepency between currentRevision and updateRevision. currentRevision: statefulset-test- updateRevision: statefulset-test- A short while later the status is changed to contain currentRevision: statefulset-test- updateRevision: statefulset-test- This change happened at Sun Apr 1 19:48:48 2018 Presumably after the rollout of statefulset-test- is complete.


Question: "k8s.io/apimachinery/pkg/util/strategicpatch" core "k8s.io/client-go/testing" <ins> "time" </ins> ) func TestRealHistory_ListControllerRevisions(t *testing.T) {

Here are all the controllerrevisions from above: kubectl get controllerrevision statefulset-test- -o yaml kubectl get controllerrevision statefulset-test- -o yaml: kubectl get controllerrevision statefulset-test- -o yaml: The differences between the above controllerrevisions are insignificant (the hash, the creation timestamp, the name, the resource version, the uid, the self link) Interestingly all these controllerrevisions were created very close together in time shortly after I created the statefulset The current controllerrevisions are After watching the state change for a while, it actually seems like the statefulset is just cycling through the updateRevision being one of the above 10 controllerrevisions Leaders during this time From kube-controller-manager logs during this time, here is one excerpt: The error sync in there is suspicous, and as you can see the controllerrevisions are changing in the logs I have the full logs from a 5 minute window on either side of the above if necessary. The reason I'm inclined to think this is a kubernetes bug is that any update to the statefulset template fixes the problem. How to reproduce it (as minimally and precisely as possible): Not really sure. Worth just trying to create the above statefulset (requires creating the I tried deleting and recreating the above statefulset and it didn't happen, so something flaky is going on Anything else we need to know?: kubernetes 1.9.3 (client and server) cloud provider: aws
/sig apps
I realize this will probably need some further debugging, happy to get some direction
Consistent repro: Create a statefulset that creates a controllerrevision whose hash is between max int32 and max uint32 (this shouldn't be that hard to do, possibly a 50/50 chance of this). Manually create a controllerrevision identical to the first (same revision), incrementing both the name and hash by one. See that the statefulset controller will continually switch which controller revision should be the target of the statefulset.
Due to the issue fixed here: , under some circumstances controllerrevisions whose revision is the same were being considered equal.


Question: "k8s.io/apimachinery/pkg/util/strategicpatch" core "k8s.io/client-go/testing" <ins> "time" </ins> ) func TestRealHistory_ListControllerRevisions(t *testing.T) {

This ends up causing this bug to manifest as follows: ListRevisions finds many controllerrevisions with the same revision (unclear how this state happens, but it can happen) The sort here randomly sorts them Due to the linked bug , FindEqualRevisions incorrectly finds that all these controller revisions are equal to the proposed udpate. The updateRevision selects the last revision found here Therefore, updateRevision, is randomly selected and causes the bug. This fixes the immediate issue, but the sort should probably be made stable (i.e not just look at the revision number, perhaps just consider the hash for tie breakers)
As an update, fixing the equality check will not be sufficient because it is possible for multiple controller revisions to be created that are actually equal, including the same hash, that have the same revision number. I believe the fix will need to be equal controllerrevisions with the same revision number from being created sort the controllerrevisions so that the controllerrevision that is the target of the statefulset is stable
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
ping I use the K8S which version is 1.11.4 and encounter the same problem. I want to reproduce the problem and find your way: Manually create a controllerrevision identical to the first (same revision), incrementing both the name and hash by one. Could you tell me how to mannually create a same controllerrevision identical to the first? Thanks in anvance.


Question: ### Hook Handler Implementations Hook handlers are the way that hooks are surfaced to containers.  Containers can select the type of hook handler they would like to implement.  Kubernetes currently supports two different hook handler types: <del> * Exec - Executes a specific command (e.g. pre-stop.sh) inside the cgroup and namespaces of the container.  Resources consumed by the command are counted against the container.  Commands which return non-zero values are treated as container failures (and will cause kubelet to forcibly restart the container).  Parameters are passed to the command as traditional linux command line flags (e.g. pre-stop.sh --reason=HEALTH) </del> <ins> * Exec - Executes a specific command (e.g. pre-stop.sh) inside the cgroup and namespaces of the container.  Resources consumed by the command are counted against the container.  Commands which print "ok" to standard out (stdout) are treated as healthy, any other output is treated as container failures (and will cause kubelet to forcibly restart the container).  Parameters are passed to the command as traditional linux command line flags (e.g. pre-stop.sh --reason=HEALTH) </ins> * HTTP - Executes an HTTP request against a specific endpoint on the container.  HTTP error codes (5xx) and non-response/failure to connect are treated as container failures. Parameters are passed to the http endpoint as query args (e.g. http://some.server.com/some/path?reason=HEALTH)

It seems that exec probes must emit "ok" to be considered success, rather than return 0. If this is what we really want, we should document it somewhere.
This is (was?) a limitation of docker exec, which didn't return the code. Brendan On Jan 28, 2015 8:38 AM, "Tim Hockin" wrote:
The doc says it looks at exit status, so we need to fix one or the other.
for kubelet
for exec
could you take a look at this? Feel free to re-assign it back to me.
FWIW the current behavior isn't SO bad, and it makes clear a failure in the thing that probed vs a failed probe. Just saying, we don't NECESSARILY need to change it. On Jan 28, 2015 5:26 PM, "Dawn Chen" wrote:
With docker 1.5 we can go back to using the exitCode instead of "ok" string. OOM kills will also have special exitCodes in docker 1.5. , Tim Hockin wrote:
ok, I'm going to change the docs for now, we can change back when we standardize on 1.5...


Question: func rcByNameContainer(name string, replicas int, image string, labels map[string]string, c api.Container) *api.ReplicationController { // Add "name": name to the labels, overwriting if it exists. labels["name"] = name <ins> gracePeriod := int64(0) </ins> return &api.ReplicationController{ TypeMeta: unversioned.TypeMeta{ Kind:       "ReplicationController",

Nodes Network when a minion node becomes unreachable [replication controller] recreates pods scheduled on the unreachable minion node AND allows scheduling of pods on a minion after it rejoins the cluster(Failed 17 times in the last 30 runs. Stability: 43 %)Failed with 0 retries
for triage
I'll look into this.
I talked to about this and he had a pretty good guess in about two seconds about what the problem is. ReplicationController sync method does this: but DaemonSet controller doesn't. (I guess when DaemonSet was copy-pasted from ReplicationController, this was accidentally removed.) Pod reflector and syncer are running in separate goroutines, and if pod reflector doesn't run then syncer will think there are no pods on the node.
To be clear: this is about the test Controller Manager should not create/delete replicas across restart which is the one that is actually failing.
I'm confused you're saying that this is about , but you're updating . I thought they are about two different tests...
Yes sorry, I put my comment in the wrong issue. Reassigning this one to you, and I'll copy my comment into the other one.
First failure: We check if all namespace are deleted. If other tests failed to delete their namespace this test will fail. I'm going to delete this logic as we already check if we have successfully deleted our namespace. See also for some more details.
Second failure: We disable network by running the following command: I checked and it seems that it results in ing master to hang. I suspect that Kubelet hangs on updating node status and depending when it breaks the connection (by timeout) it will fit in test timeouts or not.
What is the default timeout for connections using client libraries to apiserver? I'm checking the code, but can't find it.
I'm not sure if we set one, but we should. I guess I would expect to see that in the config, w/ default set by this function: , Filip Grzadkowski wrote:
So: let's add a timeout, and revert .
I think that adding a timeout and reverting are orthogonal. I we want to simulate network problems with node-master communication then using REJECT is better anyway. At the same time adding timeout makes sense for many other reasons.


Question: func rcByNameContainer(name string, replicas int, image string, labels map[string]string, c api.Container) *api.ReplicationController { // Add "name": name to the labels, overwriting if it exists. labels["name"] = name <ins> gracePeriod := int64(0) </ins> return &api.ReplicationController{ TypeMeta: unversioned.TypeMeta{ Kind:       "ReplicationController",

OK then, let's add a timeout and test both DROP and REJECT-- no sort of network problem should hang a component.
I think it didn't hang. It just that the kubelet was retrying too slow and sometimes it fit test timeout, sometime it didn't.
Could we reduce the timeout to make it always fit the test timeout? , Filip Grzadkowski
I guess we can. I'll try tomorrow.
It seems we have a new type of failure: It seems that kubelet on a machine where network is blocked doesn't notice new pod. After reading logs I assume that kubelet doesn't notice that the watch is broken and doesn't try to relist everything. It notices new pod only after watch timeout.
Ugh, I hope we don't have to start sending watch heartbeats.
It seems that all recent failures are with the following error: which means the reason is here: - can you please take a look or delegate?
I did some more debugging of it. I was looking into this failure: There is something strange happening here: run performTemporaryNetworkFailure: succesfully block the load from kubelet: the succesfully wait for node to become not ready: some time nodecontroller successfully removes the pod: replication controller creates a new pod: here is something I don't undertand - because of some reason, the test doesn't observe that the pod actually disappeared. - any thoughts on it?
BTW - this is exactly the same situation in:
OK - I think this is related to default 30s deletion grace period.
Yes - it seems that this is exactly the case. I will send out PR fixing this out for review.
I'm reopening it for tracking. If there won't be any failures within next few days - I will move the test back from flaky and close this issue.
Thanks for fixing, .
It seems that the main issue is fixed. However, there was one failure in the last 30 runs: in which one pod simply didn't started (and it was "preparation" for the test, not the test itself).
During next 2 days there was exactly one more failure of this test: it is exactly the same as the previous one. However, those are duplicates of So I'm going to move this test out of flaky suite.


Question: } if kernelVersion.LessThan(version.MustParseGeneric(connReuseMinSupportedKernelVersion)) { klog.ErrorS(nil, fmt.Sprintf("can't set sysctl %s, kernel version must be at least %s", sysctlConnReuse, connReuseMinSupportedKernelVersion)) <ins> } else if kernelVersion.AtLeast(version.MustParseGeneric(connReuseFixedKernelVersion)) { // https://github.com/kubernetes/kubernetes/issues/93297 klog.V(2).InfoS("Left as-is", "sysctl", sysctlConnReuse) </ins> } else { // Set the connection reuse mode if err := utilproxy.EnsureSysctl(sysctl, sysctlConnReuse, 0); err != nil {

There were two patches merged recently in the kernel that will significantly improve the IPVS proxier: The first patch fixes an issue where a packet is dropped whenever a stale conntrack entry is reused. This caused a 1s delay whenever the stale conntrack entry was used. This was the original reason for setting . More details in The second patch fixes an issue when a source port is reused from a given client and the corresponding destination was deleted. This is easily reproducible when a single client sends a lot traffic to a small set of destinations and one of the destination servers are removed. More details in Based on these two patches, we should start setting (default) starting on Linux v5.9 when those two patches make it to the main line. The new sysctls for the IPVS proxier would be: We should probably wait for v5.9 before making any changes to kube-proxy, this is just a tracking issue so we don't forget.
/sig network /area ipvs
Should probably hold on the PR until v5.9 is released later this year. cc
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
/remove-triage unresolved
Yes it makes sense to wait until a kernel with the patch exists ;) I think we can also avoid touching it and follow the approach "only touch the sysctl we absolutely need, otherwise trust the admin to know what they need"
I install linux-5.9-rc2 in centos7. set /proc/sys/net/ipv4/vs/connreusemode = 1 i have three service: service-a ---service-b --service-c in k8s. service-c return "helloworld" I use ab to benchmark service-a. [root ~]# ab -n -c 1000 http://172.17.2.66:/ This is ApacheBench, Version 2.3 <Copyright 1996 Adam Twiss, Zeus Technology Ltd, Licensed to The Apache Software Foundation, Benchmarking 172.17.2.66 (be patient) Completed 1000 requests Completed 2000 requests Completed 3000 requests Completed 4000 requests Completed 5000 requests Completed 6000 requests Completed 7000 requests Completed 8000 requests Completed 9000 requests Completed requests Finished requests Server Software: Server Hostname: 172.


Question: } if kernelVersion.LessThan(version.MustParseGeneric(connReuseMinSupportedKernelVersion)) { klog.ErrorS(nil, fmt.Sprintf("can't set sysctl %s, kernel version must be at least %s", sysctlConnReuse, connReuseMinSupportedKernelVersion)) <ins> } else if kernelVersion.AtLeast(version.MustParseGeneric(connReuseFixedKernelVersion)) { // https://github.com/kubernetes/kubernetes/issues/93297 klog.V(2).InfoS("Left as-is", "sysctl", sysctlConnReuse) </ins> } else { // Set the connection reuse mode if err := utilproxy.EnsureSysctl(sysctl, sysctlConnReuse, 0); err != nil {

17.2.66 Server Port: Document Path: / Document Length: 11 bytes Concurrency Level: 1000 Time taken for tests: 4.128 seconds Complete requests: Failed requests: 0 Write errors: 0 Total transferred: bytes HTML transferred: bytes Requests per second: 2422.76 [#/sec] (mean) Time per request: 412.753 [ms] (mean) Time per request: 0.413 [ms] (mean, across all concurrent requests) Transfer rate: 302.84 [Kbytes/sec] received Connection Times (ms) min mean[+/-sd] median max Connect: 0 16 71.5 4 1004 Processing: 4 216 92.9 202 2340 Waiting: 4 215 92.7 202 2340 Total: 4 232 121.5 215 2341 Percentage of the requests served within a certain time (ms) 50% 215 66% 253 75% 280 80% 299 90% 348 95% 412 98% 490 99% 556 100% 2341 (longest request) [root ~]# ab -n -c 1000 http://172.17.2.66:/ This is ApacheBench, Version 2.3 <Copyright 1996 Adam Twiss, Zeus Technology Ltd, Licensed to The Apache Software Foundation, Benchmarking 172.17.2.66 (be patient) Completed 1000 requests Completed 2000 requests Completed 3000 requests Completed 4000 requests Completed 5000 requests Completed 6000 requests Completed 7000 requests Completed 8000 requests Completed 9000 requests Completed requests Finished requests Server Software: Server Hostname: 172.17.2.66 Server Port: Document Path: / Document Length: 11 bytes Concurrency Level: 1000 Time taken for tests: 7.071 seconds Complete requests: Failed requests: 0 Write errors: 0 Total transferred: bytes HTML transferred: bytes Requests per second: 1414.21 [#/sec] (mean) Time per request: 707.110 [ms] (mean) Time per request: 0.707 [ms] (mean, across all concurrent requests) Transfer rate: 176.


Question: } if kernelVersion.LessThan(version.MustParseGeneric(connReuseMinSupportedKernelVersion)) { klog.ErrorS(nil, fmt.Sprintf("can't set sysctl %s, kernel version must be at least %s", sysctlConnReuse, connReuseMinSupportedKernelVersion)) <ins> } else if kernelVersion.AtLeast(version.MustParseGeneric(connReuseFixedKernelVersion)) { // https://github.com/kubernetes/kubernetes/issues/93297 klog.V(2).InfoS("Left as-is", "sysctl", sysctlConnReuse) </ins> } else { // Set the connection reuse mode if err := utilproxy.EnsureSysctl(sysctl, sysctlConnReuse, 0); err != nil {

78 [Kbytes/sec] received Connection Times (ms) min mean[+/-sd] median max Connect: 0 9 66.2 2 1019 Processing: 10 661 575.2 372 3603 Waiting: 1 659 573.7 372 3603 Total: 10 670 584.6 380 3607 Percentage of the requests served within a certain time (ms) 50% 380 66% 742 75% 1008 80% 1183 90% 1535 95% 1784 98% 2376 99% 2479 100% 3607 (longest request) [root ~]# ab -n -c 1000 http://172.17.2.66:/ This is ApacheBench, Version 2.3 <Copyright 1996 Adam Twiss, Zeus Technology Ltd, Licensed to The Apache Software Foundation, Benchmarking 172.17.2.66 (be patient) Completed 1000 requests Completed 2000 requests Completed 3000 requests Completed 4000 requests Completed 5000 requests Completed 6000 requests Completed 7000 requests Completed 8000 requests Completed 9000 requests Completed requests Finished requests Server Software: Server Hostname: 172.17.2.66 Server Port: Document Path: / Document Length: 11 bytes Concurrency Level: 1000 Time taken for tests: 13.580 seconds Complete requests: Failed requests: 0 Write errors: 0 Total transferred: bytes HTML transferred: bytes Requests per second: 736.40 [#/sec] (mean) Time per request: 1357.958 [ms] (mean) Time per request: 1.358 [ms] (mean, across all concurrent requests) Transfer rate: 92.05 [Kbytes/sec] received Connection Times (ms) min mean[+/-sd] median max Connect: 0 4 6.6 1 49 Processing: 11 1333 491.9 1307 3185 Waiting: 1 1330 491.9 1305 3185 Total: 17 1337 491.


Question: } if kernelVersion.LessThan(version.MustParseGeneric(connReuseMinSupportedKernelVersion)) { klog.ErrorS(nil, fmt.Sprintf("can't set sysctl %s, kernel version must be at least %s", sysctlConnReuse, connReuseMinSupportedKernelVersion)) <ins> } else if kernelVersion.AtLeast(version.MustParseGeneric(connReuseFixedKernelVersion)) { // https://github.com/kubernetes/kubernetes/issues/93297 klog.V(2).InfoS("Left as-is", "sysctl", sysctlConnReuse) </ins> } else { // Set the connection reuse mode if err := utilproxy.EnsureSysctl(sysctl, sysctlConnReuse, 0); err != nil {

9 1311 3191 Percentage of the requests served within a certain time (ms) 50% 1311 66% 1565 75% 1679 80% 1748 90% 1911 95% 2129 98% 2384 99% 2650 100% 3191 (longest request) [root ~]# ab -n -c 1000 http://172.17.2.66:/ This is ApacheBench, Version 2.3 <Copyright 1996 Adam Twiss, Zeus Technology Ltd, Licensed to The Apache Software Foundation, Benchmarking 172.17.2.66 (be patient) Completed 1000 requests Completed 2000 requests Completed 3000 requests Completed 4000 requests Completed 5000 requests Completed 6000 requests Completed 7000 requests Completed 8000 requests Completed 9000 requests Completed requests Finished requests Server Software: Server Hostname: 172.17.2.66 Server Port: Document Path: / Document Length: 11 bytes Concurrency Level: 1000 Time taken for tests: 23.720 seconds Complete requests: Failed requests: 936 (Connect: 0, Receive: 0, Length: 936, Exceptions: 0) Write errors: 0 Total transferred: bytes HTML transferred: bytes Requests per second: 421.58 [#/sec] (mean) Time per request: 2372.009 [ms] (mean) Time per request: 2.372 [ms] (mean, across all concurrent requests) Transfer rate: 47.77 [Kbytes/sec] received Connection Times (ms) min mean[+/-sd] median max Connect: 0 5 7.5 1 49 Processing: 8 2345 3364.8 333 Waiting: 0 1744 3030.9 277 Total: 8 2350 3364.8 340 Percentage of the requests served within a certain time (ms) 50% 340 66% 1434 75% 3390 80% 5490 90% 8669 95% 9233 98% 99% 100% (longest request) the last time tcp connections of service-b :
/assign Sorry, but I do not understand what are you trying to say.


Question: } if kernelVersion.LessThan(version.MustParseGeneric(connReuseMinSupportedKernelVersion)) { klog.ErrorS(nil, fmt.Sprintf("can't set sysctl %s, kernel version must be at least %s", sysctlConnReuse, connReuseMinSupportedKernelVersion)) <ins> } else if kernelVersion.AtLeast(version.MustParseGeneric(connReuseFixedKernelVersion)) { // https://github.com/kubernetes/kubernetes/issues/93297 klog.V(2).InfoS("Left as-is", "sysctl", sysctlConnReuse) </ins> } else { // Set the connection reuse mode if err := utilproxy.EnsureSysctl(sysctl, sysctlConnReuse, 0); err != nil {

: (
/cc
Will this be implemented in 1.21?
/area kube-proxy
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle stale


Question: mkdir -p "${download_path}" if [[ $(which gsutil) ]] && [[ "$url" =~ ^https://storage.googleapis.com/.* ]]; then <del> gsutil cp "${url//'https://storage.googleapis.com/'/'gs://'}" "${download_path}/${file}" </del> <ins> gsutil cp "${url//'https://storage.googleapis.com/'/gs://}" "${download_path}/${file}" </ins> elif [[ $(which curl) ]]; then # if the url belongs to GCS API we should use oauth2_token in the headers curl_headers=""

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --command fails because of I think the link is from k/k's Am I right?? kubetest command succeeds With installed, OR Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig release /sig testing
/assign
Need cherry-picks for 1.21, 1.20 to address this. is updated but downloaded downloads and executes its /reopen
Reopened this issue. <detailsIn response to : Need cherry-picks for 1.21, 1.20 to address this. Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
cc
this was cherry picked AFAICT, thank you!


Question: Expect(err).ToNot(HaveOccurred()) podZone := node.Labels[apis.LabelZoneFailureDomain] <del> // TODO (verult) Consider using node taints to simulate zonal failure instead. By("deleting instance group belonging to pod's zone") // Asynchronously detect a pod reschedule is triggered during/after instance group deletion. waitStatus := make(chan error) go func() { waitStatus <- waitForStatefulSetReplicasNotReady(statefulSet.Name, ns, c) }() cloud, err := gce.GetGCECloud() if err != nil { Expect(err).NotTo(HaveOccurred()) } instanceGroupName := framework.TestContext.CloudConfig.NodeInstanceGroup instanceGroup, err := cloud.GetInstanceGroup(instanceGroupName, podZone) Expect(err).NotTo(HaveOccurred(), "Error getting instance group %s in zone %s", instanceGroupName, podZone) templateName, err := framework.GetManagedInstanceGroupTemplateName(podZone) Expect(err).NotTo(HaveOccurred(), "Error getting instance group template in zone %s", podZone) err = framework.DeleteManagedInstanceGroup(podZone) Expect(err).NotTo(HaveOccurred(), "Error deleting instance group in zone %s", podZone) </del> <ins> By("tainting nodes in the zone the pod is scheduled in") selector := labels.SelectorFromSet(labels.Set(map[string]string{apis.LabelZoneFailureDomain: podZone})) nodesInZone, err := c.CoreV1().Nodes().List(metav1.ListOptions{LabelSelector: selector.String()}) Expect(err).ToNot(HaveOccurred()) removeTaintFunc := addTaint(c, ns, nodesInZone.Items, podZone) </ins> defer func() { <del> framework.Logf("recreating instance group %s", instanceGroup.Name) framework.ExpectNoError(framework.CreateManagedInstanceGroup(instanceGroup.Size, podZone, templateName), "Error recreating instance group %s in zone %s", instanceGroup.Name, podZone) framework.ExpectNoError(framework.WaitForReadyNodes(c, nodeCount, framework.RestartNodeReadyAgainTimeout), "Error waiting for nodes from the new instance group to become ready.") </del> <ins> framework.Logf("removing previously added node taints") removeTaintFunc() </ins> }() <del> err = <-waitStatus Expect(err).ToNot(HaveOccurred(), "Error waiting for replica to be deleted during failover: %v", err) </del> <ins> By("deleting StatefulSet pod") err = c.CoreV1().Pods(ns).Delete(pod.Name, &metav1.DeleteOptions{}) // Verify the pod is scheduled in the other zone. By("verifying the pod is scheduled in a different zone.") var otherZone string if cloudZones[0] == podZone { otherZone = cloudZones[1] } else { otherZone = cloudZones[0] } err = wait.PollImmediate(framework.Poll, statefulSetReadyTimeout, func() (bool, error) { framework.Logf("checking whether new pod is scheduled in zone %q", otherZone) pod = getPod(c, ns, regionalPDLabels) nodeName = pod.Spec.NodeName node, err = c.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{}) if err != nil { return false, nil } newPodZone := node.Labels[apis.LabelZoneFailureDomain] return newPodZone == otherZone, nil }) Expect(err).NotTo(HaveOccurred(), "Error waiting for pod to be scheduled in a different zone (%q): %v", otherZone, err) </ins> err = framework.WaitForStatefulSetReplicasReady(statefulSet.Name, ns, c, 3*time.Second, framework.RestartPodReadyAgainTimeout) if err != nil {

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . If the matter is security related, please disclose it privately via --Is this a BUG REPORT or FEATURE REQUEST?: What happened: Testgrid: https://k8s-
/assign
The test's assumption about test parameter is wrong. Previously it assumed the group name is the same across all zones, but they are different in regional clusters. Looking into using node taints as the permanent fix, as suggested by the TODO comment in the test.
Hm I think taint based evictions is still an alpha feature.
OK. I propose the following solution: Get the zone of Schedule the StatefulSet pod to that zone using node affinity; wait for the pod to run Remove the node affinity from StatefulSet Delete
Can it be simplified? Schedule the pod Figure out which zone that pod got scheduled in Delete the instance group of that zone
Unfortunately I don't know of a good way to look up the instance group given the zone, since there could be instance groups from multiple clusters in the same project
How about: Schedule the pod Figure out the zone Taint all the nodes in the zone Kill the pod
Taint with NoSchedule? yeah that sounds good


Question: cp "${src_file}" /etc/kubernetes/manifests } <ins> # Starts k8s cluster autoscaler. start_cluster_autoscaler() { if [ "${ENABLE_NODE_AUTOSCALER:-}" = "true" ]; then # Remove salt comments and replace variables with values src_file="${kube_home}/kube-manifests/kubernetes/gci-trusty/cluster-autoscaler.manifest" remove_salt_config_comments "${src_file}" local params=`sed 's/^/"/;s/ /","/g;s/$/",/' <<< "${AUTOSCALER_MIG_CONFIG}"` sed -i -e "s@"{{param}}",@${params}@g" "${src_file}" sed -i -e "s@{%.*%}@@g" "${src_file}" cp "${src_file}" /etc/kubernetes/manifests fi } </ins> # Starts a fluentd static pod for logging. start_fluentd() { if [ "${ENABLE_NODE_LOGGING:-}" = "true" ]; then

For some reason they stopped working and we even don't create manifest file for cluster autoscaler. I used the following command to start the cluster:
It's possible it's related to


Question: // Will operate in the host mount namespace if kubelet is running in a container. // Error is returned on any other error than "file not found". ExistsPath(pathname string) (bool, error) <ins> // EvalHostSymlinks returns the path name after evaluating symlinks. // Will operate in the host mount namespace if kubelet is running in a container. EvalHostSymlinks(pathname string) (string, error) </ins> // CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given // pod volume directory. CleanSubPaths(podDir string, volumeName string) error

/kind bug Using , if a block device is symlinked to the discovery directory e.g. , then MapVolume will fail with since the containerized kubelet cannot see the path on host. is mounted by containerized kubelet, so if before we try to map the device we set devicePath to the evaluation of symlink and end up with a devicePath of , containerized kubelet will see the device and successfully map the device. Are there any issues with this solution? This is on Kubernetes 1.10
/sig storage
ah yes containerized kubelet has many issues when dealing with symlinks. Check out Mounter.GetMode(), it calls a nsenter EvalSymlinks which should be able to do what you need. But actually, one thing to consider, is do you need to do the losetup on the host, or does inside the container work?
Yes it appears to work from inside the container, on the host I run which disappears after I delete the pod using the device. My understanding of running losetup in MapVolume is not that great but I suppose that as long as that /dev/loop0 is created we are fine. Thanks for the quick response, I'll create a PR using that function and tag you
Does it create it on the host or in the containerized kubelet? What happens when containerized kubelet is restarted? The purpose of losetup is to do "refcounting" on the block device to prevent possible naming collisions when devices are attached/detached under unhealthy conditions. Ie, if a device showed up as /dev/sdb and you gave that to a pod, and then for some reason it become unhealthy and disappeared (while the pod is still running), another device (maybe even another volume type) could get attached to the node and reuse /dev/sdb. Now the pod has access to a device it's not supposed to. With filesystem volumes, the refcounting is done with mount points, but for block device, loopback is the equivalent.
It is created on the host (and also visible in the container), and stays around through restart. I understand now, thank you.
Actually after I restart it becomes /vdb instead of /dev/vdb in the losetup output, idk what that is about.


Question: // Will operate in the host mount namespace if kubelet is running in a container. // Error is returned on any other error than "file not found". ExistsPath(pathname string) (bool, error) <ins> // EvalHostSymlinks returns the path name after evaluating symlinks. // Will operate in the host mount namespace if kubelet is running in a container. EvalHostSymlinks(pathname string) (string, error) </ins> // CleanSubPaths removes any bind-mounts created by PrepareSafeSubpath in given // pod volume directory. CleanSubPaths(podDir string, volumeName string) error

There is no such thing as /vdb and /dev/vdb is still intact of course. Seems inconsequential since it is disappearing after all pods using the device are deleted, i.e. it's still serving as a refcounter


Question: func GetNodeAddresses(node *v1.Node, addressType v1.NodeAddressType) (ips []string) { for j := range node.Status.Addresses { nodeAddress := &node.Status.Addresses[j] <del> if nodeAddress.Type == addressType { </del> <ins> if nodeAddress.Type == addressType && nodeAddress.Address != "" { </ins> ips = append(ips, nodeAddress.Address) } }

Public IPs are expensive resources. Given that we're running scalability tests on 5k-node clusters, we are unnecessarily consuming 5k IPs which is a lot. We should switch our tests so that nodes have only private IP addresses. We are going to do that by configuring NAT so that nodes can talk to outside world (e.g. to download image from dockerhub). But that also requires changing some correctness tests which assume public IPs.
To summarize recent work: setting up private gce clusters with nat, feature is gated via the env test and dashboard with correctness tests on gce private cluster - https://k8s- 27 tests fail on fixing the tests - should fix majority of them
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale /lifecycle frozen
- is there anything missing to be done here?
No, we can close this one. It was done long time ago. Thanks for catching /close
Closing this issue. <detailsIn response to : No, we can close this one. It was done long time ago. Thanks for catching Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: // SortControllerRevisions sorts revisions by their Revision. func SortControllerRevisions(revisions []*apps.ControllerRevision) { <del> sort.Sort(byRevision(revisions)) </del> <ins> sort.Stable(byRevision(revisions)) </ins> } // EqualRevision returns true if lhs and rhs are either both nil, or both point to non-nil ControllerRevisions that

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: Statefulsets are sometimes continually rolling pods and I don't know why statefulset created: storageclass referenced: kubectl describe statefulset statefulset-test: As you can see from the above describe, the pod in the statefulset has been restarted a lot of times. Observing the created pod, 1) The pod comes up ready 2) immediately transitions to terminating 3) Is terminated 4) A new pod is created 5) Go back to (1) (repeats forever) Despite no updates to the statefulset spec ocurring. Making a change to the statefulset template (for example a trivial change adding a nonsense annotation) stops the rolling and fixes the issue When this statefulset was created, there was a mutating webhook enabled. After the observed pod rolling. I disabled this by making its configuration invalid and making it non required, and the rolling persisted. The status of the statefulset: : You can see that there is a discrepency between currentRevision and updateRevision. currentRevision: statefulset-test- updateRevision: statefulset-test- A short while later the status is changed to contain currentRevision: statefulset-test- updateRevision: statefulset-test- This change happened at Sun Apr 1 19:48:48 2018 Presumably after the rollout of statefulset-test- is complete.


Question: // SortControllerRevisions sorts revisions by their Revision. func SortControllerRevisions(revisions []*apps.ControllerRevision) { <del> sort.Sort(byRevision(revisions)) </del> <ins> sort.Stable(byRevision(revisions)) </ins> } // EqualRevision returns true if lhs and rhs are either both nil, or both point to non-nil ControllerRevisions that

Here are all the controllerrevisions from above: kubectl get controllerrevision statefulset-test- -o yaml kubectl get controllerrevision statefulset-test- -o yaml: kubectl get controllerrevision statefulset-test- -o yaml: The differences between the above controllerrevisions are insignificant (the hash, the creation timestamp, the name, the resource version, the uid, the self link) Interestingly all these controllerrevisions were created very close together in time shortly after I created the statefulset The current controllerrevisions are After watching the state change for a while, it actually seems like the statefulset is just cycling through the updateRevision being one of the above 10 controllerrevisions Leaders during this time From kube-controller-manager logs during this time, here is one excerpt: The error sync in there is suspicous, and as you can see the controllerrevisions are changing in the logs I have the full logs from a 5 minute window on either side of the above if necessary. The reason I'm inclined to think this is a kubernetes bug is that any update to the statefulset template fixes the problem. How to reproduce it (as minimally and precisely as possible): Not really sure. Worth just trying to create the above statefulset (requires creating the I tried deleting and recreating the above statefulset and it didn't happen, so something flaky is going on Anything else we need to know?: kubernetes 1.9.3 (client and server) cloud provider: aws
/sig apps
I realize this will probably need some further debugging, happy to get some direction
Consistent repro: Create a statefulset that creates a controllerrevision whose hash is between max int32 and max uint32 (this shouldn't be that hard to do, possibly a 50/50 chance of this). Manually create a controllerrevision identical to the first (same revision), incrementing both the name and hash by one. See that the statefulset controller will continually switch which controller revision should be the target of the statefulset.
Due to the issue fixed here: , under some circumstances controllerrevisions whose revision is the same were being considered equal.


Question: // SortControllerRevisions sorts revisions by their Revision. func SortControllerRevisions(revisions []*apps.ControllerRevision) { <del> sort.Sort(byRevision(revisions)) </del> <ins> sort.Stable(byRevision(revisions)) </ins> } // EqualRevision returns true if lhs and rhs are either both nil, or both point to non-nil ControllerRevisions that

This ends up causing this bug to manifest as follows: ListRevisions finds many controllerrevisions with the same revision (unclear how this state happens, but it can happen) The sort here randomly sorts them Due to the linked bug , FindEqualRevisions incorrectly finds that all these controller revisions are equal to the proposed udpate. The updateRevision selects the last revision found here Therefore, updateRevision, is randomly selected and causes the bug. This fixes the immediate issue, but the sort should probably be made stable (i.e not just look at the revision number, perhaps just consider the hash for tie breakers)
As an update, fixing the equality check will not be sufficient because it is possible for multiple controller revisions to be created that are actually equal, including the same hash, that have the same revision number. I believe the fix will need to be equal controllerrevisions with the same revision number from being created sort the controllerrevisions so that the controllerrevision that is the target of the statefulset is stable
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
ping I use the K8S which version is 1.11.4 and encounter the same problem. I want to reproduce the problem and find your way: Manually create a controllerrevision identical to the first (same revision), incrementing both the name and hash by one. Could you tell me how to mannually create a same controllerrevision identical to the first? Thanks in anvance.


Question: embed = [":go_default_library"], deps = [ "//pkg/kubelet/cri/remote/fake:go_default_library", <ins> "//pkg/kubelet/cri/remote/util:go_default_library", </ins> "//staging/src/k8s.io/cri-api/pkg/apis:go_default_library", "//staging/src/k8s.io/cri-api/pkg/apis/testing:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library",

Which jobs are flaking: periodics-kubernetes-bazel-test Which test(s) are flaking: //pkg/kubelet/cri/remote TestVersion Testgrid link: Reason for failure: Error: Received unexpected error: rpc error: code = Unavailable desc = transport is closing panic: runtime error: invalid memory address or nil pointer dereference [recovered] panic: runtime error: invalid memory address or nil pointer dereference Anything else we need to know: To reproduce with stress: go test -race -c ./pkg/kubelet/cri/remote stress - TestVersion Detailed logs
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig testing
I 'll give it a see and fix it :)
/assign
/remove-sig testing SIG node owns their unit tests. SIG testing owns the framework, infrastructure, etc.
/sig node


Question: cmd.SetUsageFunc(func(cmd *cobra.Command) error { fmt.Fprintf(cmd.OutOrStderr(), usageFmt, cmd.UseLine()) cliflag.PrintSections(cmd.OutOrStderr(), namedFlagSets, cols) <ins> cliflag.PrintSections(cmd.OutOrStderr(), additionalFlags, cols) </ins> return nil }) cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) { fmt.Fprintf(cmd.OutOrStdout(), "%snn"+usageFmt, cmd.Long, cmd.UseLine()) cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols) <ins> cliflag.PrintSections(cmd.OutOrStdout(), additionalFlags, cols) </ins> }) return cmd

The function in accepts an parameter which it describes as a way to "controller specific flags to be included in the complete set of controller manager flags". However, when the usage string is shown (either via or when parsing options fails), these additional options are not included in the usage string. This was identified via code inspection while trying to figure out what options did in . For e.g. , the output of should include a section corresponding to . This section should in turn list all of . a cloud provider executable that includes additional options, such as the additional options are not shown in the output No response_ n/a (all cloud providers are affected) n/a n/a n/a n/a
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig cloud-provider


Question: return p.Attach.Attach("POST", req.URL(), p.Config, p.In, p.Out, p.Err, t.Raw, sizeQueue) } <ins> if !p.Quiet && stderr != nil { fmt.Fprintln(stderr, "If you don't see a command prompt, try pressing enter.") } </ins> if err := t.Safe(fn); err != nil { return err }

First, the formatting. Second, "init-ctr-demo_demos" is an internal artifact, not fit for human-facing errors. Third, the container DOES exist, it's just in a terminating state.
The formatting issue should be fixed by . Regarding to the pod name, I think is better. As for the pod name, the error message is from the HTTP response of kubelet server. I've looked at the code, functions defined in interface are using ($podName_podNamespace) to identify a pod. Without spiting the parameter into two separated parameters, it could be tricky to change the output format of an error message. Personally I would prefer to change the signatures, but I don't know if we are planing to do it. In order to improve the readability, I a client-side check for pod status. See .


Question: deps = [ "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/container/testing:go_default_library", <ins> "//pkg/kubelet/metrics:go_default_library", </ins> "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", <ins> "//vendor/github.com/prometheus/client_model/go:go_default_library", </ins> "//vendor/github.com/stretchr/testify/assert:go_default_library", ], )

Some kubelet metrics use non-standard prometheus collectors unnecessarily. Specifically, we have which can be simple gauges. Since we cannot migrate non-standard prometheus collectors using the internal metrics stability framework, we will need to convert these metrics into standard prometheus variants. /sig instrumentation /assign
GitHub didn't allow me to assign the following users: solodov. Note that only and repo collaborators can be assigned and that issues/PRs can only have 10 assignees at the same time. For more information please see <detailsIn response to : Some kubelet metrics use non-standard prometheus collectors unnecessarily. Specifically, we have which can be simple gauges. Since we cannot migrate non-standard prometheus collectors using the internal metrics stability framework, we will need to convert these metrics into standard prometheus variants. /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
recommends that we generate those metrics in the GenericPLEG during the relist. We can generate the data directly from the pods call there ().
I would be interested to work on this issue if no one else is working on it currently
Go for it!
/assign


Question: "k8s.io/klog" ) <ins> var ( // ErrorNoAuth indicates that no credentials are provided. ErrorNoAuth = fmt.Errorf("no credentials provided for Azure cloud provider") ) </ins> // AzureAuthConfig holds auth related part of cloud config type AzureAuthConfig struct { // The cloud environment identifier. Takes values from https://github.com/Azure/go-autorest/blob/ec5f4903f77ed9927ac95b19ab8e44ada64c1356/autorest/azure/environments.go#L13

Currently there is no way to run without any identity (be it service principal or E/MSI). In reality code running on a worker node can use via - and only via - without the need to access ARM. Here is the proposed change a new mode authN = this mode, is basically telling the cloud provider to work any identity, this means only IMDS calls is allowed. This mode can be detected from if no SPN set and both MSI settings are set to . to use a config map within the cluster. This config map can be loaded on the cluster statically. or created - if not existent - in that then only is used in The net new result of the above is users bootstrapping clusters can configure the worker node to use without any identity on said worker nodes.
/sig azure These SIGs are my best guesses for this issue. Please comment if I am incorrect about one. I am a bot run by
In-tree AzureDisk volume plugin is also depending on Azure credential to fetch lun id. we should figure out how to solve it without credential
fetch disk lun id is also a dependency in kubelet code now, after my investigation, it could be removed on Linux node, while it cannot be removed on Windows node, need to change the k/k upstream code, there is a discussion on going here: , let' wait for the discussion result.
For Linux: 1- use IMDS my understanding is new api versions will have the disk/lun pairs. 2- don't use arm for the return of is the second parameter passed to . For Windows: I don't understand the dep on ARM for this. The above still apply, we just need to find a way to encode the value as ut return from that allow us to find the disk by on a windows machine. might be a good source for this. Ideally speaking. The value should be the same. But i will leave it for you to decide.
could be a good idea since currently it's a LUN num returned by Attach() and then it could be changed to the windows disk num, I will make the code change on v1.15 later, thanks for discussion.
Update: PR would remove the VM API calls when getting the LUN number for AzureDisk.


Question: Containers: []v1.Container{ { Image: image.GetE2EImage(), <del> Name:  podName, </del> <ins> Name:  "stat-container", </ins> Command: []string{ "powershell.exe", "-Command",

<!-- Please only use this template for submitting reports about continuously failing tests or jobs in Kubernetes CI --Which jobs are failing: aks-engine-azure-windows-master-containerd-serial-slow Which test(s) are failing: [sig-windows] [Feature:Windows] Kubelet-Stats [Serial] Kubelet stats collection for Windows nodes when running 10 pods should return within 10 seconds Since when has it been failing: Unknown. Testgrid link: Reason for failure: The test spawns 10 pods with the same pod name, which contains multiple containers with the same container name. Because of this, the test fails. Logs: Anything else we need to know:
/sig windows These SIGs are my best guesses for this issue. Please comment if I am incorrect about one. I am a bot run by vllry.


Question: GetProtocol() v1.Protocol // GetHealthCheckNodePort returns service health check node port if present.  If return 0, it means not present. GetHealthCheckNodePort() int <ins> // GetNodePort returns a service Node port if present. If return 0, it means not present. GetNodePort() int </ins> } // Endpoint in an interface which abstracts information about an endpoint.

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: BUG REPORT What happened: When deleting/recreating pods backing a UDP service, traffic from external senders stops flowing. What you expected to happen: Traffic would continue to flow to new pods. How to reproduce it (as minimally and precisely as possible): Based on reading of similar issues and commit messages it seems like this would be most noticable on high-volume UDP applications, statsd and syslog are good examples. a deployment and service for a UDP app. In my test case, this would be the container. an external node, create a small test script utilizing a statsd library such as pystatsd. The most important part is that the sender uses the same source-port for every packet sent. Ex: the sender. Install tcpdump into the pod and watch for incoming packets, example from the test case: the pod tcpdump in the newly created pod. Observe that no new matching packets are flowing. existing conntrack rules on all nodes in the cluster. You should see traffic start flowing to the pod again. example command for clearing conntrack rules on the nodes: Anything else we need to know?: I have tested this only on GKE-managed clusters that utilize google forwarding-rules for external LBs, but I believe the issue may be present on any platform where iptables mode for kube-proxy is used. Tested and verified on 1.8.6-gke.0 and 1.7.11-gke.1. Also tested with both Google Container-optimized OS and the Ubuntu-based node images. I have only test this on GKE It appears this was a known issue and should have been cleared up by: From my reading of , it seems as if the func should exec() to clear up UDP entries on service endpoint change, but the only place I see that func called is from syncProxyRules(). That func looks complex and possibly like it is only run occasionally and not on service/endpoint changes. Environment: Kubernetes version (use ): (Also tested on 1.7.11-gke.1) Cloud provider or hardware configuration: GKE-managed clusters, google cloud. OS (e.g.


Question: GetProtocol() v1.Protocol // GetHealthCheckNodePort returns service health check node port if present.  If return 0, it means not present. GetHealthCheckNodePort() int <ins> // GetNodePort returns a service Node port if present. If return 0, it means not present. GetNodePort() int </ins> } // Endpoint in an interface which abstracts information about an endpoint.

from /etc/os-release): tried both: and Kernel (e.g. ): and Install tools: Others:
/sig network
We also ran into this in kubernetes v1.7.6 using OpenShift 3.7:
~I believe this is the same as ~If so, it is fixed in 1.10: These are about HostPort, not NodePort, sorry for the noise.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Is there any other known workaround for this besides updating conntrack rules on the kube nodes? I tried deleting and recreating the service, but this didn't update the rules. When I restart the stream on the client side data flows correctly again (perhaps it's perceived as a new connection), but I have to restart the stream for each sender.
/remove-lifecycle stale
FWIW, this is not fixed in 1.10 - that fix was for HostPort, and this is still a bug for NodePort.
I am seeing this issue in 1.11. I am using LoadBalancer type.
has been working on a fix for this, FYI.
This issue is extremely impactful for telephony protocols that rely on static source ip and source port. Any update on a possible workaround? I was thinking a potential workaround might be to reduce the UDP timeout to something really short so the stale conntrack entries go away faster. Or maybe there is a way to flush conntrack on the event of pod deletion? Edit: Removed stupid idea of a workaround to disable conntrack for UDP. Can't disable conntrack for UDP because it's used for DNAT from node to container.
The workaround is to not use NodePorts. Other kinds of services are already correct.
not sure what you mean. I am seeing the issue with a LoadBalancer type so it does not affect NodePorts only. As a workaround I have created a small python script that watches the K8s API for pod deletions and clears the related conntrack entries when that happens. I can share it when I get to work on Monday if you want to try it out.


Question: GetProtocol() v1.Protocol // GetHealthCheckNodePort returns service health check node port if present.  If return 0, it means not present. GetHealthCheckNodePort() int <ins> // GetNodePort returns a service Node port if present. If return 0, it means not present. GetNodePort() int </ins> } // Endpoint in an interface which abstracts information about an endpoint.

I've my script . Note that it's not tested much so don't use it in a production environment. Also it cleans up all conntrack entries for a deleted pod, not only the UDP related ones.
We are also seeing this issue with type loadbalancer. We are on 1.11. Our use-case is the following: multiple devices are sending to an aggreagator machine. This machine forwards the UDP packages to a loadBalancer in gcp. When debugging the service, I noticed the following: The traffic is assigned to one pod, and the packages are not loadbalanced (client and server are the same cfr use-case) When redeploying a new cluster, Conntrack is cleared on termination signal of the old pod. (client receives connection refused) red/black and highlander deployment strategies have the same effect. How would I expect this to be working then? New cluster is . Conntrack is cleared Data is routed to NEW pods old pods are terminated Best Regards, Tim
I am seeing this exact same issue on our cluster which has k8s v1.14.5 . We have endpoint nginx pods backing a UDP service with nodeport. Suddenly one day we see traffic not hitting into nginx pod but reaching the node hosting the nginx pod. We could simulate the problem by restarting the nginx pods. I am assuming with this issue marked as closed; and having made its way into v1.14; we should not be seeing this. Would also like to point out; the one thing, in addition we have in the service is ExternalTrafficPolicy set to Local. Can anyone please guide/help on why we are seeing this in v.1.14.5 of our cluster.
Seeing this with k8s v1.16.5


Question: timedout := make(chan bool) go func() { time.Sleep(gracefulWait) <del> timedout <- true </del> <ins> close(timedout) </ins> }() go func() { select {

It's at line 111 When the server stopped before timeout, the first goroutine will be block on because the receiver in the second goroutine is not exist. Changing to a buffered one may fix it. no goroutine leaks run related test with leak checker No response <detailsdev </details<detailsunknown </details<details</details<details</details<details</details<details</details
/sig api-machinery
/assign /triage accepted
/cc
It should close the channel rather than write to it, that's the easiest fix


Question: import ( "fmt" "os" <ins> "path/filepath" </ins> "strings" "github.com/golang/glog"

<!-- Thanks for filing an issue! Before hitting the button, please answer these questions.--Is this a request for help? No What keywords did you search in Kubernetes issues before filing this one? static manifest Is this a BUG REPORT or FEATURE REQUEST? (choose one): Bug report Kubernetes version (use ): Environment: Cloud provider or hardware configuration: Bare metal and reproduced in Vagrant OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: custom scratch install and reproduced with kubeadm Others: Calico for networking. Used (Also able to reproduce without calico) What happened: Modified one of the static manifest files in . The kubelet killed the running pod and did NOT create a new pod. (waited 30 minutes, still no pod) What you expected to happen: Kubelet should have killed the pod and started a new one. How to reproduce it (as minimally and precisely as possible): Started with single node, vanilla Ubuntu Xenial, installed kubernetes via kubeadm, nothing special. Modify , for example change the logging level or even just whitespace at the end of the file. Observe the kubelet kill the controller manager pod, and does not restart it. Anything else we need to know: If I remove the file from the directory, and then replace the file, kubelet will start it up. Kubelet logs from the period where I updated the file:
Did you modify the yaml file using an editor (e.g., vim)? If the editor creates a temporary file in the same directory, kubelet can get confused and thinking that's a new pod (with the same name & namespace). When the temporary file gets removed, kubelet'd inadvertently delete the pod from its cache. Since kubelet relies on inotify for changes, if it doesn't get notified for any new changes from the original yaml file, it'd forever ignore the pod. There is an issue open to add periodic listing in addition to inotify (introduced in 1.5). This would make sure kubelet eventually recover from the situation. I can't find the issue, but there is another one for inotify: . Another fix is to make sure kubelet ignore such known temporary files (it already ignores files starting with ). In general, don't edit the file directly.


Question: import ( "fmt" "os" <ins> "path/filepath" </ins> "strings" "github.com/golang/glog"

Copy the file to another place, edit, and copy it back to the manifests directory.
I'm seeing a similar problem except I never edited the file, might be fit for a separate issue
Yes I was using vim, and this explanation makes sense. Would be nice if it was smart about it. Would a PR to ignore the vim temp file be acceptable
I think it's reasonable. I doubt if anyone uses those file names on purpose.
The dot ignore should have covered the vim temp file too, but it doesn't work because the filename passed in is the absolute filename including the path. So the first character is not a dot. I'll try to submit a PR fix.
PR submitted
fixed?
How to work around? I coudn't start api server when using kubeadm to deploy k8s. Thanks!


Question: }) It("verify monitoring pods and all cluster nodes are available on influxdb using heapster.", func() { <del> if testContext.Provider != "gce" { </del> <ins> if !providerIs("gce") { </ins> By(fmt.Sprintf("Skipping Monitoring test, which is only supported for provider gce (not %s)", testContext.Provider)) return

I noticed while running e2e tests locally that some tests were apparently being incorrectly skipped: I checked, and this is also true for our Jenkins runs. Currently it looks like and have this bug, though only the former actually skips the test, I think. The issue appears to be that the code inside the top-level in these files is being run before the testContext struct is filled out (and before is even called), and so testContext.Provider is empty. I'm not immediately sure what the proper fix is; I'm also worried that there might be other things broken by the fact that these blocks are called before initialization is complete. We also probably investigate if there is a better way to mark these tests as skipped; right now, there's no indication in the test report (, Jenkins test report, etc) that these tests were skipped.
I merged and which tried to disable these tests for non-{gke,gce} but apparently didn't follow-up and verify that they were still running on our platforms. /cc
Seems easy enough to fix. I'll have a PR out to fix this before Monday.