Question: } } <ins> func TestPIDPressure_VerifyPodStatus(t *testing.T) { testCases := map[string]struct { wantPodStatus v1.PodStatus }{ "eviction due to pid pressure": { wantPodStatus: v1.PodStatus{ Phase:   v1.PodFailed, Reason:  "Evicted", Message: "The node was low on resource: pids. Threshold quantity: 1200, available: 500. ", }, }, } for name, tc := range testCases { for _, enablePodDisruptionConditions := range []bool{true, false} { t.Run(fmt.Sprintf("%s;PodDisruptionConditions=%v", name, enablePodDisruptionConditions), func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodDisruptionConditions, enablePodDisruptionConditions)() podMaker := makePodWithPIDStats summaryStatsMaker := makePIDStats podsToMake := []podToMake{ {name: "pod1", priority: lowPriority, pidUsage: 500}, {name: "pod2", priority: defaultPriority, pidUsage: 500}, } pods := []*v1.Pod{} podStats := map[*v1.Pod]statsapi.PodStats{} for _, podToMake := range podsToMake { pod, podStat := podMaker(podToMake.name, podToMake.priority, 2) pods = append(pods, pod) podStats[pod] = podStat } activePodsFunc := func() []*v1.Pod { return pods } fakeClock := testingclock.NewFakeClock(time.Now()) podKiller := &mockPodKiller{} diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: ptr.To(false)} diskGC := &mockDiskGC{err: nil} nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""} config := Config{ PressureTransitionPeriod: time.Minute * 5, Thresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalPIDAvailable, Operator: evictionapi.OpLessThan, Value: evictionapi.ThresholdValue{ Quantity: quantityMustParse("1200"), }, }, }, } summaryProvider := &fakeSummaryProvider{result: summaryStatsMaker("1500", "1000", podStats)} manager := &managerImpl{ clock:                        fakeClock, killPodFunc:                  podKiller.killPodNow, imageGC:                      diskGC, containerGC:                  diskGC, config:                       config, recorder:                     &record.FakeRecorder{}, summaryProvider:              summaryProvider, nodeRef:                      nodeRef, nodeConditionsLastObservedAt: nodeConditionsObservedAt{}, thresholdsFirstObservedAt:    thresholdsObservedAt{}, } // synchronize to detect the PID pressure _, err := manager.synchronize(diskInfoProvider, activePodsFunc) if err != nil { t.Fatalf("Manager expects no error but got %v", err) } // verify PID pressure is detected if !manager.IsUnderPIDPressure() { t.Fatalf("Manager should have detected PID pressure") } // verify a pod is selected for eviction if podKiller.pod == nil { t.Fatalf("Manager should have selected a pod for eviction") } wantPodStatus := tc.wantPodStatus.DeepCopy() if enablePodDisruptionConditions { wantPodStatus.Conditions = append(wantPodStatus.Conditions, v1.PodCondition{ Type:    "DisruptionTarget", Status:  "True", Reason:  "TerminationByKubelet", Message: "The node was low on resource: pids. Threshold quantity: 1200, available: 500. ", }) } // verify the pod status after applying the status update function podKiller.statusFn(&podKiller.pod.Status) if diff := cmp.Diff(*wantPodStatus, podKiller.pod.Status, cmpopts.IgnoreFields(v1.PodCondition{}, "LastProbeTime", "LastTransitionTime")); diff != "" { t.Errorf("Unexpected pod status of the evicted pod (-want,+got):n%s", diff) } }) } } } </ins> func TestDiskPressureNodeFs_VerifyPodStatus(t *testing.T) { testCases := map[string]struct { nodeFsStats                   string

We should add a unit test to Eviction manager that accounts for PID Pressure. The test should cover similar cases as DiskPressure or MemoryPressure. 1) PIDPressure doesn't exist on node. 2) Force PidPressure on node We should include test cases for reclaming resources in terms of pressure and verifying differences between soft and hard eviction.
/sig node /kind feature /help
This request has been marked as needing help from a contributor. Please ensure that the issue body includes answers to the following questions: Why are we solving this issue? To address this issue, are there any code changes? If there are code changes, what needs to be done in the code and what places can the assignee treat as reference points? Does this issue have zero to low barrier of entry? How can the assignee reach out to you for help? For more details on the requirements of such an issue, please see and ensure that they are met. If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : /sig node /kind feature /help </details
/triage accepted /priority important-longterm
/good-first-issue
This request has been marked as suitable for new contributors. Please ensure that the issue body includes answers to the following questions: Why are we solving this issue? To address this issue, are there any code changes? If there are code changes, what needs to be done in the code and what places can the assignee treat as reference points? Does this issue have zero to low barrier of entry? How can the assignee reach out to you for help? For more details on the requirements of such an issue, please see and ensure that they are met. If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : /good-first-issue Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign
it looks like has requested to work on this.
/assign
Hi What do you mean by, "verifying differences between soft and hard eviction."?
It’s the same idea as what we do for disk, memory or inode pressure.


Question: } } <ins> func TestPIDPressure_VerifyPodStatus(t *testing.T) { testCases := map[string]struct { wantPodStatus v1.PodStatus }{ "eviction due to pid pressure": { wantPodStatus: v1.PodStatus{ Phase:   v1.PodFailed, Reason:  "Evicted", Message: "The node was low on resource: pids. Threshold quantity: 1200, available: 500. ", }, }, } for name, tc := range testCases { for _, enablePodDisruptionConditions := range []bool{true, false} { t.Run(fmt.Sprintf("%s;PodDisruptionConditions=%v", name, enablePodDisruptionConditions), func(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodDisruptionConditions, enablePodDisruptionConditions)() podMaker := makePodWithPIDStats summaryStatsMaker := makePIDStats podsToMake := []podToMake{ {name: "pod1", priority: lowPriority, pidUsage: 500}, {name: "pod2", priority: defaultPriority, pidUsage: 500}, } pods := []*v1.Pod{} podStats := map[*v1.Pod]statsapi.PodStats{} for _, podToMake := range podsToMake { pod, podStat := podMaker(podToMake.name, podToMake.priority, 2) pods = append(pods, pod) podStats[pod] = podStat } activePodsFunc := func() []*v1.Pod { return pods } fakeClock := testingclock.NewFakeClock(time.Now()) podKiller := &mockPodKiller{} diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: ptr.To(false)} diskGC := &mockDiskGC{err: nil} nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""} config := Config{ PressureTransitionPeriod: time.Minute * 5, Thresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalPIDAvailable, Operator: evictionapi.OpLessThan, Value: evictionapi.ThresholdValue{ Quantity: quantityMustParse("1200"), }, }, }, } summaryProvider := &fakeSummaryProvider{result: summaryStatsMaker("1500", "1000", podStats)} manager := &managerImpl{ clock:                        fakeClock, killPodFunc:                  podKiller.killPodNow, imageGC:                      diskGC, containerGC:                  diskGC, config:                       config, recorder:                     &record.FakeRecorder{}, summaryProvider:              summaryProvider, nodeRef:                      nodeRef, nodeConditionsLastObservedAt: nodeConditionsObservedAt{}, thresholdsFirstObservedAt:    thresholdsObservedAt{}, } // synchronize to detect the PID pressure _, err := manager.synchronize(diskInfoProvider, activePodsFunc) if err != nil { t.Fatalf("Manager expects no error but got %v", err) } // verify PID pressure is detected if !manager.IsUnderPIDPressure() { t.Fatalf("Manager should have detected PID pressure") } // verify a pod is selected for eviction if podKiller.pod == nil { t.Fatalf("Manager should have selected a pod for eviction") } wantPodStatus := tc.wantPodStatus.DeepCopy() if enablePodDisruptionConditions { wantPodStatus.Conditions = append(wantPodStatus.Conditions, v1.PodCondition{ Type:    "DisruptionTarget", Status:  "True", Reason:  "TerminationByKubelet", Message: "The node was low on resource: pids. Threshold quantity: 1200, available: 500. ", }) } // verify the pod status after applying the status update function podKiller.statusFn(&podKiller.pod.Status) if diff := cmp.Diff(*wantPodStatus, podKiller.pod.Status, cmpopts.IgnoreFields(v1.PodCondition{}, "LastProbeTime", "LastTransitionTime")); diff != "" { t.Errorf("Unexpected pod status of the evicted pod (-want,+got):n%s", diff) } }) } } } </ins> func TestDiskPressureNodeFs_VerifyPodStatus(t *testing.T) { testCases := map[string]struct { nodeFsStats                   string

Hard eviction means we trigger eviction with no grace period while soft eviction means that we obey a grace period. Test should check for that.
I have opened a PR based on what I could understand using a debugger and the other tests. This is my first time contributing to the codebase. Happy to take brutal feedback.
Hi, i would like to support this issue. Let me know if any help is required.
/assign


Question: "k8s.io/apimachinery/pkg/labels" "k8s.io/apiserver/pkg/admission" genericadmissioninitializers "k8s.io/apiserver/pkg/admission/initializer" <del> utilfeature "k8s.io/apiserver/pkg/util/feature" </del> "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" schedulingv1listers "k8s.io/client-go/listers/scheduling/v1" <ins> "k8s.io/component-base/featuregate" </ins> "k8s.io/kubernetes/pkg/apis/core" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/scheduling"

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Following the documentation at the , I tried enabling quota for a namespace other than but I was still not able to create a pod with priorityClass in my namespace with following error: Error from server (Forbidden): error when creating "": pods "memory-demo" is forbidden: pods with system-cluster-critical priorityClass is not permitted in my-namespace namespace What you expected to happen: Expected pod to be created with namespace where I allowed quota. I can see that both ResourceQuota and Priority Plugins are enabled from logs. I0408 17:15:08. ] Loaded 8 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,MutatingAdmissionWebhook. I0408 17:15:08. ] Loaded 6 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,ValidatingAdmissionWebhook,ResourceQuota. How to reproduce it (as minimally and precisely as possible): Follow the steps at Anything else we need to know?: I think, the issue is with priorityPlugin where we are rejecting instead of letting resourceQuota plugin to handle the admission when is enabled. Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Others:
/sig scheduling /cc
Same happens to me. Correction, it does not allow to use the default permissions outside of kube-system
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
See also:
/remove-lifecycle stale
/assign Per offline discussion with
Same issue here.


Question: "k8s.io/apimachinery/pkg/labels" "k8s.io/apiserver/pkg/admission" genericadmissioninitializers "k8s.io/apiserver/pkg/admission/initializer" <del> utilfeature "k8s.io/apiserver/pkg/util/feature" </del> "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" schedulingv1listers "k8s.io/client-go/listers/scheduling/v1" <ins> "k8s.io/component-base/featuregate" </ins> "k8s.io/kubernetes/pkg/apis/core" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/scheduling"

We are using namespaces other than kube-system for system pods as it is getting too crowded, therefore we would like to run cluster-node-critical pods in other namespaces.
Waiting for this to be available myself. No reason my Node Exporter dataset has to run in the namespace instead of my namespace.


Question: func (nc *Controller) doNoScheduleTaintingPass(node *v1.Node) error { // Map node's condition to Taints. <del> taints := []v1.Taint{} </del> <ins> var taints []v1.Taint </ins> for _, condition := range node.Status.Conditions { <del> if _, found := nodeConditionToTaintKeyMap[condition.Type]; found { if condition.Status == v1.ConditionTrue { </del> <ins> if taint, found := nodeConditionToTaintKeyStatusMap[condition.Type]; found { if condition.Status == taint.noScheduleStatus { </ins> taints = append(taints, v1.Taint{ <del> Key:    nodeConditionToTaintKeyMap[condition.Type], </del> <ins> Key:    taint.taintKey, </ins> Effect: v1.TaintEffectNoSchedule, }) }

Is this a BUG REPORT or FEATURE REQUEST?: What happened: admission control plugin in apiserver. a deployment with 1 replica. The pod is successfully launched. By default the admission control plugin will add following toleration in the pod. the node where the above pod is running. of image locality policy and above toleration in the pod, scheduler still choose the node to launch the pod. The pod is in state and failed to create. What you expected to happen: The pod should be scheduled to healthy node and created successfully. How to reproduce it (as minimally and precisely as possible): See above steps. Anything else we need to know?: Environment: Kubernetes version (use ): v1.10 Cloud provider or hardware configuration: openstack OS (e.g. from /etc/os-release): Ubuntu 16.04 Kernel (e.g. ): 4.4.0-121-generic Install tools: N/A Others:
/sig scheduler
/sig scheduling /sig node
I think that we should avoid scheduling pods to nodes which is not ready even it can tolerate and , comments
As the pod already claimed to tolerate NotReady, this is an expected behavior. P.S. If the eviction dose not happen after 5m, that should be the bug. Please reopen the issue and modify the description. This is what tolerations is designed for :) And there are many use cases for pod to be scheduled to NotReady nodes (mostly during cluster deployment).
I think maybe you can configure effect to avoid scheduling to those nodes.
scheduler still choose the NotReady node to launch the pod For running, the toleration is used for graceful termination by ; if is enabled, it's better to also taint node with for .
Wondering how the pod can start to run if the Node is not ready at deployment time? Can you specify a use case here?
It won't need to start, system pod pending will be expected :) The point is, freedom for users to decide what tolerations they want matters here. I don't think it's reasonable to combine with in this case.
Perhaps, add to default toleration of admission control plugin could eliminate the confusion.
We should align the behavior when enabled or disabled : if is disabled, will not schedule pod to not-ready node. if is enabled, we should also taint node with .


Question: func (nc *Controller) doNoScheduleTaintingPass(node *v1.Node) error { // Map node's condition to Taints. <del> taints := []v1.Taint{} </del> <ins> var taints []v1.Taint </ins> for _, condition := range node.Status.Conditions { <del> if _, found := nodeConditionToTaintKeyMap[condition.Type]; found { if condition.Status == v1.ConditionTrue { </del> <ins> if taint, found := nodeConditionToTaintKeyStatusMap[condition.Type]; found { if condition.Status == taint.noScheduleStatus { </ins> taints = append(taints, v1.Taint{ <del> Key:    nodeConditionToTaintKeyMap[condition.Type], </del> <ins> Key:    taint.taintKey, </ins> Effect: v1.TaintEffectNoSchedule, }) }

with timeout is used for graceful termination of 'running pods' for feature. We should not re-schedule pod to not-ready unless tolerating it explicitly.
reopen this for more discussion :)
Let me extracted the mixed problems in this issue :-) First one: These two default tolerations attached to the pod by means: if the node is tainted as or , wait for 300s before the pod is evicted from the node. While according to the issue description, is working as expected (pods are not evicted immediately). This part should be closed. Second one: The discussion raised is actually about taint during eviction: if TaintNodesByCondition is enabled, we should also taint node with NotReady:NoSchedule. My thoughts is: it's reasonable to taint node with as well for consistency consideration (but I'm hesitate if only for eliminating confusion as pods are expected to pending on NotReady nodes) I think it should be clearer now.
I open a PR If TaintNodesByCondition is enable, taint node with NotReady:NoSchedule and Unreachable:NoSchedule.
, we did not catch 1.11 release, so this issue will be handled in 1.12 :)
in 1.9.7enable the TaintNodesByCondition and taint nodee.g. node01 with NotReady:NoSchedulethen pod also can not be scheduled to node01 even when node01 is healthy. What you expected to happen: The pod should be scheduled to node01 when it is healthy node and not be scheduled to node01 when it is not ready. Thanks
How did you do that? by kubectl? The fix is not ready until 1.12.
YesI tried "kubectl taint nodes node01 ",and also tried "kubectl taint nodes node01 "
this feature did not handle taint manually
Ok, thanks a lot! So I can not solve the problem manually now and what I should do is just waiting for the version 1.12, right?


Question: // detach controller. Add it to desired state of world. adc.desiredStateOfWorld.AddNode(nodeName) } adc.processVolumesInUse(nodeName, node.Status.VolumesInUse) } <del> func (adc *attachDetachController) nodeUpdate(oldObj, newObj interface{}) { // The flow for update is the same as add. adc.nodeAdd(newObj) } </del> func (adc *attachDetachController) nodeDelete(obj interface{}) { node, ok := obj.(*v1.Node) if node == nil || !ok {

Is this a BUG REPORT or FEATURE REQUEST? BUG REPORT Environment: Cloud provider or hardware configuration: GCE OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others: What happened: $ kubectl describe pods test-ipj93 Name: test-ipj93 Namespace: default Node: kubernetes-minion-group-yxra/10.128.0.3 Start Time: Mon, 28 Nov 2016 14:39:28 -0800 Labels: <noneStatus: Pending IP: Controllers: <noneContainers: busybox: Container ID: Image: busybox Image ID: Port: Command: sh -c while true; do echo (hostname)-(cat /proc/stat cut -f 4 -d" "); sleep 30; done QoS Tier: cpu: Burstable memory: BestEffort Requests: cpu: 100m State: Waiting Reason: ContainerCreating Ready: False Restart Count: 0 Environment Variables: Conditions: Type Status Initialized True Ready False PodScheduled True Volumes: data: Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace) ClaimName: test ReadOnly: false default-token-kjoqg: Type: Secret (a volume populated by a Secret) SecretName: default-token-kjoqg Events: FirstSeen LastSeen Count From SubobjectPath Type Reason Message --------- -------- ----- ---- ------------- -------- ------ ------- 57m 57m 1 {default-scheduler } Normal Scheduled Successfully assigned test-ipj93 to kubernetes-minion-group-yxra 57m 57m 1 {kubelet kubernetes-minion-group-yxra} spec.containers{busybox} Normal Pulling pulling image "busybox" 57m 57m 1 {kubelet kubernetes-minion-group-yxra} spec.containers{busybox} Normal Pulled Successfully pulled image "busybox" 57m 57m 1 {kubelet kubernetes-minion-group-yxra} spec.containers{busybox} Normal Created Created container with docker id ; Security:[seccomp=unconfined] 57m 57m 1 {kubelet kubernetes-minion-group-yxra} spec.containers{busybox} Normal Started Started container with docker id 47m 47m 6 {kubelet kubernetes-minion-group-yxra} Warning FailedMount MountVolume.


Question: // detach controller. Add it to desired state of world. adc.desiredStateOfWorld.AddNode(nodeName) } adc.processVolumesInUse(nodeName, node.Status.VolumesInUse) } <del> func (adc *attachDetachController) nodeUpdate(oldObj, newObj interface{}) { // The flow for update is the same as add. adc.nodeAdd(newObj) } </del> func (adc *attachDetachController) nodeDelete(obj interface{}) { node, ok := obj.(*v1.Node) if node == nil || !ok {

SetUp failed for volume "" (spec.Name: "default-token-kjoqg") pod "-b5bb-11e6-bab0-" (UID: "-b5bb-11e6-bab0-") with: Get https://kubernetes-master/api/v1/namespaces/default/secrets/default-token-kjoqg: dial tcp: lookup kubernetes-master on 169.254.169.254:53: no such host 46m 46m 1 {kubelet kubernetes-minion-group-yxra} Warning FailedMount MountVolume.SetUp failed for volume "" (spec.Name: "default-token-kjoqg") pod "-b5bb-11e6-bab0-" (UID: "-b5bb-11e6-bab0-") with: Get https://kubernetes-master/api/v1/namespaces/default/secrets/default-token-kjoqg: dial tcp 10.128.0.2:443: i/o timeout 45m 45m 1 {kubelet kubernetes-minion-group-yxra} Warning FailedMount MountVolume.SetUp failed for volume "" (spec.Name: "default-token-kjoqg") pod "-b5bb-11e6-bab0-" (UID: "-b5bb-11e6-bab0-") with: Get https://kubernetes-master/api/v1/namespaces/default/secrets/default-token-kjoqg: dial tcp 10.128.0.2:443: getsockopt: connection refused 11m 11m 2 {controller-manager } Warning FailedMount Failed to attach volume "test" on node "kubernetes-minion-group-yxra" with: error getting instance "kubernetes-minion-group-yxra" 9m 9m 1 {kubelet kubernetes-minion-group-yxra} Warning FailedSync Error syncing pod, skipping: network is not ready: [Kubenet does not have netConfig. This is most likely due to lack of PodCIDR] 9m 9m 2 {kubelet kubernetes-minion-group-yxra} Warning FailedSync Error syncing pod, skipping: network is not ready: [Kubenet does not have netConfig.


Question: // detach controller. Add it to desired state of world. adc.desiredStateOfWorld.AddNode(nodeName) } adc.processVolumesInUse(nodeName, node.Status.VolumesInUse) } <del> func (adc *attachDetachController) nodeUpdate(oldObj, newObj interface{}) { // The flow for update is the same as add. adc.nodeAdd(newObj) } </del> func (adc *attachDetachController) nodeDelete(obj interface{}) { node, ok := obj.(*v1.Node) if node == nil || !ok {

This is most likely due to lack of PodCIDR] 6m 14s 4 {kubelet kubernetes-minion-group-yxra} Warning FailedMount Unable to mount volumes for pod "test-ipj93_default(-b5bb-11e6-bab0-)": timeout expired waiting for volumes to attach/mount for pod "test-ipj93"/"default". list of unattached/unmounted volumes=[data] 6m 14s 4 {kubelet kubernetes-minion-group-yxra} Warning FailedSync Error syncing pod, skipping: timeout expired waiting for volumes to attach/mount for pod "test-ipj93"/"default". list of unattached/unmounted volumes=[data] What you expected to happen: GCE PD to attach and mount successfully post the upgrade How to reproduce it (as minimally and precisely as possible): Running the upgrade steps in the doc for upgrade of manual test from 1.4 to 1.5 Anything else do we need to know:
Ran the upgrade again. This time the .
kube-controller-
Current theory is because kubelet deletes and recreates the node API object during upgrade (because externalID changed), see the new object does not contain the correct status (it is written by the Attach/Detach controller to the old object but not subseqently updated). If this proves to be the issue, the proposed fix is to force the attach/detach controller to update the node status when the node's external ID changes. is working to verify this theory by executing this code path (without a full upgrade). Once he has validated the theory, he will work on the fix, verify it, merge it to master, and get it cherry-picked to 1.5.
: is expected to fix this problem?
: Yes


Question: # Base images - name: "k8s.gcr.io/debian-base: dependents" <del> version: 2.1.3 </del> <ins> version: buster-v1.2.0 </ins> refPaths: - path: build/workspace.bzl match: tag = - path: cluster/images/etcd/Makefile <del> match: BASEIMAGE?=k8s.gcr.io/build-image/debian-base:vd+.d+.d+ </del> <ins> match: BASEIMAGE?=k8s.gcr.io/build-image/debian-base:[a-zA-Z]+-v((([0-9]+).([0-9]+).([0-9]+)(?:-([0-9a-zA-Z-]+(?:.[0-9a-zA-Z-]+)*))?)(?:+([0-9a-zA-Z-]+(?:.[0-9a-zA-Z-]+)*))?) </ins> - path: cluster/images/etcd/Makefile <del> match: BASEIMAGE?=k8s.gcr.io/build-image/debian-base-arm:vd+.d+.d+ </del> <ins> match: BASEIMAGE?=k8s.gcr.io/build-image/debian-base-arm:[a-zA-Z]+-v((([0-9]+).([0-9]+).([0-9]+)(?:-([0-9a-zA-Z-]+(?:.[0-9a-zA-Z-]+)*))?)(?:+([0-9a-zA-Z-]+(?:.[0-9a-zA-Z-]+)*))?) </ins> - path: cluster/images/etcd/Makefile <del> match: BASEIMAGE?=k8s.gcr.io/build-image/debian-base-arm64:vd+.d+.d+ </del> <ins> match: BASEIMAGE?=k8s.gcr.io/build-image/debian-base-arm64:[a-zA-Z]+-v((([0-9]+).([0-9]+).([0-9]+)(?:-([0-9a-zA-Z-]+(?:.[0-9a-zA-Z-]+)*))?)(?:+([0-9a-zA-Z-]+(?:.[0-9a-zA-Z-]+)*))?) </ins> - path: cluster/images/etcd/Makefile <del> match: BASEIMAGE?=k8s.gcr.io/build-image/debian-base-ppc64le:vd+.d+.d+ </del> <ins> match: BASEIMAGE?=k8s.gcr.io/build-image/debian-base-ppc64le:[a-zA-Z]+-v((([0-9]+).([0-9]+).([0-9]+)(?:-([0-9a-zA-Z-]+(?:.[0-9a-zA-Z-]+)*))?)(?:+([0-9a-zA-Z-]+(?:.[0-9a-zA-Z-]+)*))?) </ins> - path: cluster/images/etcd/Makefile <del> match: BASEIMAGE?=k8s.gcr.io/build-image/debian-base-s390x:vd+.d+.d+ </del> <ins> match: BASEIMAGE?=k8s.gcr.io/build-image/debian-base-s390x:[a-zA-Z]+-v((([0-9]+).([0-9]+).([0-9]+)(?:-([0-9a-zA-Z-]+(?:.[0-9a-zA-Z-]+)*))?)(?:+([0-9a-zA-Z-]+(?:.[0-9a-zA-Z-]+)*))?) </ins> - name: "k8s.gcr.io/debian-iptables: dependents" <del> version: 12.1.2 </del> <ins> version: buster-v1.3.0 </ins> refPaths: - path: build/common.sh match: debian_iptables_version= - path: build/workspace.bzl match: tag = - path: test/utils/image/manifest.go <del> match: configs[DebianIptables] = Config{buildImageRegistry, "debian-iptables", "vd+.d+.d+"} </del> <ins> match: configs[DebianIptables] = Config{buildImageRegistry, "debian-iptables", "[a-zA-Z]+-v((([0-9]+).([0-9]+).([0-9]+)(?:-([0-9a-zA-Z-]+(?:.[0-9a-zA-Z-]+)*))?)(?:+([0-9a-zA-Z-]+(?:.[0-9a-zA-Z-]+)*))?)"} </ins> - name: "k8s.gcr.io/go-runner: dependents" version: buster-v2.0.1

What would you like to be : Have debian-iptables with iptables 1.8.5+ Why is this needed: To fix (nodelocaldns + iptables-nft) More informations Found a bug in iptables-nft, fixed in iptables 1.8.5, and now available in buster-backports https://alioth-
/sig api-machinery
ping as you did the last 2 updates
Yep, this is on my list. Will take of it next week. /sig release network /area release-eng /milestone v1.20 /assign
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
Opened to bump the debian-iptables images in kubernetes/kubernetes. /remove-triage unresolved
Here are the digests for the new image ():
Please reopen if you run into any issues with the new images. /close
Closing this issue. <detailsIn response to : Please reopen if you run into any issues with the new images. /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: "//pkg/apis/core:go_default_library", "//pkg/apis/extensions:go_default_library", "//pkg/client/clientset_generated/internalclientset:go_default_library", <del> "//pkg/controller:go_default_library", </del> "//pkg/controller/daemon:go_default_library", "//pkg/controller/deployment/util:go_default_library", "//pkg/controller/job:go_default_library",

Please see : https://k8s- fails just after this line: Failure was introduced in: But this was not noticed until it started running in [Conformance] tests. You can see the same problem in other dashboards too: https://k8s- https://k8s-
cc /sig apps
/priority critical-urgent
/kind bug /priority failing-test
/milestone v1.10
Is this just an oversight of not removing the template generation when the API update was done?
/cc
triaged in progress
i have never touched this test before ... so i just threw in a WIP, feel free to use/discard it. there seems to be lot of methods in there that is no longer used, may be we can rip those out too. (see )
xref
[MILESTONENOTIFIER] Milestone Issue Needs Attention Action Required: This issue has not been updated since Mar 5. Please provide an update. Note: This issue is marked as , and must be updated every 1 day during code freeze. Example update: <details<summaryIssue Labels</summary- : Issue will be escalated to these SIGs if needed. : Never automatically move issue out of a release milestone; continually escalate to contributor and SIG through all available channels. : Fixes a bug discovered during the current release. </details<details<summaryHelp</summary<ul<li<a href=""Additional instructions</a</li<li<a href=""Commands for setting labels</a</li</ul</details


Question: "time" "k8s.io/api/core/v1" <ins> "k8s.io/apimachinery/pkg/api/resource" </ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets"

<!-- Please only use this template for submitting enhancement requests --What would you like to be : Today, the scheduler re-processes unschedulable nodes . Node updates are sent by Kubelet frequently (every 10 seconds) and often times they don't change any of the node properties that could potentially make them feasible. We should be smarter about these updates and move pods to the schedulable queue only when certain properties of nodes change that could make them feasible. These properties include node allocatable, node conditions, taints, labels, .... Why is this needed: Improves performance of the scheduler and avoids doing unnecessary attempts to schedule pods. This is particularly helpful when a cluster has a lot of unschedulable pods. <!-- DO NOT EDIT BELOW THIS LINE --/kind feature This is not a feature really. It is performance optimization, but in the absence of such a label, I need to use "feature" here. /sig scheduling
I can help to fix it. /assign mlmhl
The main reason for this problem is / update their status too frequently and most of these updates are heartbeats without any meaningful change. Recently sig-node introduces a new to alleviate this problem. With this mechanism, kubelet will split heartbeat and node status updates into two independent operations, and reduce the frequency of status updates to once per 1 minute(or more). In the long run this mechanism can indeed alleviate our problems, but at present it does not help us much as this is still an alpha feature and the node status updates frequency is still a bit high(per 1 minute right now).
Maybe we can try comparing old and new nodes after stripping their heartbeat info, like this:
I think that had sent earlier is more efficient. However, that PR does not filter out . In Wei's code, the heartbeat time is removed correctly. I think we should fix this issue and resubmit that PR.
could you resubmit a PR to: disregard in conditions comparison: and (optionally) disregard timestamp in taints comparison:
Thanks for all your advice, a new PR is submitted: This PR does not include your second suggestion(disregard timestamp in taints comparison) as I'm not sure if can be changed when node updating.


Question: "time" "k8s.io/api/core/v1" <ins> "k8s.io/apimachinery/pkg/api/resource" </ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets"

If it can be changed, I will add this filter later, otherwise I think we can keep the current implementation to make the compare operation simpler and faster. What's more, just as I said in the PR's description, I'm not sure how much side effects this change has on scheduler's resource consumption, as it always copy node's conditions before comparison.
I am thinking whether we could move this logic(whether node became more schedulable) to kubelet, kubelet reports it through heartbeats.


Question: opts.CombinedInsecureServing.Healthz.Listener = makeListener(t) defer opts.CombinedInsecureServing.Healthz.Listener.Close() <del> for _, f := range opts.Flags().FlagSets { fs.AddFlagSet(f) } if err := fs.Parse(tc.flags); err != nil { t.Fatal(err) } </del> ctx, cancel := context.WithCancel(context.Background()) defer cancel() _, sched, err := Setup(ctx, opts)

ci-kubernetes-e2e-gci-gce-alpha-features kubetest.Up Jul 02 2021, 01:26:42 UTC+0200 /sig release
based on testgrid probably cc <img width="1203" alt="Screen Shot 2021-07-02 at 4 24 25 PM" src="https://user-"
/cc working on it
/assign
the scheduler fails to start with this error: it seems the feature gates from the command line does not take effect in the defaulting phrase, do you know how to fix it?
ah, found there is a cycle here: The scheduler options , but the default config's default values depend on the feature gates options. I think this disables other feature gate controlled logic in the defaulting too:
I tried to fix it by updating the component config after the CLI flags (including the feature gates) are parsed: PTAL
fixed the flake. is back green now.
thanks!


Question: // 'short' accounted ones. calls := &sync.WaitGroup{} calls.Add(AllowedInflightRequestsNo * 2) <ins> // Responses is used to wait until all responses are // received. This prevents some async requests getting EOF // errors from prematurely closing the server responses := sync.WaitGroup{} responses.Add(AllowedInflightRequestsNo * 2) </ins> // Block is used to keep requests in flight for as long as we need to. All requests will // be unblocked at the same time. block := sync.WaitGroup{}

Have only seen this once, personally.
This flaked in Jenkins as well. [job/kubernetes-test-go/5351]
can you take a look at this? I think has a lot already :)


Question: linux/amd64=python:3.6-slim-stretch <del> linux/arm64=arm64v8/python:3.6-slim-stretch </del>

E2eNode Suite.[sig-node] Node Performance Testing [Serial] [Slow] Run node performance testing with pre-defined workloads TensorFlow workload From the testgrid, it has been 1/2 flaky since the start of history (03/20/2022) and has been failing constantly (0/2) since 04/01/2022 No response /sig node
/triage accepted /priority important-soon
The failing test seems to be expecting GPU on machine and the image seems to be broken. I tried the image manually: Logs:
/assign as discussed at SIG Node CI meeting, can you please take a look? I see that pod is now running for couple minutes and then stops. I wonder if this is something environment related - maybe pod was evicted or something. I doesn't seem to be able to repro the failure of the pod locally
GitHub didn't allow me to assign the following users: xmcqueen. Note that only , repo collaborators and people who have commented on this issue/PR can be assigned. Additionally, issues/PRs can only have 10 assignees at the same time. For more information please see <detailsIn response to : /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
This is no longer a flake. It has failed every run since the start of June. Locally this test runs perfectly with a legit correct local run of the test suite code, doing the full tensorflow run of 300 epochs. The failure in the testgrid job seems to be fast (173 sec to build and fail, locally its 160 sec before it starts the tensorflow pod). I wonder if there's a blocked port or something from the data initializing python program. It needs access to: . Here's the command it runs: Is it feasible to add debugging code, like a pre-check for that http port? It would be good to at least force it to fail with a meaningful error message.
Note that HEAD is now at image version 1.2, and it is still failing per . The successful local runs of the test mentioned above used image version 1.2.


Question: linux/amd64=python:3.6-slim-stretch <del> linux/arm64=arm64v8/python:3.6-slim-stretch </del>

Locally image version 1.1 does produce the cuda/GPU error shown above by :
More info is needed to find the RC for this test failure. I've put in a PR to get the test framework to preserve the container logs when the pod fails.
The pod logs show that it is timing out on an http connection, and it looks like it's the one where it tries to download the test data set for the tensorflow run. You can see it . Search down to . The data_download script in the image attempts to pull down a dataset from: . That url is live and healthy, so they might be blocking the gcloud.
This test depending on that external data doesn't look quite right :/
I'm discussing this test with sig-testing. I'll post an update later.
Talked to from sig-testing: he's contacting the remote to see if they are blocking us, because it looks like they are is pulling the script from a that is So I see three choices: upgrade this thing to a more recent edition delete it grab everything that's needed to run this and keep it in-house <- clear winner here what do you think: Maybe the folks over at sig-testing have a place to store the artifacts in-house? They are not massive (about 10MB see below) Size of data about 10MB: the tools and libs for the tensorflow run: the training and eval files for the testrun:
Here's a relevant issue in a related k8s repo about using/not using docker registries: Registries used in k/k CI should be on Kubernetes Community infra
looks good for the subject of putting up a new docker image containing the deps. This seems feasible and effective. I'd prefer it to deleting the test. I'll discuss it a bit more with a few others, but this seems like the way to go forward with this. TBS, I plan to take the above listed test requirements, and build a docker image containing all that is needed, and reset the test to pull the new docker image.
I've got some Dockerfile changes to fix this. I've tested it locally with no network and it does the full 300 epochs. PR coming soon.
Is it worth pushing up 182 files to get this green? It is about 6M of python, and 5M of a test dataset.


Question: linux/amd64=python:3.6-slim-stretch <del> linux/arm64=arm64v8/python:3.6-slim-stretch </del>

Its all stashed away in the to get the docker image to contain the prereqs. It is selected subdirectories of the quite old archived version (v1.9.0) of the code from here: current version v2.7.2 and two small test data sets from here:
OK I got it down to 43 files, including the 2 datasets. That's the drivers and the datasets and the libs they need. It all gets packed into the docker image.
quick question - is there another repository that is under sig-node that can host this image?
To be clear, the test's docker image is currently in a repo, the : This PR adds artifacts to an existing docker image in the above repo.
I've cancelled the above PR. I'll try to get something else going.
There has been a decent amount of discussion in slack relating to handling the test's required artifacts. The solution that seems best is to get some practical version of the test requirements into locally owned and controlled storage, so the endpoints that provide access to the assets will at least be under internal control. This is not code, and is not under development. It is a static dataset, that is now stored in an offsite archive, and will not require maintenance. The datasets are published with the "": (see bottom right) "This allows for the sharing and adaptation of the datasets for any purpose, provided that the appropriate credit is given." Here are some links to the dataset and where its hosted: https://archive-
I'd like to get the dataset that holds the test data (about 650k) hosted in GCS where I can get it via http. With that change it is expected that the test will go green. The failures show it is failing when it tries to download the artifact from the uncontrolled remote endpoint. Somebody, please help me get this artifact into an internally controlled server. Thanks
Was the idea of downloading this only on image creation in dockerfile rejected?
Thanks for the reminder I did not fairly evaluate that proposal. I just jumped to my earlier proposal. Thinking about this proposal I see that it would mean downloading the test data for inclusion into the docker image, while the image's test driver tooling is built to pull down the dataset itself from a remote.


Question: linux/amd64=python:3.6-slim-stretch <del> linux/arm64=arm64v8/python:3.6-slim-stretch </del>

That is a simple change to handle, while simply re-hosting the artifact on GCS would require only changing the endpoint. Also building the docker would probably still require network access to the artifact, which would probably require us to put it somewhere anyway. I assume, putting it into the docker image would best be done by downloading the test data artifact from somewhere. Would we then be storing the artifact twice - once in the docker image, and once in a locally controlled endpoint? If the docker build server has unrestricted access to the artifact, this proposal would be good. I don't have access to the servers to run tests. However, I could put up an RB for the Dockerfile to try it out. This is probably worth the effort. I'll get an PR going for trying to pull down the remote test-data artifact from the image build server, and bunde it into the docker image, and I'll link it to this ticket.
thank you. We do not change the image for a long time, so downloading it once would likely be enough. Once it is self-contained, it is even better for the continuous perf runs in tests.
/reopen
You can't reopen an issue/PR unless you authored it or you are a collaborator. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Currently the image builds for the first arch (amd64) but fails when it enters the cross compiling phase (arm64) and specifically when it tries to do apt installs from within the qemu-*-static emulator: The other images in node-perf fail to build in the same way. The jessie-dnsutils image build fails to build in the same way. I have tried pulling in newer base images up to buster, but it did not fix the problem. -Start focusing on the version of qemu. - qemu is version locked in the Makefile and has not changed in two years. There was an and the images were building at that time.


Question: linux/amd64=python:3.6-slim-stretch <del> linux/arm64=arm64v8/python:3.6-slim-stretch </del>

Try to find clues over there.
I tried pinning many relevant versions of from but none helped. Here's a clue: This ugly hack gets it past the observed error:
Note the previous PR was missing an important change, but I do not see any feasible way that could explain the current problem with failing, so I have not felt any urgency to put up a PR for it. FTR I need to bump the VERSION. I did not bump the version in I thought maybe the CI would bump it automatically.
There's an image version in . Its still pulling the old version and failing. Bump it from 1.2 to 1.3. So one more commit is needed.
The new image is present and runs perfectly. Tested it on local linux and in google cloud: It ran to completion. Note the new image URL. Its NOT the same as the old one:
podspec references the image Here's the . tf-wide-deep's url is built The above has staging in it. . Its already built and pushed to staging. It must be promoted when this new version number goes out.
FTR: My "testing" above did not catch the current problem because I did not run the full e2e_test binary. I only tested the image via a . AND I should have read way back at the beginning.
Local run of : has to change too.
Local test using the published 1.3 image run the standard way was successful with the above: ran the test successfully. So promote the image per bump the image version in the e2e test manifest change the
After the image promotion is merged, there's STILL one more commit coming: bump the image version in the e2e test manifest change the
its promoted and is found at the expected url local containerd cleared out local test was rerun pulling from the new prod registry local test using that new registry passes
its fixed. the test is green again.


Question: nodeInstancePrefix       string   // If non-"", an advisory prefix for all nodes in the cluster useMetadataServer        bool operationPollRateLimiter flowcontrol.RateLimiter <ins> manager                  ServiceManager } type ServiceManager interface { // Creates a new persistent disk on GCE with the given disk spec. CreateDisk(project string, zone string, disk *compute.Disk) (*compute.Operation, error) // Gets the persistent disk from GCE with the given diskName. GetDisk(project string, zone string, diskName string) (*compute.Disk, error) // Deletes the persistent disk from GCE with the given diskName. DeleteDisk(project string, zone string, disk string) (*compute.Operation, error) // Waits until GCE reports the given operation in the given zone as done. WaitForZoneOp(op *compute.Operation, zone string, mc *metricContext) error } type GCEServiceManager struct { gce *GCECloud </ins> } type Config struct {

Unit tests are needed for the GCE cloud provider disk interface.
Assigned to
/assign
GitHub didn't allow me to assign the following users: verult. Note that only can be assigned.. <detailsIn response to : /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. I understand the commands that are listed . </details


Question: `'%v' 2>/dev/null`, testDaemonHTTPPort, string(jsonBytes)) <del> framework.RunHostCmdOrDie(fr.Namespace.Name, "e2e-net-client", cmd) </del> <ins> // Run the closewait command in a subroutine so it keeps waiting during postFinTimeoutSeconds // otherwise the pod is deleted and the connection is closed loosing the conntrack entry go func() { defer ginkgo.GinkgoRecover() _, err = framework.RunHostCmd(fr.Namespace.Name, "e2e-net-client", cmd) framework.ExpectNoError(err) }() </ins> <-time.After(time.Duration(1) * time.Second) ginkgo.By("Checking /proc/net/nf_conntrack for the timeout") <del> // If test flakes occur here, then this check should be performed // in a loop as there may be a race with the client connecting. e2essh.IssueSSHCommandWithResult( fmt.Sprintf("sudo cat /proc/net/nf_conntrack | grep 'dport=%v'", testDaemonTCPPort), framework.TestContext.Provider, clientNodeInfo.node) // Timeout in seconds is available as the fifth column from // /proc/net/nf_conntrack. result, err := e2essh.IssueSSHCommandWithResult( fmt.Sprintf( "sudo cat /proc/net/nf_conntrack "+ "| grep 'CLOSE_WAIT.*dst=%v.*dport=%v' "+ "| tail -n 1"+ "| awk '{print $5}' ", serverNodeInfo.nodeIP, testDaemonTCPPort), framework.TestContext.Provider, clientNodeInfo.node) framework.ExpectNoError(err) timeoutSeconds, err := strconv.Atoi(strings.TrimSpace(result.Stdout)) framework.ExpectNoError(err) </del> // These must be synchronized from the default values set in // pkg/apis/../defaults.go ConntrackTCPCloseWaitTimeout. The // current defaults are hidden in the initialization code. const epsilonSeconds = 60 const expectedTimeoutSeconds = 60 * 60 <del> framework.Logf("conntrack entry timeout was: %v, expected: %v", timeoutSeconds, expectedTimeoutSeconds) gomega.Expect(math.Abs(float64(timeoutSeconds - expectedTimeoutSeconds))).Should( gomega.BeNumerically("<", (epsilonSeconds))) </del> <ins> // the conntrack file uses the IPv6 expanded format ip := fullIPv6(net.ParseIP(serverNodeInfo.nodeIP)) // Obtain the corresponding conntrack entry on the host checking // the nf_conntrack file from the pod e2e-net-exec. // It retries in a loop if the entry is not found. cmd = fmt.Sprintf("cat /rootfs/proc/net/nf_conntrack "+ "| grep -m 1 'CLOSE_WAIT.*dst=%v.*dport=%v' ", ip, testDaemonTCPPort) if err := wait.PollImmediate(5*time.Second, 30*time.Second, func() (bool, error) { result, err := framework.RunHostCmd(fr.Namespace.Name, "e2e-net-exec", cmd) // retry if we can't obtain the conntrack entry if err != nil { framework.Logf("failed to obtain conntrack entry: %v %v", result, err) return false, nil } framework.Logf("conntrack entry for node %v and port %v:  %v", serverNodeInfo.nodeIP, testDaemonTCPPort, result) // Timeout in seconds is available as the fifth column of // the matched entry in /proc/net/nf_conntrack. line := strings.Fields(result) if len(line) < 5 { return false, fmt.Errorf("conntrack entry does not have a timeout field: %v", line) } timeoutSeconds, err := strconv.Atoi(line[4]) if err != nil { return false, fmt.Errorf("failed to convert matched timeout %s to integer: %v", line[4], err) } if math.Abs(float64(timeoutSeconds-expectedTimeoutSeconds)) < epsilonSeconds { return true, nil } return false, fmt.Errorf("wrong TCP CLOSE_WAIT timeout: %v expected: %v", timeoutSeconds, expectedTimeoutSeconds) }); err != nil { framework.Failf("no conntrack entry for port %d on node %s", testDaemonTCPPort, serverNodeInfo.nodeIP) } </ins> }) // Regression test for #74839, where:

Which jobs are failing: kops-aws and many other jobs Which test(s) are failing: This isn't the actual test, but the text we're consistently seeing is Since when has it been failing: Testgrid links: (ref: ) Reason for failure: TBD Anything else we need to know: Very rarely tests pass when one of the tests passes: Seems related to , that was merged about the same time:
/sig cluster-lifecycle /sig aws
The label(s) cannot be applied, because the repository doesn't have them <detailsIn response to : /sig cluster-lifecycle /sig aws Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/priority critical-urgent this is merge blocking
/assign /cc /cc /cc
The PR we think we want to revert
is also failing, I don't think this is aws-specific
/cc
/assign
it failed with kind too :/ with the same message
Opened to revert
/unassign /sig network /sig testing /remove-sig cluster-lifecycle
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
This may be the actual cause PR to address But we're going with revert first to be sure we've identified the offending PR
I think I've identified the root cause
Revert brought us back to green: Awaiting root cause fix
/remove-triage unresolved
/reopen merged but we're now seeing multiple tests fail since it merged Looking at it went red between these commits:
Reopened this issue. <detailsIn response to : /reopen merged but we're now seeing multiple tests fail since it merged Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
is an attempt to fix
reverting in
is the failure cluster I suspect will go down once the revert goes in
/close Failure cluster has gone down, calling this done <img width="998" alt="Screen Shot 2020-01-10 at 9 56 17 AM" src="https://user-"
Closing this issue.


Question: `'%v' 2>/dev/null`, testDaemonHTTPPort, string(jsonBytes)) <del> framework.RunHostCmdOrDie(fr.Namespace.Name, "e2e-net-client", cmd) </del> <ins> // Run the closewait command in a subroutine so it keeps waiting during postFinTimeoutSeconds // otherwise the pod is deleted and the connection is closed loosing the conntrack entry go func() { defer ginkgo.GinkgoRecover() _, err = framework.RunHostCmd(fr.Namespace.Name, "e2e-net-client", cmd) framework.ExpectNoError(err) }() </ins> <-time.After(time.Duration(1) * time.Second) ginkgo.By("Checking /proc/net/nf_conntrack for the timeout") <del> // If test flakes occur here, then this check should be performed // in a loop as there may be a race with the client connecting. e2essh.IssueSSHCommandWithResult( fmt.Sprintf("sudo cat /proc/net/nf_conntrack | grep 'dport=%v'", testDaemonTCPPort), framework.TestContext.Provider, clientNodeInfo.node) // Timeout in seconds is available as the fifth column from // /proc/net/nf_conntrack. result, err := e2essh.IssueSSHCommandWithResult( fmt.Sprintf( "sudo cat /proc/net/nf_conntrack "+ "| grep 'CLOSE_WAIT.*dst=%v.*dport=%v' "+ "| tail -n 1"+ "| awk '{print $5}' ", serverNodeInfo.nodeIP, testDaemonTCPPort), framework.TestContext.Provider, clientNodeInfo.node) framework.ExpectNoError(err) timeoutSeconds, err := strconv.Atoi(strings.TrimSpace(result.Stdout)) framework.ExpectNoError(err) </del> // These must be synchronized from the default values set in // pkg/apis/../defaults.go ConntrackTCPCloseWaitTimeout. The // current defaults are hidden in the initialization code. const epsilonSeconds = 60 const expectedTimeoutSeconds = 60 * 60 <del> framework.Logf("conntrack entry timeout was: %v, expected: %v", timeoutSeconds, expectedTimeoutSeconds) gomega.Expect(math.Abs(float64(timeoutSeconds - expectedTimeoutSeconds))).Should( gomega.BeNumerically("<", (epsilonSeconds))) </del> <ins> // the conntrack file uses the IPv6 expanded format ip := fullIPv6(net.ParseIP(serverNodeInfo.nodeIP)) // Obtain the corresponding conntrack entry on the host checking // the nf_conntrack file from the pod e2e-net-exec. // It retries in a loop if the entry is not found. cmd = fmt.Sprintf("cat /rootfs/proc/net/nf_conntrack "+ "| grep -m 1 'CLOSE_WAIT.*dst=%v.*dport=%v' ", ip, testDaemonTCPPort) if err := wait.PollImmediate(5*time.Second, 30*time.Second, func() (bool, error) { result, err := framework.RunHostCmd(fr.Namespace.Name, "e2e-net-exec", cmd) // retry if we can't obtain the conntrack entry if err != nil { framework.Logf("failed to obtain conntrack entry: %v %v", result, err) return false, nil } framework.Logf("conntrack entry for node %v and port %v:  %v", serverNodeInfo.nodeIP, testDaemonTCPPort, result) // Timeout in seconds is available as the fifth column of // the matched entry in /proc/net/nf_conntrack. line := strings.Fields(result) if len(line) < 5 { return false, fmt.Errorf("conntrack entry does not have a timeout field: %v", line) } timeoutSeconds, err := strconv.Atoi(line[4]) if err != nil { return false, fmt.Errorf("failed to convert matched timeout %s to integer: %v", line[4], err) } if math.Abs(float64(timeoutSeconds-expectedTimeoutSeconds)) < epsilonSeconds { return true, nil } return false, fmt.Errorf("wrong TCP CLOSE_WAIT timeout: %v expected: %v", timeoutSeconds, expectedTimeoutSeconds) }); err != nil { framework.Failf("no conntrack entry for port %d on node %s", testDaemonTCPPort, serverNodeInfo.nodeIP) } </ins> }) // Regression test for #74839, where:

<detailsIn response to : /close Failure cluster has gone down, calling this done <img width="998" alt="Screen Shot 2020-01-10 at 9 56 17 AM" src="https://user-"</details


Question: <tspan x="-34.876" y="715.256">Colocated, or spread across machines,</tspan> <tspan x="-34.876" y="731.256">as dictated by cluster size.</tspan> </text> <del> <text font-size="12.7998" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="506.887" y="611.5"> <tspan x="506.887" y="611.5"></tspan> </del> <ins> <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="506.888" y="611.5"> <tspan x="506.888" y="611.5"></tspan> </ins> </text> <del> <text font-size="12.7998" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="506.887" y="611.5"> <tspan x="506.887" y="611.5"></tspan> </del> <ins> <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="506.888" y="611.5"> <tspan x="506.888" y="611.5"></tspan> </ins> </text> <g> <rect style="fill: #ffffff" x="136.717" y="468.5" width="172.175" height="70"/> <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="136.717" y="468.5" width="172.175" height="70"/> <del> <text font-size="12.7998" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="222.804" y="491.4"> </del> <ins> <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="222.804" y="491.4"> </ins> <tspan x="222.804" y="491.4">REST</tspan> <tspan x="222.804" y="507.4">(pods, services,</tspan> <tspan x="222.804" y="523.4">rep. controllers)</tspan>

Architecture picture: says kubecfg is deprecated. The architecture picture should be updated to show kubectl instead of kubecfg.
ref /cc


Question: http2Options := &http2.Server{} if err := http2.ConfigureServer(s.Config, http2Options); err != nil { <del> t.Fatalf("failed to configure test server to be HTTP2 server, err: %v", err) </del> <ins> return nil, fmt.Errorf("failed to configure test server to be HTTP2 server, err: %v", err) </ins> } s.TLS = s.Config.TLSConfig <del> s.StartTLS() defer s.Close() </del> <ins> return s, nil } // watchResponse wraps watch response with data which server send and an error may occur. type watchResponse struct { // body is the response data which test GOAWAY server sent to client body []byte // err will be set to be a non-nil value if watch request is not end with EOF nor http2.GoAwayError err error } // newGOAWAYClient return a configured http client which used to request test GOAWAY server, a dial may specified // to encounter the TCP connection. func newGOAWAYClient(dial func(network, addr string, cfg *tls.Config) (net.Conn, error)) (*http.Client, error) { </ins> tlsConfig := &tls.Config{ InsecureSkipVerify: true, NextProtos:         []string{http2.NextProtoTLS}, } <ins> tr := &http.Transport{ Proxy:               http.ProxyFromEnvironment, TLSHandshakeTimeout: 10 * time.Second, TLSClientConfig:     tlsConfig, MaxIdleConnsPerHost: 25, DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { if dial != nil { return dial(network, addr, tlsConfig) } return tls.Dial(network, addr, tlsConfig) }, } if err := http2.ConfigureTransport(tr); err != nil { return nil, err } client := &http.Client{ Transport: tr, } return client, nil } // requestGOAWAYServer request test GOAWAY server using specified method and data according to the given url. // A non-nil channel will be returned if the request is watch, and a watchResponse can be got from the channel when watch done. func requestGOAWAYServer(client *http.Client, serverBaseURL, url string) (<-chan watchResponse, error) { method := http.MethodGet var reqBody io.Reader if url == urlPost || url == urlPostWithGoaway { method = http.MethodPost reqBody = bytes.NewReader(requestPostBody) } req, err := http.NewRequest(method, serverBaseURL+url, reqBody) if err != nil { return nil, fmt.Errorf("unexpect new request error: %v", err) } resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("failed request test server, err: %v", err) } if resp.StatusCode != http.StatusOK { defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("failed to read response body and status code is %d, error: %v", resp.StatusCode, err) } return nil, fmt.Errorf("expect response status code: %d, but got: %d. response body: %s", http.StatusOK, resp.StatusCode, body) } // encounter watch bytes received, does not expect to be broken if url == urlWatch || url == urlWatchWithGoaway { ch := make(chan watchResponse) go func() { defer resp.Body.Close() body := make([]byte, 0) buffer := make([]byte, 1) for { n, err := resp.Body.Read(buffer) if err != nil { // urlWatch will receive io.EOF, // urlWatchWithGoaway will receive http2.GoAwayError if err == io.EOF { err = nil } else if _, ok := err.(http2.GoAwayError); ok { err = nil } ch <- watchResponse{ body: body, err:  err, } return } body = append(body, buffer[0:n]...) } }() return ch, nil } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("failed to read response body, error: %v", err) } if !reflect.DeepEqual(responseBody, body) { return nil, fmt.Errorf("expect response body: %s, got: %s", string(responseBody), string(body)) } return nil, nil } // TestClientReceivedGOAWAY tests the in-flight watch requests will not be affected and new requests use a new // connection after client received GOAWAY. func TestClientReceivedGOAWAY(t *testing.T) { s, err := newTestGOAWAYServer() if err != nil { t.Fatalf("failed to set-up test GOAWAY http server, err: %v", err) } s.StartTLS() defer s.Close() </ins> cases := []struct { name string reqs []string

What happened: We enabled feature in our large-scale cluster, and we found client(Kubelet) may got error(500 http code response) like after received GOAWAY frame. I had already re-produce this bug and open an in golang repo. So far I think this bug only occur with a small probability when client request server in high concurrency use a same httpclient instance, and it's a golang sdk bug in http client. Before I solved this issue, I suggest do NOT enable GOAWAY() feature. As it's a golang sdk bug(currently I known and doubt) in http client, revert GOAWAY feature(codes) is not necessary.
cc
/sig api-machinery
Thanks for the report.
Based on not setting the field, I think it defaults that way, so that's good. We will leave the option and filter so that when golang updates its client, we will have a simple way to turn it on and execute it.
Sure. I am trying to fix the golang http client issue and will update progress at . After golang http client issue fixed, I will add more concurrent test cases to see if there are any other bugs may occur by GOAWAY feature. BTW, we still enable GOAWAY feature in our production cluster, because there is no any evidence told us this error will influence SLO, as all most components(kubelet, controller-manager, scheduler) is controller work mode and will re-try after got error. /assign
Golang version or required, which will release at 07/01/2020.
Kubernetes v1.19(or master branch) now is using golang 1.15, and golang 1.15 has fixed the issue. And is on going. Let me add more test cases to ensure the error will not happen again.
I think we should re-open this issue because of . /re-open


Question: allErrors = append(allErrors, fmt.Errorf("invalid configuration: option %q specified for HairpinMode (--hairpin-mode). Valid options are %q, %q or %q", kc.HairpinMode, kubeletconfig.HairpinNone, kubeletconfig.HairpinVeth, kubeletconfig.PromiscuousBridge)) } <ins> if err := validateKubeletOSConfiguration(kc); err != nil { allErrors = append(allErrors, err) } </ins> return utilerrors.NewAggregate(allErrors) }

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: What happened: Pods stuck in terminating state on Windows node after deleting it What you expected to happen: Pods get terminated and should disappear from kubectl How to reproduce it (as minimally and precisely as possible): Run a pod with windows container Delete it Pods are still terminating Anything else we need to know?: Could you pls help to resolve this issue. K8s pods stuck as Terminating after getting deleted on windows node with windows container(IIS). Docker ps -a not showing this container, so it is completely deleted, but in kubectl it is marked as Terminating. For networking we using OVN. kubectl logs: kubelet logs: Environment: Kubernetes version (use ):Cloud provider or hardware configuration: Hyper-V VMs OS (e.g. from /etc/os-release): Windows Node: Master Node: NAME="Ubuntu" VERSION="16.04.4 LTS (Xenial Xerus)" ID=ubuntu IDLIKE=debian PRETTYNAME="Ubuntu 16.04.4 LTS" VERSIONID="16.04" HOMEURL="" SUPPORTURL="" BUGREPORTURL="" VERSIONCODENAME=xenial UBUNTUCODENAME=xenial Kernel (e.g. ): Linux esd-kbrnts-lnx-1 4.13.0-37-generic ~16.04.1-Ubuntu SMP Wed Mar 7 16:03:28 UTC 2018 x8664 x8664 x8664 GNU/Linux Install tools: OVN 2.7 on windows OVN 2.7.2 on linux Others: Docker Version on Windows node: Server Version: 17.06.2-ee-6
/sig windows
It needs sometime for kubelet to remove the pod. Could you wait a while and check whether the Pod removed?
Hi actually I have waited for 1 day, and it is not deleting. Can delete it only using --grace-period=0 --force. But I don't want to use this manual approach). Could you pls give some advice on this? Thanks in advance!
is a reference to this issue. Checking kubelet logs will always help to troubleshoot such problem.


Question: allErrors = append(allErrors, fmt.Errorf("invalid configuration: option %q specified for HairpinMode (--hairpin-mode). Valid options are %q, %q or %q", kc.HairpinMode, kubeletconfig.HairpinNone, kubeletconfig.HairpinVeth, kubeletconfig.PromiscuousBridge)) } <ins> if err := validateKubeletOSConfiguration(kc); err != nil { allErrors = append(allErrors, err) } </ins> return utilerrors.NewAggregate(allErrors) }

Actually, I have attached kubelet log in the ticket and think the main reason is Also as there any option for kubelet to automatically force delete Terminating pods after some period? And --containerized is also not solving my issue. Thanks!
Kubelet won't delete the pod from apiserver. Since it is already not exist on the node, you could force delete it by kubectl.
It is clear, so how we can resolve this error: ? I think this is the core item of our issue. Thanks.
I'm not using OVN. Have you met this problem when using OVN?
adding Bob to comment on this. i have seen this issue before, but it has been quite a while.
Hi, Guys will be very grateful if you will provide some solution for it. We have tried to add to deployment spec but it doesn't help. Thanks.
The error message of is actually a warning message. It shouldn't block the removal of pods. Could you recheck the following things: the node, check whether the pod is still existing, e.g. by the pod exists, check if there are other error or warning message in kubelet logs; or else, the pod could be force removed safely from apiserver
Hi, I had posted this info in the ticket description. not showing this container, so it is completely deleted, but in kubectl it is displayed as Terminating.
i think you may get more support and faster responses in our slack channel for sig-windows.
So it could be safely force removed from API. And agreed with slack is a more faster way to such troubleshooting issues.
The issue seems to be related to . The only platform specific implementation for PodContainerManager is the one for . There is a default implementation (), that always returns true in the Exists call. Disabling the CgroupsPerQOS option on the Windows nodes resolves the issue.
Windows doesn't support cgroups. You should always disable it.
it is clear enough that Windows does not support cgroups). Unfortunately, it's not specified in the documentation. So to disable CgroupsPerQOS you should need to add this flags to kubelet: Maybe it will help someone... Thank you all for discussion!


Question: requests                 v1.ResourceList limits                   v1.ResourceList memoryWorkingSet         string <ins> pidUsage                 uint64 </ins> rootFsUsed               string logsFsUsed               string logsFsInodesUsed         string

We should add a unit test to Eviction manager that accounts for PID Pressure. The test should cover similar cases as DiskPressure or MemoryPressure. 1) PIDPressure doesn't exist on node. 2) Force PidPressure on node We should include test cases for reclaming resources in terms of pressure and verifying differences between soft and hard eviction.
/sig node /kind feature /help
This request has been marked as needing help from a contributor. Please ensure that the issue body includes answers to the following questions: Why are we solving this issue? To address this issue, are there any code changes? If there are code changes, what needs to be done in the code and what places can the assignee treat as reference points? Does this issue have zero to low barrier of entry? How can the assignee reach out to you for help? For more details on the requirements of such an issue, please see and ensure that they are met. If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : /sig node /kind feature /help </details
/triage accepted /priority important-longterm
/good-first-issue
This request has been marked as suitable for new contributors. Please ensure that the issue body includes answers to the following questions: Why are we solving this issue? To address this issue, are there any code changes? If there are code changes, what needs to be done in the code and what places can the assignee treat as reference points? Does this issue have zero to low barrier of entry? How can the assignee reach out to you for help? For more details on the requirements of such an issue, please see and ensure that they are met. If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : /good-first-issue Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign
it looks like has requested to work on this.
/assign
Hi What do you mean by, "verifying differences between soft and hard eviction."?
It’s the same idea as what we do for disk, memory or inode pressure.


Question: requests                 v1.ResourceList limits                   v1.ResourceList memoryWorkingSet         string <ins> pidUsage                 uint64 </ins> rootFsUsed               string logsFsUsed               string logsFsInodesUsed         string

Hard eviction means we trigger eviction with no grace period while soft eviction means that we obey a grace period. Test should check for that.
I have opened a PR based on what I could understand using a debugger and the other tests. This is my first time contributing to the codebase. Happy to take brutal feedback.
Hi, i would like to support this issue. Let me know if any help is required.
/assign


Question: Operator: v1.TolerationOpExists, Effect:   v1.TaintEffectNoSchedule, }, <ins> { Key:      algorithm.TaintNodeNotReady, Operator: v1.TolerationOpExists, Effect:   v1.TaintEffectNoSchedule, }, </ins> }, }, }

Is this a BUG REPORT or FEATURE REQUEST?: What happened: admission control plugin in apiserver. a deployment with 1 replica. The pod is successfully launched. By default the admission control plugin will add following toleration in the pod. the node where the above pod is running. of image locality policy and above toleration in the pod, scheduler still choose the node to launch the pod. The pod is in state and failed to create. What you expected to happen: The pod should be scheduled to healthy node and created successfully. How to reproduce it (as minimally and precisely as possible): See above steps. Anything else we need to know?: Environment: Kubernetes version (use ): v1.10 Cloud provider or hardware configuration: openstack OS (e.g. from /etc/os-release): Ubuntu 16.04 Kernel (e.g. ): 4.4.0-121-generic Install tools: N/A Others:
/sig scheduler
/sig scheduling /sig node
I think that we should avoid scheduling pods to nodes which is not ready even it can tolerate and , comments
As the pod already claimed to tolerate NotReady, this is an expected behavior. P.S. If the eviction dose not happen after 5m, that should be the bug. Please reopen the issue and modify the description. This is what tolerations is designed for :) And there are many use cases for pod to be scheduled to NotReady nodes (mostly during cluster deployment).
I think maybe you can configure effect to avoid scheduling to those nodes.
scheduler still choose the NotReady node to launch the pod For running, the toleration is used for graceful termination by ; if is enabled, it's better to also taint node with for .
Wondering how the pod can start to run if the Node is not ready at deployment time? Can you specify a use case here?
It won't need to start, system pod pending will be expected :) The point is, freedom for users to decide what tolerations they want matters here. I don't think it's reasonable to combine with in this case.
Perhaps, add to default toleration of admission control plugin could eliminate the confusion.
We should align the behavior when enabled or disabled : if is disabled, will not schedule pod to not-ready node. if is enabled, we should also taint node with .


Question: Operator: v1.TolerationOpExists, Effect:   v1.TaintEffectNoSchedule, }, <ins> { Key:      algorithm.TaintNodeNotReady, Operator: v1.TolerationOpExists, Effect:   v1.TaintEffectNoSchedule, }, </ins> }, }, }

with timeout is used for graceful termination of 'running pods' for feature. We should not re-schedule pod to not-ready unless tolerating it explicitly.
reopen this for more discussion :)
Let me extracted the mixed problems in this issue :-) First one: These two default tolerations attached to the pod by means: if the node is tainted as or , wait for 300s before the pod is evicted from the node. While according to the issue description, is working as expected (pods are not evicted immediately). This part should be closed. Second one: The discussion raised is actually about taint during eviction: if TaintNodesByCondition is enabled, we should also taint node with NotReady:NoSchedule. My thoughts is: it's reasonable to taint node with as well for consistency consideration (but I'm hesitate if only for eliminating confusion as pods are expected to pending on NotReady nodes) I think it should be clearer now.
I open a PR If TaintNodesByCondition is enable, taint node with NotReady:NoSchedule and Unreachable:NoSchedule.
, we did not catch 1.11 release, so this issue will be handled in 1.12 :)
in 1.9.7enable the TaintNodesByCondition and taint nodee.g. node01 with NotReady:NoSchedulethen pod also can not be scheduled to node01 even when node01 is healthy. What you expected to happen: The pod should be scheduled to node01 when it is healthy node and not be scheduled to node01 when it is not ready. Thanks
How did you do that? by kubectl? The fix is not ready until 1.12.
YesI tried "kubectl taint nodes node01 ",and also tried "kubectl taint nodes node01 "
this feature did not handle taint manually
Ok, thanks a lot! So I can not solve the problem manually now and what I should do is just waiting for the version 1.12, right?


Question: t.Errorf("expected %d ProxyLoops running, got %d", want, got) } <ins> func waitForNumProxyClients(t *testing.T, s *serviceInfo, want int, timeout time.Duration) { var got int now := time.Now() deadline := now.Add(timeout) for time.Now().Before(deadline) { s.activeClients.mu.Lock() got = len(s.activeClients.clients) s.activeClients.mu.Unlock() if got == want { return } time.Sleep(500 * time.Millisecond) } t.Errorf("expected %d ProxyClients live, got %d", want, got) } </ins> func TestTCPProxy(t *testing.T) { lb := NewLoadBalancerRR() service := proxy.ServicePortName{NamespacedName: types.NamespacedName{Namespace: "testnamespace", Name: "echo"}, Port: "p"}

Depends on UDP clients have a timeout (so we don't leak connections forever), but the test does not cover this. To do this, we probably need to track active connections, and plumb that all the way down to the goroutines spawned by the ProxyLoop routines. This is probably something we want to do anyway to be able to monitor.


Question: // Reference to this node. nodeRef *v1.ObjectReference <del> // Track initialization initialized bool </del> // imageCache is the cache of latest image list. imageCache imageCache

when kubelet restart,all images which notused cleaned. but there is a flag in kubelet,,explainMinimum age for an unused image before it is garbage collected. Examples: '300ms', '10s' or '2h45m'. i set to ,it's means if a image not used,it will live 20m and then clean by kubeletif has disk pressure.if kubelet not restart, it work,but if restart ther is a bug, not work,image will be cleaneven if it's no enough time. work, image even if not used,it will live 20m and then clean by kubelet. 1make node trigger disk pressure. 2pull some image which not used by pod 3restart kubelet 4image cleaned by kubelet after around 10s i see the source code, and maybe there is a bug. in func,well detect image and update firstdetect time for every image. and in func,well check images livetime by firstDetected. if kubelet running normal,it work,because there is a func. in this line,,in first time,ts is zero. so image's firstDetected time is zero,and in freeSpace this check is always false,so image will be cleaned. maybe we can change to ,can fix this bug. <details</details<detailsNone </details<details</details<details</details<details</details<details</details
/sig node
/assign
/triage accepted /priority backlog initial assessment of priority, subject to change


Question: # Deepcopy # local input_pkgs=() <del> while read -r file; do dir="$(dirname "${file}")" </del> <ins> while read -r dir; do </ins> pkg="$(cd "${dir}" && GO111MODULE=on go list -find .)" input_pkgs+=("${pkg}") done < <( <del> ( kube::codegen::internal::git_grep -l  </del> <ins> ( kube::codegen::internal::git_grep -l --null  </ins> -e '+k8s:deepcopy-gen='  ":(glob)${root}"/'**/*.go'  || true  <del> ) | LC_ALL=C sort -u </del> <ins> ) | xargs -0 -n1 dirname  | LC_ALL=C sort -u </ins> ) if [ "${#input_pkgs[@]}" != 0 ]; then

function in script has following issues: function only supports file but we have multiple types file with name as in . Hence this function is not working in our case. For the mitigation there is already an open . I have asked for including our changes as well. We have multiple files under a for example and . Because of this for appears twice and it generates duplicate informers We only need unique elements before passing it to . I followed this approach for unique elements: We have packages under . Earlier we were generating client only for and . But with the new I am not able to do that. I am using the following call to function: It results in following flags to As there is no directory as , it doesnot generate anything. Is it still possible to call on specific folder as it was done with earlier scripts? One mitigation I can think of is to call function on whole and then remove the marker from the types for which genclient is not needed. But the issue is we won't be able to pass the flag for each pkg. function in script is not working as expected.
/sig api-machinery
/cc
/triage accepted
Hi . I knew that there were all sorts of clients out there doing all sorts of things with these scripts, so I am not surprised to find you. :) Do you have a PR which attempts to use kube_codegen, so I can repro your issues directly?
I think you have one too many in there Aside from that, the real problem here is that these old codegen tools make some deeeeeeep assumptions about the ability to string-join various arguments and get the right results. It's awful. I was trying to make it simpler in this script, which I did for all of the in-tree consumers, but out of tree people apparently don't always follow the same conventions. For example, we generate clients for every directory that requests it. For a reason I do not know, you have files like which say but you don't actually want a client? If you specify then it t
So this code expects that you generate an omnibus client. What if I add a flag, like which you can use to call it for each API to make distinct clients? In truth the codegen tools need to be fixed, but that has SO MUCH ripple I just can't right now.


Question: # Deepcopy # local input_pkgs=() <del> while read -r file; do dir="$(dirname "${file}")" </del> <ins> while read -r dir; do </ins> pkg="$(cd "${dir}" && GO111MODULE=on go list -find .)" input_pkgs+=("${pkg}") done < <( <del> ( kube::codegen::internal::git_grep -l  </del> <ins> ( kube::codegen::internal::git_grep -l --null  </ins> -e '+k8s:deepcopy-gen='  ":(glob)${root}"/'**/*.go'  || true  <del> ) | LC_ALL=C sort -u </del> <ins> ) | xargs -0 -n1 dirname  | LC_ALL=C sort -u </ins> ) if [ "${#input_pkgs[@]}" != 0 ]; then

As the codegen was breaking us at several step, hence didn't prepared the PR.
Thanks for the PR. I have one more concern, is it possible to call and client-gen on the same pkg? With the current script, that is not possible because of the line It removes all the generated files first and then calls the . Hence the later clientset gen will remove the earlier generated clients.
Why are you generating a client for an internal version that isn't served?
My mistake. We need to stop using clients for internal versions.


Question: # See the License for the specific language governing permissions and # limitations under the License. <del> FROM alpine:latest </del> <ins> FROM BASEIMAGE CROSS_BUILD_COPY qemu-QEMUARCH-static /usr/bin/ </ins> COPY nnp /usr/local/bin/nnp RUN chmod +s /usr/local/bin/nnp

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: /kind feature This issue is for creating image as multiarch
/assign
There are no sig labels on this issue. Please by: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. You can find the group list and label list . The in the method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals
/sig testing


Question: // true. SyncKnownPods(desiredPods []*v1.Pod) map[types.UID]PodWorkType <ins> // IsPodKnownTerminated returns true if the provided pod UID is known by the pod // worker to be terminated. If the pod has been force deleted and the pod worker // has completed termination this method will return false, so this method should // only be used to filter out pods from the desired set such as in admission. // // Intended for use by the kubelet config loops, but not subsystems, which should // use ShouldPod*(). IsPodKnownTerminated(uid types.UID) bool </ins> // CouldHaveRunningContainers returns true before the pod workers have synced, // once the pod workers see the pod (syncPod could be called), and returns false // after the pod has been terminated (running containers guaranteed stopped).

There appears to be a change of behavior / possible regression in k8s 1.22 with respect to pod lifecycle behavior. We've started to see some cases when multiple pods are being created (often as part of a k8s job) the following will happen: pods get created by e.g. job controller (e.g. say 5). Each pod in job requests majority of node memory so only one such pod can fit on node at a time. schedules [first] pod to node, rest (4 pods) stay in pending accepts first pod, runs it to completion pod get scheduled to the node rejects the pod during pod admission, e.g. with , despite the fact that the first pod has already completed successfully This results in many of pods backing the job getting into Failed phase and having to be recreated by job controller. The job can often get into BackofLimit exceeded to hitting backoffLimit. The root issue seems to be a mismatch of when kubelet actually considers the pod to complete and what it reports to API server (which causes the followup pods to be scheduled to it). I'm still investigating further, but this may be related to the pod lifecycle changes in 1.22 -- I've created a e2e test as minimal repro of this behavior -- (Test fails) 22 build log -- 22 kubelet log -- (Test passes) 21 build log -- 21 kubelet log -- <!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --- Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository.


Question: // true. SyncKnownPods(desiredPods []*v1.Pod) map[types.UID]PodWorkType <ins> // IsPodKnownTerminated returns true if the provided pod UID is known by the pod // worker to be terminated. If the pod has been force deleted and the pod worker // has completed termination this method will return false, so this method should // only be used to filter out pods from the desired set such as in admission. // // Intended for use by the kubelet config loops, but not subsystems, which should // use ShouldPod*(). IsPodKnownTerminated(uid types.UID) bool </ins> // CouldHaveRunningContainers returns true before the pod workers have synced, // once the pod workers see the pod (syncPod could be called), and returns false // after the pod has been terminated (running containers guaranteed stopped).

</details
/sig node
/cc
Looking at the kubelet log in 1.22 () here's what I see -- First pod starts up... starts dies after ~10 seconds since the job is completed (the container simply runs a ) Pod completes, status is updated to API server Next pod comes in but is rejected with despite the earlier pod already being completed/terminated
did the kubelet pod lifecycle rework result in a reordering of kubelet resource bookeeping w.r.t. pod status? it seems reasonable for the scheduler to send another pod to a kubelet once it reports the pod complete and the use of the resources is done from an API perspective
I am also seeing this same behavior, but with "OutOfPods" rather than "OutOfMemory", when running a job with parallelism=100, completions=400 on GKE (which has a 110 pod per node limit).
this is easily reproducible on a local cluster as well start with this: then create this job: result on 1.22 is the job hangs with 2 completions and a bunch of rejections by the kubelet: on 1.21, the same job completes all pods successfully in a couple minutes with no OutOfpods errors
verified introduced the OutOfPods issue
Shame we didn’t have an e2e which caught this given how easy the reproducer is. This looks to be the opposite of the previous cpu admission bug (ignoring things that are terminating but not terminated). We are weak on admission testing in general. This potentially implies admission is looking at the wrong set of pods (admitted + terminating). Note there is already a bug in 1.21 and before where force deleted pods are incorrectly ignored from the set of allocated resources while they are terminating, but this doesn’t immediately seem to be that issue (since i doubt those jobs are being force deleted)
yeah, there was no force deletion involved in (or any deletion at all, actually... the completed pods just sat there, completed)
The set of admitted pods considered for pod count is everything in the pod manager filtered by filterOutTerminatingPods, which calls CouldHaveRunningContainers which returns true once a pod is seen by the workers (shortly after admission, and until we’ve confirmed via syncTerminatingPod that no containers are running). Whatever the problem is it’s likely in the interaction of that.


Question: // true. SyncKnownPods(desiredPods []*v1.Pod) map[types.UID]PodWorkType <ins> // IsPodKnownTerminated returns true if the provided pod UID is known by the pod // worker to be terminated. If the pod has been force deleted and the pod worker // has completed termination this method will return false, so this method should // only be used to filter out pods from the desired set such as in admission. // // Intended for use by the kubelet config loops, but not subsystems, which should // use ShouldPod*(). IsPodKnownTerminated(uid types.UID) bool </ins> // CouldHaveRunningContainers returns true before the pod workers have synced, // once the pod workers see the pod (syncPod could be called), and returns false // after the pod has been terminated (running containers guaranteed stopped).

A pod the pod worker hasn’t seen yet is also filtered out (that means bursts of pods can get in, which seems the opposite of this problem, and is the bug i mentioned before).
Edit: one wrong thing spotted while looking - I'm not sure it fixes the issue you're seeing, but it's definitely wrong for admission (admission is filtering the known pod set down to remove pods that are terminated, but shouldn't filter pods that aren't admitted yet). This needs a node e2e reproducer that can run in serial (resource consumption is really hard to test from parallel, is slow in serial, and node e2e is back in a good state now) - / is that something either of you could help me with? I believe the reason the existing tests don't catch this is the job controller is creating pods fast enough that we see races in admission (multiple pods are admitted in one call to HandlePodAdditions) that are not caught in the current serial node e2e admission tests which rarely create more than one at a time.
Job pods still go through the scheduler to get assigned to a node, so even though the job controller can create pods in parallel, I'd still expect them to arrive at the node in an orderly fashion. It's still possible multiple arrive in a single HandlePodAdditions call, of course. It seems like kubelet admission is using laggy information that doesn't reflect pod completion the kubelet itself wrote to the pod API.
The bug turned out to be much simpler. Pods that are completed never go to the pod worker, so once a pod has reached terminal state it was being back to the list of active pods. The set of e2es that would have prevented this regression: node e2e that creates multiple pods at the same time and verifies that admission works correctly when HandlePodAdditions takes two pods at once node e2e that verifies that completed pods don't consume resources for admission purposes in the kubelet job controller e2e that attempts to verify some part of this reliably (ideally in a way that can be run in parallel e2e)
another related issue:
Linking here for completeness - a similar symptom might get exposed after this fix as described in The kubelet considers resources for terminating pods to be in use (they are!), but the scheduler ignores terminating pods and schedules new pods.


Question: // true. SyncKnownPods(desiredPods []*v1.Pod) map[types.UID]PodWorkType <ins> // IsPodKnownTerminated returns true if the provided pod UID is known by the pod // worker to be terminated. If the pod has been force deleted and the pod worker // has completed termination this method will return false, so this method should // only be used to filter out pods from the desired set such as in admission. // // Intended for use by the kubelet config loops, but not subsystems, which should // use ShouldPod*(). IsPodKnownTerminated(uid types.UID) bool </ins> // CouldHaveRunningContainers returns true before the pod workers have synced, // once the pod workers see the pod (syncPod could be called), and returns false // after the pod has been terminated (running containers guaranteed stopped).

Because the kubelet now considers terminating pods, it rejects those rapidly rescheduled pods. The follow up is more involved but requires the kubelet to allow pods to be potentially admitted that can't be run currently due to those terminating pods. See the linked issue.


Question: NUM_ADDITIONAL_NODES=${NUM_ADDITIONAL_NODES:-} ADDITIONAL_MACHINE_TYPE=${ADDITIONAL_MACHINE_TYPE:-} <del> # Set etcd image (e.g. k8s.gcr.io/etcd) and version (e.g. v3.5.0-rc.0-0) if you need </del> <ins> # Set etcd image (e.g. k8s.gcr.io/etcd) and version (e.g. v3.5.0-0) if you need </ins> # non-default version. export ETCD_IMAGE=${TEST_ETCD_IMAGE:-} export ETCD_DOCKER_REPOSITORY=${TEST_ETCD_DOCKER_REPOSITORY:-}

/sig scalability This issue is tracking process of upgrading etcd to 3.5.0 planned for 1.22 K8s release. Etcd has recently released Beta versions and is starting the process of correctness and performance testing. To reduce change of finding performance regression late, we will start qualifying Beta Etcd releases on GKE master branch and will upgrade to 3.5.0 as soon as it is available. Steps: [x] Do preliminary scale testing of etcd v3.5.0-beta.3 release [x] Release etcd v3.5.0-beta.3 image (, ) [x] Release etcd v3.5.0-rc.0 image () [x] Upgrade etcd server to v3.5.0-rc.0 () [x] Wait for 3.5.0 release and upgrade etcd client () [x] Release etcd v3.5.0 image (, ) [x] Upgrade etcd server to v3.5.0 /milestone v1.22
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/area etcd
Does this meant that the 3.5.0 should be promoted like
Yes, created
/area release-eng release-team /sig release For tracking: cc:
The label(s) cannot be applied, because the repository doesn't have them. <detailsIn response to : /area release-eng release-team /sig release cc: Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
I do think this can be pointed in the Major Themes for the 1.22 release. I would like your opinions on this. If you do think it has a place in it, I will add it to the .
How can I be a contributor to this issue? what are the technical checklist that needs to be achieved to close out on the issue?


Question: NUM_ADDITIONAL_NODES=${NUM_ADDITIONAL_NODES:-} ADDITIONAL_MACHINE_TYPE=${ADDITIONAL_MACHINE_TYPE:-} <del> # Set etcd image (e.g. k8s.gcr.io/etcd) and version (e.g. v3.5.0-rc.0-0) if you need </del> <ins> # Set etcd image (e.g. k8s.gcr.io/etcd) and version (e.g. v3.5.0-0) if you need </ins> # non-default version. export ETCD_IMAGE=${TEST_ETCD_IMAGE:-} export ETCD_DOCKER_REPOSITORY=${TEST_ETCD_DOCKER_REPOSITORY:-}

PR to update Etcd is ready, but we found a blocker for merging . There is one test failing I will take a look into it on Monday, still more eyes on the problem could speed up resolution.
We'll also need a new kube-cross image w/ etcd v3.5.0. I'm working on that here:
We just need to upgrade the server to use new image like in
ok, looks like is busy. Let me take a look /assign
Thanks.


Question: validateEvent(strconv.Itoa(index), actualEvent, item.expect, t) } <del> logWatcher1.Stop() logWatcher2.Stop() </del> <ins> logWatcher.Stop() </ins> } sinkWatcher.Stop() }

I wasn't able to find automatically create issue for this, but I may not know what to search for. Is our bot creating issues for unit test failures? - this failure looks to me as something in client testing libraries.
If it's exposed in JUnit output, then the bot should be making issues for it.
It's not exposed in junit due to :(
Lots of failures in , probably all this bug.
Yeah this seems to be happening a lot. Raising priority.
I thought was reverted but it's not. I'll revert it.
golang 1.6 doesn't fix the race:
Sent .


Question: classWait      = "wait" classImmediate = "immediate" <ins> sharedClasses = map[storagev1.VolumeBindingMode]*storagev1.StorageClass{ modeImmediate: makeStorageClass(classImmediate, &modeImmediate), modeWait:      makeStorageClass(classWait, &modeWait), } </ins> ) const (

Add test for "on PVC add", see Test for "on PVC update" is not needed, because it has already been tested in all volume scheduling tests.
/sig
docs:


Question: pod := getSwapTestPod(f, qosClass, memoryRequestEqualLimit) pod = runPodAndWaitUntilScheduled(f, pod) <del> gomega.Expect(isPodCgroupV2(f, pod)).To(gomega.BeTrueBecause("cgroup v2 is required for swap")) </del> <ins> if !isPodCgroupV2(f, pod) { e2eskipper.Skipf("swap tests require cgroup v2") } </ins> gomega.Expect(getSwapBehavior()).To(gomega.Equal(types.LimitedSwap)) expectedSwapLimit := calcSwapForBurstablePod(f, pod)

master-informing: ci-crio-cgroupv1-node-e2e-conformanceE2eNode Suite.[It] [sig-node] Swap [LinuxOnly] [NodeFeature:NodeSwap] [NodeConformance] with configuration QOS Best-effort E2eNode Suite.[It] [sig-node] Swap [LinuxOnly] [NodeFeature:NodeSwap] [NodeConformance] with configuration QOS Burstable E2eNode Suite.[It] [sig-node] Swap [LinuxOnly] [NodeFeature:NodeSwap] [NodeConformance] with configuration QOS Burstable with memory request equals to limit E2eNode Suite.[It] [sig-node] Swap [LinuxOnly] [NodeFeature:NodeSwap] [NodeConformance] with configuration QOS Guaranteed After was merged No response /sig node
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign
/cc


Question: "k8s.io/klog" <del> "k8s.io/api/core/v1" </del> <ins> v1 "k8s.io/api/core/v1" </ins> storage "k8s.io/api/storage/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" meta "k8s.io/apimachinery/pkg/apis/meta/v1"

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!--What happened: Followup to In 1.15, we can merge this commit: <!-- DO NOT EDIT BELOW THIS LINE --/kind bug
/assign
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale