Question:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: With a /22 pod network and a /24 assigned to each node, a third node cannot get a pod CIDR if kube-controller-manager has ever been restarted. What you expected to happen: Once a third or fourth node joins, it gets a /24. How to reproduce it (as minimally and precisely as possible): 150% usage on the IPAM controller is not a good sign. Anything else we need to know?: During the creation of the rangeAllocator, Occupy() is called twice on the pod networks. The first time is before subscribing to the node informer. The second time is during the node informer callbacks because on subscribing it calls Add for all existing nodes. I will submit a proposed fix shortly. Environment: Kubernetes version (use ): 1.19.2
/sig network
/triage accepted thanks for finding the bug I 've introduced :)

} } <ins> func TestDoubleOccupyRelease(t *testing.T) { // Run a sequence of operations and check the number of occupied CIDRs // after each one. clusterCIDRStr := "10.42.0.0/16" operations := []struct { cidrStr     string operation   string numOccupied int }{ // Occupy 1 element: +1 { cidrStr:     "10.42.5.0/24", operation:   "occupy", numOccupied: 1, }, // Occupy 1 more element: +1 { cidrStr:     "10.42.9.0/24", operation:   "occupy", numOccupied: 2, }, // Occupy 4 elements overlapping with one from the above: +3 { cidrStr:     "10.42.8.0/22", operation:   "occupy", numOccupied: 5, }, // Occupy an already-coccupied element: no change { cidrStr:     "10.42.9.0/24", operation:   "occupy", numOccupied: 5, }, // Release an coccupied element: -1 { cidrStr:     "10.42.9.0/24", operation:   "release", numOccupied: 4, }, // Release an unoccupied element: no change { cidrStr:     "10.42.9.0/24", operation:   "release", numOccupied: 4, }, // Release 4 elements, only one of which is occupied: -1 { cidrStr:     "10.42.4.0/22", operation:   "release", numOccupied: 3, }, } // Check that there are exactly that many allocatable CIDRs after all // operations have been executed. numAllocatable24s := (1 << 8) - 3 _, clusterCIDR, _ := net.ParseCIDR(clusterCIDRStr) a, err := NewCIDRSet(clusterCIDR, 24) if err != nil { t.Fatalf("Error allocating CIDRSet") } // Execute the operations for _, op := range operations { _, cidr, _ := net.ParseCIDR(op.cidrStr) switch op.operation { case "occupy": a.Occupy(cidr) case "release": a.Release(cidr) default: t.Fatalf("test error: unknown operation %v", op.operation) } if a.allocatedCIDRs != op.numOccupied { t.Fatalf("Expected %d occupied CIDRS, got %d", op.numOccupied, a.allocatedCIDRs) } } // Make sure that we can allocate exactly `numAllocatable24s` elements. for i := 0; i < numAllocatable24s; i++ { _, err := a.AllocateNext() if err != nil { t.Fatalf("Expected to be able to allocate %d CIDRS, failed after %d", numAllocatable24s, i) } } _, err = a.AllocateNext() if err == nil { t.Fatalf("Expected to be able to allocate exactly %d CIDRS, got one more", numAllocatable24s) } } </ins> func TestGetBitforCIDR(t *testing.T) { cases := []struct { clusterCIDRStr string


Question:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: With a /22 pod network and a /24 assigned to each node, a third node cannot get a pod CIDR if kube-controller-manager has ever been restarted. What you expected to happen: Once a third or fourth node joins, it gets a /24. How to reproduce it (as minimally and precisely as possible): 150% usage on the IPAM controller is not a good sign. Anything else we need to know?: During the creation of the rangeAllocator, Occupy() is called twice on the pod networks. The first time is before subscribing to the node informer. The second time is during the node informer callbacks because on subscribing it calls Add for all existing nodes. I will submit a proposed fix shortly. Environment: Kubernetes version (use ): 1.19.2
/sig network
/triage accepted thanks for finding the bug I 've introduced :)

NodeCIDRMaskSizes:    []int{24, 98, 24}, }, }, <ins> { description: "no double counting", fakeNodeHandler: &testutil.FakeNodeHandler{ Existing: []*v1.Node{ { ObjectMeta: metav1.ObjectMeta{ Name: "node0", }, Spec: v1.NodeSpec{ PodCIDRs: []string{"10.10.0.0/24"}, }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "node1", }, Spec: v1.NodeSpec{ PodCIDRs: []string{"10.10.2.0/24"}, }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "node2", }, }, }, Clientset: fake.NewSimpleClientset(), }, allocatorParams: CIDRAllocatorParams{ ClusterCIDRs: func() []*net.IPNet { _, clusterCIDR, _ := net.ParseCIDR("10.10.0.0/22") return []*net.IPNet{clusterCIDR} }(), ServiceCIDR:          nil, SecondaryServiceCIDR: nil, NodeCIDRMaskSizes:    []int{24}, }, expectedAllocatedCIDR: map[int]string{ 0: "10.10.1.0/24", }, }, </ins> } // test function testFunc := func(tc testCase) { <ins> fakeNodeInformer := getFakeNodeInformer(tc.fakeNodeHandler) nodeList, _ := tc.fakeNodeHandler.List(context.TODO(), metav1.ListOptions{}) </ins> // Initialize the range allocator. <del> allocator, err := NewCIDRRangeAllocator(tc.fakeNodeHandler, getFakeNodeInformer(tc.fakeNodeHandler), tc.allocatorParams, nil) </del> <ins> allocator, err := NewCIDRRangeAllocator(tc.fakeNodeHandler, fakeNodeInformer, tc.allocatorParams, nodeList) </ins> if err != nil { t.Errorf("%v: failed to create CIDRRangeAllocator with error %v", tc.description, err) return


Question:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: With a /22 pod network and a /24 assigned to each node, a third node cannot get a pod CIDR if kube-controller-manager has ever been restarted. What you expected to happen: Once a third or fourth node joins, it gets a /24. How to reproduce it (as minimally and precisely as possible): 150% usage on the IPAM controller is not a good sign. Anything else we need to know?: During the creation of the rangeAllocator, Occupy() is called twice on the pod networks. The first time is before subscribing to the node informer. The second time is during the node informer callbacks because on subscribing it calls Add for all existing nodes. I will submit a proposed fix shortly. Environment: Kubernetes version (use ): 1.19.2
/sig network
/triage accepted thanks for finding the bug I 've introduced :)

t.Fatalf("%v: unexpected error when occupying CIDR %v: %v", tc.description, allocated, err) } } <del> if err := allocator.AllocateOrOccupyCIDR(tc.fakeNodeHandler.Existing[0]); err != nil { t.Errorf("%v: unexpected error in AllocateOrOccupyCIDR: %v", tc.description, err) </del> <ins> } updateCount := 0 for _, node := range tc.fakeNodeHandler.Existing { if node.Spec.PodCIDRs == nil { updateCount++ </ins> } <del> if err := waitForUpdatedNodeWithTimeout(tc.fakeNodeHandler, 1, wait.ForeverTestTimeout); err != nil { t.Fatalf("%v: timeout while waiting for Node update: %v", tc.description, err) </del> <ins> if err := allocator.AllocateOrOccupyCIDR(node); err != nil { t.Errorf("%v: unexpected error in AllocateOrOccupyCIDR: %v", tc.description, err) </ins> } } <ins> if updateCount != 1 { t.Fatalf("test error: all tests must update exactly one node") } if err := waitForUpdatedNodeWithTimeout(tc.fakeNodeHandler, updateCount, wait.ForeverTestTimeout); err != nil { t.Fatalf("%v: timeout while waiting for Node update: %v", tc.description, err) } </ins> if len(tc.expectedAllocatedCIDR) == 0 { // nothing further expected


Question:
What happened: Test create method for non-namespaced subresources generate invalid code. I have the following code generation annotations: which generated test method with generated fake client: What you expected to happen: It should be: How to reproduce it (as minimally and precisely as possible): Generate clients for non-namespaced resources with subresources (like scale) like this: /sig api-machinery /area code-generator /assign
The label(s) cannot be applied, because the repository doesn't have them <detailsIn response to : What happened: What you expected to happen: It should be: How to reproduce it (as minimally and precisely as possible): Generate clients for non-namespaced resources with subresources (like scale) like this: /sig api-machinery /area code-generator /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign
/triage accepted

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object // +genclient:method=GetScale,verb=get,subresource=scale,result=k8s.io/api/autoscaling/v1.Scale // +genclient:method=UpdateScale,verb=update,subresource=scale,input=k8s.io/api/autoscaling/v1.Scale,result=k8s.io/api/autoscaling/v1.Scale <ins> // +genclient:method=CreateScale,verb=create,subresource=scale,input=k8s.io/api/autoscaling/v1.Scale,result=k8s.io/api/autoscaling/v1.Scale </ins> type ClusterTestType struct { metav1.TypeMeta `json:",inline"`


Question:
What happened: Test create method for non-namespaced subresources generate invalid code. I have the following code generation annotations: which generated test method with generated fake client: What you expected to happen: It should be: How to reproduce it (as minimally and precisely as possible): Generate clients for non-namespaced resources with subresources (like scale) like this: /sig api-machinery /area code-generator /assign
The label(s) cannot be applied, because the repository doesn't have them <detailsIn response to : What happened: What you expected to happen: It should be: How to reproduce it (as minimally and precisely as possible): Generate clients for non-namespaced resources with subresources (like scale) like this: /sig api-machinery /area code-generator /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign
/triage accepted

Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions, subresources ...string) (result *v1.ClusterTestType, err error) GetScale(ctx context.Context, clusterTestTypeName string, options metav1.GetOptions) (*autoscalingv1.Scale, error) UpdateScale(ctx context.Context, clusterTestTypeName string, scale *autoscalingv1.Scale, opts metav1.UpdateOptions) (*autoscalingv1.Scale, error) <ins> CreateScale(ctx context.Context, clusterTestTypeName string, scale *autoscalingv1.Scale, opts metav1.CreateOptions) (*autoscalingv1.Scale, error) </ins> ClusterTestTypeExpansion }


Question:
What happened: Test create method for non-namespaced subresources generate invalid code. I have the following code generation annotations: which generated test method with generated fake client: What you expected to happen: It should be: How to reproduce it (as minimally and precisely as possible): Generate clients for non-namespaced resources with subresources (like scale) like this: /sig api-machinery /area code-generator /assign
The label(s) cannot be applied, because the repository doesn't have them <detailsIn response to : What happened: What you expected to happen: It should be: How to reproduce it (as minimally and precisely as possible): Generate clients for non-namespaced resources with subresources (like scale) like this: /sig api-machinery /area code-generator /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign
/triage accepted

Into(result) return } <ins> // CreateScale takes the representation of a scale and creates it.  Returns the server's representation of the scale, and an error, if there is any. func (c *clusterTestTypes) CreateScale(ctx context.Context, clusterTestTypeName string, scale *autoscalingv1.Scale, opts metav1.CreateOptions) (result *autoscalingv1.Scale, err error) { result = &autoscalingv1.Scale{} err = c.client.Post(). Resource("clustertesttypes"). Name(clusterTestTypeName). SubResource("scale"). VersionedParams(&opts, scheme.ParameterCodec). Body(scale). Do(ctx). Into(result) return } </ins>


Question:
What happened: Test create method for non-namespaced subresources generate invalid code. I have the following code generation annotations: which generated test method with generated fake client: What you expected to happen: It should be: How to reproduce it (as minimally and precisely as possible): Generate clients for non-namespaced resources with subresources (like scale) like this: /sig api-machinery /area code-generator /assign
The label(s) cannot be applied, because the repository doesn't have them <detailsIn response to : What happened: What you expected to happen: It should be: How to reproduce it (as minimally and precisely as possible): Generate clients for non-namespaced resources with subresources (like scale) like this: /sig api-machinery /area code-generator /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign
/triage accepted

} return obj.(*autoscalingv1.Scale), err } <ins> // CreateScale takes the representation of a scale and creates it.  Returns the server's representation of the scale, and an error, if there is any. func (c *FakeClusterTestTypes) CreateScale(ctx context.Context, clusterTestTypeName string, scale *autoscalingv1.Scale, opts v1.CreateOptions) (result *autoscalingv1.Scale, err error) { obj, err := c.Fake. Invokes(testing.NewRootCreateSubresourceAction(clustertesttypesResource, clusterTestTypeName, "scale", scale), &autoscalingv1.Scale{}) if obj == nil { return nil, err } return obj.(*autoscalingv1.Scale), err } </ins>


Question:
What happened: Test create method for non-namespaced subresources generate invalid code. I have the following code generation annotations: which generated test method with generated fake client: What you expected to happen: It should be: How to reproduce it (as minimally and precisely as possible): Generate clients for non-namespaced resources with subresources (like scale) like this: /sig api-machinery /area code-generator /assign
The label(s) cannot be applied, because the repository doesn't have them <detailsIn response to : What happened: What you expected to happen: It should be: How to reproduce it (as minimally and precisely as possible): Generate clients for non-namespaced resources with subresources (like scale) like this: /sig api-machinery /area code-generator /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign
/triage accepted

func (c *Fake$.type|publicPlural$) Create(ctx context.Context, $.type|private$Name string, $.inputType|private$ *$.inputType|raw$, opts $.CreateOptions|raw$) (result *$.resultType|raw$, err error) { obj, err := c.Fake. $if .namespaced$Invokes($.NewCreateSubresourceAction|raw$($.type|allLowercasePlural$Resource, $.type|private$Name, "$.subresourcePath$", c.ns, $.inputType|private$), &$.resultType|raw${}) <del> $else$Invokes($.NewRootCreateSubresourceAction|raw$($.type|allLowercasePlural$Resource, "$.subresourcePath$", $.inputType|private$), &$.resultType|raw${})$end$ </del> <ins> $else$Invokes($.NewRootCreateSubresourceAction|raw$($.type|allLowercasePlural$Resource, $.type|private$Name, "$.subresourcePath$", $.inputType|private$), &$.resultType|raw${})$end$ </ins> if obj == nil { return nil, err }


Question:
When configuring new behaviors section in HPA I two scenarios that I think are incorrect. First one: stabilizationWindowSeconds set up to 5 minutes for both scale up and scale down min replica count 2 Application consumes big amount of resources during warm-up 0 minute: current replica count: 2 1 minute: desired replica count: 3 - no scale up stabilization window worked 2 minute: desired replica count: 1 (!) - scale up to 3 (max number from previous recommendations during stabilization window period) ! Second one Stabilization window for scale up 0 Stabilization window for scale down 5 minutes min replica count 2 behavior set to scale up new pod in 2 minutes scaleUp: policies: type: Pods value: 1 periodSeconds: 120 What happens: 0 minute: 2 replicas 1 minute : performance spike scale up to 3 (desired value 4) 2 minute : no spike no scale up 3 minute: no spike scale up to 4(!) - supposedly desired value 1 and supposedly worked this way because of scale down window set to 5 minutes picking max value from that 5 minutes window that was 4 replicas Expected behaviors 1 Scenario no scale up for 5 minutes 2 Scenario correct measurement and no scale up when there is no need for it Environment: Kubernetes version (use ): client 1.19.3 Server 1.19.0 Cloud provider or hardware configuration: Azure aks
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig autoscaling
/assign josephburnett
The HPA controller keeps a flat history of recommendations for stabilization. ! When downscale stabilizing the maximum recommendation is selected. Purple is pre-stabilized recommendation. Green is final recommendation. Blue is max recommendation within window. ! When upscale stabilization the minimum recommendation is selected. ! However when both up and down scale stabilization is configured, the interpretation of the history changes depending on the direction of movement. Red is the undesired, actuated recommendation. ! What we want is to keep the stabilized recommendation within the envelope of the minimum and maximum over configured stabilization windows. We should only move when the envelope forces a move. ! We first stabilize up and down, and check if the current value fits within it. If it does (or the range is "inverted"), we do nothing. If it does not, we emit the "nearest possible" value, which will be lower or upper bound, depending on where the current size is in relation to the range. (thanks and for the paragraph above)

// - replaces old recommendation with the newest recommendation, // - returns {max,min} of recommendations that are not older than constraints.Scale{Up,Down}.DelaySeconds func (a *HorizontalController) stabilizeRecommendationWithBehaviors(args NormalizationArg) (int32, string, string) { <del> recommendation := args.DesiredReplicas </del> <ins> now := time.Now() </ins> foundOldSample := false oldSampleIndex := 0 <del> var scaleDelaySeconds int32 var reason, message string var betterRecommendation func(int32, int32) int32 </del> <ins> upRecommendation := args.DesiredReplicas upDelaySeconds := *args.ScaleUpBehavior.StabilizationWindowSeconds upCutoff := now.Add(-time.Second * time.Duration(upDelaySeconds)) </ins> <del> if args.DesiredReplicas >= args.CurrentReplicas { scaleDelaySeconds = *args.ScaleUpBehavior.StabilizationWindowSeconds betterRecommendation = min reason = "ScaleUpStabilized" message = "recent recommendations were lower than current one, applying the lowest recent recommendation" } else { scaleDelaySeconds = *args.ScaleDownBehavior.StabilizationWindowSeconds betterRecommendation = max reason = "ScaleDownStabilized" message = "recent recommendations were higher than current one, applying the highest recent recommendation" } maxDelaySeconds := max(*args.ScaleUpBehavior.StabilizationWindowSeconds, *args.ScaleDownBehavior.StabilizationWindowSeconds) obsoleteCutoff := time.Now().Add(-time.Second * time.Duration(maxDelaySeconds)) </del> <ins> downRecommendation := args.DesiredReplicas downDelaySeconds := *args.ScaleDownBehavior.StabilizationWindowSeconds downCutoff := now.Add(-time.Second * time.Duration(downDelaySeconds)) </ins> <del> cutoff := time.Now().Add(-time.Second * time.Duration(scaleDelaySeconds)) </del> <ins> // Calculate the upper and lower stabilization limits. </ins> for i, rec := range a.recommendations[args.Key] { <del> if rec.timestamp.After(cutoff) { recommendation = betterRecommendation(rec.recommendation, recommendation) </del> <ins> if rec.timestamp.After(upCutoff) { upRecommendation = min(rec.recommendation, upRecommendation) } if rec.timestamp.After(downCutoff) { downRecommendation = max(rec.recommendation, downRecommendation) </ins> } <del> if rec.timestamp.Before(obsoleteCutoff) { </del> <ins> if rec.timestamp.Before(upCutoff) && rec.timestamp.Before(downCutoff) { </ins> foundOldSample = true oldSampleIndex = i } } <ins> // Bring the recommendation to within the upper and lower limits (stabilize). recommendation := args.CurrentReplicas if recommendation < upRecommendation { recommendation = upRecommendation } if recommendation > downRecommendation { recommendation = downRecommendation } // Record the unstabilized recommendation. </ins> if foundOldSample { a.recommendations[args.Key][oldSampleIndex] = timestampedRecommendation{args.DesiredReplicas, time.Now()} } else { a.recommendations[args.Key] = append(a.recommendations[args.Key], timestampedRecommendation{args.DesiredReplicas, time.Now()}) } <ins> // Determine a human-friendly message. var reason, message string if args.DesiredReplicas >= args.CurrentReplicas { reason = "ScaleUpStabilized" message = "recent recommendations were lower than current one, applying the lowest recent recommendation" } else { reason = "ScaleDownStabilized" message = "recent recommendations were higher than current one, applying the highest recent recommendation" } </ins> return recommendation, reason, message }


Question:
When configuring new behaviors section in HPA I two scenarios that I think are incorrect. First one: stabilizationWindowSeconds set up to 5 minutes for both scale up and scale down min replica count 2 Application consumes big amount of resources during warm-up 0 minute: current replica count: 2 1 minute: desired replica count: 3 - no scale up stabilization window worked 2 minute: desired replica count: 1 (!) - scale up to 3 (max number from previous recommendations during stabilization window period) ! Second one Stabilization window for scale up 0 Stabilization window for scale down 5 minutes min replica count 2 behavior set to scale up new pod in 2 minutes scaleUp: policies: type: Pods value: 1 periodSeconds: 120 What happens: 0 minute: 2 replicas 1 minute : performance spike scale up to 3 (desired value 4) 2 minute : no spike no scale up 3 minute: no spike scale up to 4(!) - supposedly desired value 1 and supposedly worked this way because of scale down window set to 5 minutes picking max value from that 5 minutes window that was 4 replicas Expected behaviors 1 Scenario no scale up for 5 minutes 2 Scenario correct measurement and no scale up when there is no need for it Environment: Kubernetes version (use ): client 1.19.3 Server 1.19.0 Cloud provider or hardware configuration: Azure aks
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig autoscaling
/assign josephburnett
The HPA controller keeps a flat history of recommendations for stabilization. ! When downscale stabilizing the maximum recommendation is selected. Purple is pre-stabilized recommendation. Green is final recommendation. Blue is max recommendation within window. ! When upscale stabilization the minimum recommendation is selected. ! However when both up and down scale stabilization is configured, the interpretation of the history changes depending on the direction of movement. Red is the undesired, actuated recommendation. ! What we want is to keep the stabilized recommendation within the envelope of the minimum and maximum over configured stabilization windows. We should only move when the envelope forces a move. ! We first stabilize up and down, and check if the current value fits within it. If it does (or the range is "inverted"), we do nothing. If it does not, we emit the "nearest possible" value, which will be lower or upper bound, depending on where the current size is in relation to the range. (thanks and for the paragraph above)

} func TestNormalizeDesiredReplicasWithBehavior(t *testing.T) { <ins> now := time.Now() </ins> type TestCase struct { name                                string key                                 string


Question:
When configuring new behaviors section in HPA I two scenarios that I think are incorrect. First one: stabilizationWindowSeconds set up to 5 minutes for both scale up and scale down min replica count 2 Application consumes big amount of resources during warm-up 0 minute: current replica count: 2 1 minute: desired replica count: 3 - no scale up stabilization window worked 2 minute: desired replica count: 1 (!) - scale up to 3 (max number from previous recommendations during stabilization window period) ! Second one Stabilization window for scale up 0 Stabilization window for scale down 5 minutes min replica count 2 behavior set to scale up new pod in 2 minutes scaleUp: policies: type: Pods value: 1 periodSeconds: 120 What happens: 0 minute: 2 replicas 1 minute : performance spike scale up to 3 (desired value 4) 2 minute : no spike no scale up 3 minute: no spike scale up to 4(!) - supposedly desired value 1 and supposedly worked this way because of scale down window set to 5 minutes picking max value from that 5 minutes window that was 4 replicas Expected behaviors 1 Scenario no scale up for 5 minutes 2 Scenario correct measurement and no scale up when there is no need for it Environment: Kubernetes version (use ): client 1.19.3 Server 1.19.0 Cloud provider or hardware configuration: Azure aks
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig autoscaling
/assign josephburnett
The HPA controller keeps a flat history of recommendations for stabilization. ! When downscale stabilizing the maximum recommendation is selected. Purple is pre-stabilized recommendation. Green is final recommendation. Blue is max recommendation within window. ! When upscale stabilization the minimum recommendation is selected. ! However when both up and down scale stabilization is configured, the interpretation of the history changes depending on the direction of movement. Red is the undesired, actuated recommendation. ! What we want is to keep the stabilized recommendation within the envelope of the minimum and maximum over configured stabilization windows. We should only move when the envelope forces a move. ! We first stabilize up and down, and check if the current value fits within it. If it does (or the range is "inverted"), we do nothing. If it does not, we emit the "nearest possible" value, which will be lower or upper bound, depending on where the current size is in relation to the range. (thanks and for the paragraph above)

prenormalizedDesiredReplicas: 5, expectedStabilizedReplicas:   5, expectedRecommendations: []timestampedRecommendation{ <del> {5, time.Now()}, </del> <ins> {5, now}, </ins> }, }, { name: "simple scale down stabilization", key:  "", recommendations: []timestampedRecommendation{ <del> {4, time.Now().Add(-2 * time.Minute)}, {5, time.Now().Add(-1 * time.Minute)}}, </del> <ins> {4, now.Add(-2 * time.Minute)}, {5, now.Add(-1 * time.Minute)}}, </ins> currentReplicas:              100, prenormalizedDesiredReplicas: 3, expectedStabilizedReplicas:   5, expectedRecommendations: []timestampedRecommendation{ <del> {4, time.Now()}, {5, time.Now()}, {3, time.Now()}, </del> <ins> {4, now}, {5, now}, {3, now}, </ins> }, scaleDownStabilizationWindowSeconds: 60 * 3, },


Question:
When configuring new behaviors section in HPA I two scenarios that I think are incorrect. First one: stabilizationWindowSeconds set up to 5 minutes for both scale up and scale down min replica count 2 Application consumes big amount of resources during warm-up 0 minute: current replica count: 2 1 minute: desired replica count: 3 - no scale up stabilization window worked 2 minute: desired replica count: 1 (!) - scale up to 3 (max number from previous recommendations during stabilization window period) ! Second one Stabilization window for scale up 0 Stabilization window for scale down 5 minutes min replica count 2 behavior set to scale up new pod in 2 minutes scaleUp: policies: type: Pods value: 1 periodSeconds: 120 What happens: 0 minute: 2 replicas 1 minute : performance spike scale up to 3 (desired value 4) 2 minute : no spike no scale up 3 minute: no spike scale up to 4(!) - supposedly desired value 1 and supposedly worked this way because of scale down window set to 5 minutes picking max value from that 5 minutes window that was 4 replicas Expected behaviors 1 Scenario no scale up for 5 minutes 2 Scenario correct measurement and no scale up when there is no need for it Environment: Kubernetes version (use ): client 1.19.3 Server 1.19.0 Cloud provider or hardware configuration: Azure aks
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig autoscaling
/assign josephburnett
The HPA controller keeps a flat history of recommendations for stabilization. ! When downscale stabilizing the maximum recommendation is selected. Purple is pre-stabilized recommendation. Green is final recommendation. Blue is max recommendation within window. ! When upscale stabilization the minimum recommendation is selected. ! However when both up and down scale stabilization is configured, the interpretation of the history changes depending on the direction of movement. Red is the undesired, actuated recommendation. ! What we want is to keep the stabilized recommendation within the envelope of the minimum and maximum over configured stabilization windows. We should only move when the envelope forces a move. ! We first stabilize up and down, and check if the current value fits within it. If it does (or the range is "inverted"), we do nothing. If it does not, we emit the "nearest possible" value, which will be lower or upper bound, depending on where the current size is in relation to the range. (thanks and for the paragraph above)

name: "simple scale up stabilization", key:  "", recommendations: []timestampedRecommendation{ <del> {4, time.Now().Add(-2 * time.Minute)}, {5, time.Now().Add(-1 * time.Minute)}}, </del> <ins> {4, now.Add(-2 * time.Minute)}, {5, now.Add(-1 * time.Minute)}}, </ins> currentReplicas:              1, prenormalizedDesiredReplicas: 7, expectedStabilizedReplicas:   4, expectedRecommendations: []timestampedRecommendation{ <del> {4, time.Now()}, {5, time.Now()}, {7, time.Now()}, </del> <ins> {4, now}, {5, now}, {7, now}, </ins> }, scaleUpStabilizationWindowSeconds: 60 * 5, },


Question:
When configuring new behaviors section in HPA I two scenarios that I think are incorrect. First one: stabilizationWindowSeconds set up to 5 minutes for both scale up and scale down min replica count 2 Application consumes big amount of resources during warm-up 0 minute: current replica count: 2 1 minute: desired replica count: 3 - no scale up stabilization window worked 2 minute: desired replica count: 1 (!) - scale up to 3 (max number from previous recommendations during stabilization window period) ! Second one Stabilization window for scale up 0 Stabilization window for scale down 5 minutes min replica count 2 behavior set to scale up new pod in 2 minutes scaleUp: policies: type: Pods value: 1 periodSeconds: 120 What happens: 0 minute: 2 replicas 1 minute : performance spike scale up to 3 (desired value 4) 2 minute : no spike no scale up 3 minute: no spike scale up to 4(!) - supposedly desired value 1 and supposedly worked this way because of scale down window set to 5 minutes picking max value from that 5 minutes window that was 4 replicas Expected behaviors 1 Scenario no scale up for 5 minutes 2 Scenario correct measurement and no scale up when there is no need for it Environment: Kubernetes version (use ): client 1.19.3 Server 1.19.0 Cloud provider or hardware configuration: Azure aks
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig autoscaling
/assign josephburnett
The HPA controller keeps a flat history of recommendations for stabilization. ! When downscale stabilizing the maximum recommendation is selected. Purple is pre-stabilized recommendation. Green is final recommendation. Blue is max recommendation within window. ! When upscale stabilization the minimum recommendation is selected. ! However when both up and down scale stabilization is configured, the interpretation of the history changes depending on the direction of movement. Red is the undesired, actuated recommendation. ! What we want is to keep the stabilized recommendation within the envelope of the minimum and maximum over configured stabilization windows. We should only move when the envelope forces a move. ! We first stabilize up and down, and check if the current value fits within it. If it does (or the range is "inverted"), we do nothing. If it does not, we emit the "nearest possible" value, which will be lower or upper bound, depending on where the current size is in relation to the range. (thanks and for the paragraph above)

name: "no scale down stabilization", key:  "", recommendations: []timestampedRecommendation{ <del> {1, time.Now().Add(-2 * time.Minute)}, {2, time.Now().Add(-1 * time.Minute)}}, </del> <ins> {1, now.Add(-2 * time.Minute)}, {2, now.Add(-1 * time.Minute)}}, </ins> currentReplicas:              100, // to apply scaleDown delay we should have current > desired prenormalizedDesiredReplicas: 3, expectedStabilizedReplicas:   3, expectedRecommendations: []timestampedRecommendation{ <del> {1, time.Now()}, {2, time.Now()}, {3, time.Now()}, </del> <ins> {1, now}, {2, now}, {3, now}, </ins> }, scaleUpStabilizationWindowSeconds: 60 * 5, },


Question:
When configuring new behaviors section in HPA I two scenarios that I think are incorrect. First one: stabilizationWindowSeconds set up to 5 minutes for both scale up and scale down min replica count 2 Application consumes big amount of resources during warm-up 0 minute: current replica count: 2 1 minute: desired replica count: 3 - no scale up stabilization window worked 2 minute: desired replica count: 1 (!) - scale up to 3 (max number from previous recommendations during stabilization window period) ! Second one Stabilization window for scale up 0 Stabilization window for scale down 5 minutes min replica count 2 behavior set to scale up new pod in 2 minutes scaleUp: policies: type: Pods value: 1 periodSeconds: 120 What happens: 0 minute: 2 replicas 1 minute : performance spike scale up to 3 (desired value 4) 2 minute : no spike no scale up 3 minute: no spike scale up to 4(!) - supposedly desired value 1 and supposedly worked this way because of scale down window set to 5 minutes picking max value from that 5 minutes window that was 4 replicas Expected behaviors 1 Scenario no scale up for 5 minutes 2 Scenario correct measurement and no scale up when there is no need for it Environment: Kubernetes version (use ): client 1.19.3 Server 1.19.0 Cloud provider or hardware configuration: Azure aks
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig autoscaling
/assign josephburnett
The HPA controller keeps a flat history of recommendations for stabilization. ! When downscale stabilizing the maximum recommendation is selected. Purple is pre-stabilized recommendation. Green is final recommendation. Blue is max recommendation within window. ! When upscale stabilization the minimum recommendation is selected. ! However when both up and down scale stabilization is configured, the interpretation of the history changes depending on the direction of movement. Red is the undesired, actuated recommendation. ! What we want is to keep the stabilized recommendation within the envelope of the minimum and maximum over configured stabilization windows. We should only move when the envelope forces a move. ! We first stabilize up and down, and check if the current value fits within it. If it does (or the range is "inverted"), we do nothing. If it does not, we emit the "nearest possible" value, which will be lower or upper bound, depending on where the current size is in relation to the range. (thanks and for the paragraph above)

name: "no scale up stabilization", key:  "", recommendations: []timestampedRecommendation{ <del> {4, time.Now().Add(-2 * time.Minute)}, {5, time.Now().Add(-1 * time.Minute)}}, </del> <ins> {4, now.Add(-2 * time.Minute)}, {5, now.Add(-1 * time.Minute)}}, </ins> currentReplicas:              1, // to apply scaleDown delay we should have current > desired prenormalizedDesiredReplicas: 3, expectedStabilizedReplicas:   3, expectedRecommendations: []timestampedRecommendation{ <del> {4, time.Now()}, {5, time.Now()}, {3, time.Now()}, </del> <ins> {4, now}, {5, now}, {3, now}, </ins> }, scaleDownStabilizationWindowSeconds: 60 * 5, },


Question:
When configuring new behaviors section in HPA I two scenarios that I think are incorrect. First one: stabilizationWindowSeconds set up to 5 minutes for both scale up and scale down min replica count 2 Application consumes big amount of resources during warm-up 0 minute: current replica count: 2 1 minute: desired replica count: 3 - no scale up stabilization window worked 2 minute: desired replica count: 1 (!) - scale up to 3 (max number from previous recommendations during stabilization window period) ! Second one Stabilization window for scale up 0 Stabilization window for scale down 5 minutes min replica count 2 behavior set to scale up new pod in 2 minutes scaleUp: policies: type: Pods value: 1 periodSeconds: 120 What happens: 0 minute: 2 replicas 1 minute : performance spike scale up to 3 (desired value 4) 2 minute : no spike no scale up 3 minute: no spike scale up to 4(!) - supposedly desired value 1 and supposedly worked this way because of scale down window set to 5 minutes picking max value from that 5 minutes window that was 4 replicas Expected behaviors 1 Scenario no scale up for 5 minutes 2 Scenario correct measurement and no scale up when there is no need for it Environment: Kubernetes version (use ): client 1.19.3 Server 1.19.0 Cloud provider or hardware configuration: Azure aks
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig autoscaling
/assign josephburnett
The HPA controller keeps a flat history of recommendations for stabilization. ! When downscale stabilizing the maximum recommendation is selected. Purple is pre-stabilized recommendation. Green is final recommendation. Blue is max recommendation within window. ! When upscale stabilization the minimum recommendation is selected. ! However when both up and down scale stabilization is configured, the interpretation of the history changes depending on the direction of movement. Red is the undesired, actuated recommendation. ! What we want is to keep the stabilized recommendation within the envelope of the minimum and maximum over configured stabilization windows. We should only move when the envelope forces a move. ! We first stabilize up and down, and check if the current value fits within it. If it does (or the range is "inverted"), we do nothing. If it does not, we emit the "nearest possible" value, which will be lower or upper bound, depending on where the current size is in relation to the range. (thanks and for the paragraph above)

name: "no scale down stabilization, reuse recommendation element", key:  "", recommendations: []timestampedRecommendation{ <del> {10, time.Now().Add(-10 * time.Minute)}, {9, time.Now().Add(-9 * time.Minute)}}, </del> <ins> {10, now.Add(-10 * time.Minute)}, {9, now.Add(-9 * time.Minute)}}, </ins> currentReplicas:              100, // to apply scaleDown delay we should have current > desired prenormalizedDesiredReplicas: 3, expectedStabilizedReplicas:   3, expectedRecommendations: []timestampedRecommendation{ <del> {10, time.Now()}, {3, time.Now()}, </del> <ins> {10, now}, {3, now}, </ins> }, }, { name: "no scale up stabilization, reuse recommendation element", key:  "", recommendations: []timestampedRecommendation{ <del> {10, time.Now().Add(-10 * time.Minute)}, {9, time.Now().Add(-9 * time.Minute)}}, </del> <ins> {10, now.Add(-10 * time.Minute)}, {9, now.Add(-9 * time.Minute)}}, </ins> currentReplicas:              1, prenormalizedDesiredReplicas: 100, expectedStabilizedReplicas:   100, expectedRecommendations: []timestampedRecommendation{ <del> {10, time.Now()}, {100, time.Now()}, </del> <ins> {10, now}, {100, now}, </ins> }, }, { name: "scale down stabilization, reuse one of obsolete recommendation element", key:  "", recommendations: []timestampedRecommendation{ <del> {10, time.Now().Add(-10 * time.Minute)}, {4, time.Now().Add(-1 * time.Minute)}, {5, time.Now().Add(-2 * time.Minute)}, {9, time.Now().Add(-9 * time.Minute)}}, </del> <ins> {10, now.Add(-10 * time.Minute)}, {4, now.Add(-1 * time.Minute)}, {5, now.Add(-2 * time.Minute)}, {9, now.Add(-9 * time.Minute)}}, </ins> currentReplicas:              100, prenormalizedDesiredReplicas: 3, expectedStabilizedReplicas:   5, expectedRecommendations: []timestampedRecommendation{ <del> {10, time.Now()}, {4, time.Now()}, {5, time.Now()}, {3, time.Now()}, </del> <ins> {10, now}, {4, now}, {5, now}, {3, now}, </ins> }, scaleDownStabilizationWindowSeconds: 3 * 60, },


Question:
When configuring new behaviors section in HPA I two scenarios that I think are incorrect. First one: stabilizationWindowSeconds set up to 5 minutes for both scale up and scale down min replica count 2 Application consumes big amount of resources during warm-up 0 minute: current replica count: 2 1 minute: desired replica count: 3 - no scale up stabilization window worked 2 minute: desired replica count: 1 (!) - scale up to 3 (max number from previous recommendations during stabilization window period) ! Second one Stabilization window for scale up 0 Stabilization window for scale down 5 minutes min replica count 2 behavior set to scale up new pod in 2 minutes scaleUp: policies: type: Pods value: 1 periodSeconds: 120 What happens: 0 minute: 2 replicas 1 minute : performance spike scale up to 3 (desired value 4) 2 minute : no spike no scale up 3 minute: no spike scale up to 4(!) - supposedly desired value 1 and supposedly worked this way because of scale down window set to 5 minutes picking max value from that 5 minutes window that was 4 replicas Expected behaviors 1 Scenario no scale up for 5 minutes 2 Scenario correct measurement and no scale up when there is no need for it Environment: Kubernetes version (use ): client 1.19.3 Server 1.19.0 Cloud provider or hardware configuration: Azure aks
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig autoscaling
/assign josephburnett
The HPA controller keeps a flat history of recommendations for stabilization. ! When downscale stabilizing the maximum recommendation is selected. Purple is pre-stabilized recommendation. Green is final recommendation. Blue is max recommendation within window. ! When upscale stabilization the minimum recommendation is selected. ! However when both up and down scale stabilization is configured, the interpretation of the history changes depending on the direction of movement. Red is the undesired, actuated recommendation. ! What we want is to keep the stabilized recommendation within the envelope of the minimum and maximum over configured stabilization windows. We should only move when the envelope forces a move. ! We first stabilize up and down, and check if the current value fits within it. If it does (or the range is "inverted"), we do nothing. If it does not, we emit the "nearest possible" value, which will be lower or upper bound, depending on where the current size is in relation to the range. (thanks and for the paragraph above)

name: "scale up stabilization, reuse one of obsolete recommendation element", key:  "", recommendations: []timestampedRecommendation{ <del> {10, time.Now().Add(-100 * time.Minute)}, {6, time.Now().Add(-1 * time.Minute)}, {5, time.Now().Add(-2 * time.Minute)}, {9, time.Now().Add(-3 * time.Minute)}}, </del> <ins> {10, now.Add(-100 * time.Minute)}, {6, now.Add(-1 * time.Minute)}, {5, now.Add(-2 * time.Minute)}, {9, now.Add(-3 * time.Minute)}}, </ins> currentReplicas:              1, prenormalizedDesiredReplicas: 100, expectedStabilizedReplicas:   5, expectedRecommendations: []timestampedRecommendation{ <del> {100, time.Now()}, {6, time.Now()}, {5, time.Now()}, {9, time.Now()}, </del> <ins> {100, now}, {6, now}, {5, now}, {9, now}, </ins> }, scaleUpStabilizationWindowSeconds: 300, <ins> }, { name: "scale up and down stabilization, do not scale up when prenormalized rec goes down", key:  "", recommendations: []timestampedRecommendation{ {2, now.Add(-100 * time.Minute)}, {3, now.Add(-3 * time.Minute)}, }, currentReplicas:                     2, prenormalizedDesiredReplicas:        1, expectedStabilizedReplicas:          2, scaleUpStabilizationWindowSeconds:   300, scaleDownStabilizationWindowSeconds: 300, }, { name: "scale up and down stabilization, do not scale down when prenormalized rec goes up", key:  "", recommendations: []timestampedRecommendation{ {2, now.Add(-100 * time.Minute)}, {1, now.Add(-3 * time.Minute)}, }, currentReplicas:                     2, prenormalizedDesiredReplicas:        3, expectedStabilizedReplicas:          2, scaleUpStabilizationWindowSeconds:   300, scaleDownStabilizationWindowSeconds: 300, </ins> }, } for _, tc := range tests {


Question:
When configuring new behaviors section in HPA I two scenarios that I think are incorrect. First one: stabilizationWindowSeconds set up to 5 minutes for both scale up and scale down min replica count 2 Application consumes big amount of resources during warm-up 0 minute: current replica count: 2 1 minute: desired replica count: 3 - no scale up stabilization window worked 2 minute: desired replica count: 1 (!) - scale up to 3 (max number from previous recommendations during stabilization window period) ! Second one Stabilization window for scale up 0 Stabilization window for scale down 5 minutes min replica count 2 behavior set to scale up new pod in 2 minutes scaleUp: policies: type: Pods value: 1 periodSeconds: 120 What happens: 0 minute: 2 replicas 1 minute : performance spike scale up to 3 (desired value 4) 2 minute : no spike no scale up 3 minute: no spike scale up to 4(!) - supposedly desired value 1 and supposedly worked this way because of scale down window set to 5 minutes picking max value from that 5 minutes window that was 4 replicas Expected behaviors 1 Scenario no scale up for 5 minutes 2 Scenario correct measurement and no scale up when there is no need for it Environment: Kubernetes version (use ): client 1.19.3 Server 1.19.0 Cloud provider or hardware configuration: Azure aks
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig autoscaling
/assign josephburnett
The HPA controller keeps a flat history of recommendations for stabilization. ! When downscale stabilizing the maximum recommendation is selected. Purple is pre-stabilized recommendation. Green is final recommendation. Blue is max recommendation within window. ! When upscale stabilization the minimum recommendation is selected. ! However when both up and down scale stabilization is configured, the interpretation of the history changes depending on the direction of movement. Red is the undesired, actuated recommendation. ! What we want is to keep the stabilized recommendation within the envelope of the minimum and maximum over configured stabilization windows. We should only move when the envelope forces a move. ! We first stabilize up and down, and check if the current value fits within it. If it does (or the range is "inverted"), we do nothing. If it does not, we emit the "nearest possible" value, which will be lower or upper bound, depending on where the current size is in relation to the range. (thanks and for the paragraph above)

} r, _, _ := hc.stabilizeRecommendationWithBehaviors(arg) assert.Equal(t, tc.expectedStabilizedReplicas, r, "expected replicas do not match") <del> if !assert.Len(t, hc.recommendations[tc.key], len(tc.expectedRecommendations), "stored recommendations differ in length") { return } for i, r := range hc.recommendations[tc.key] { expectedRecommendation := tc.expectedRecommendations[i] assert.Equal(t, expectedRecommendation.recommendation, r.recommendation, "stored recommendation differs at position %d", i) </del> <ins> if tc.expectedRecommendations != nil { if !assert.Len(t, hc.recommendations[tc.key], len(tc.expectedRecommendations), "stored recommendations differ in length") { return } for i, r := range hc.recommendations[tc.key] { expectedRecommendation := tc.expectedRecommendations[i] assert.Equal(t, expectedRecommendation.recommendation, r.recommendation, "stored recommendation differs at position %d", i) } </ins> } }) }


Question:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Tried to create an ingress with a backend serviceport pointing to a service with a port name with length 15 chars that is valid from the service point of view. But failed because the validation for Ingress does not match the one for service. Service Validation (uses DNS Label): ) Ingress Service Port Validation (uses IANA format like for container ports): What you expected to happen: Service port validation should match Ingress ServicePort validation How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Client Version: version.Info{Major:"1", Minor:"19", GitVersion:"v1.19.4", GitCommit:"", GitTreeState:"clean", BuildDate:"2020-11-16T17:18:52Z", GoVersion:"go1.15.4", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"18", GitVersion:"v1.18.10", GitCommit:"", GitTreeState:"clean", BuildDate:"2020-10-15T01:43:56Z", GoVersion:"go1.13.15", Compiler:"gc", Platform:"linux/amd64"} Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig network
It's not friendly to users who create service with long service port name, to create ingress with port reference. Although, the port name of service is editable. Maybe, we can add warning message for service creation and update when the port name is not a valid ingress port name.
I'm fine with a doc change mentioning the restriction similar to the restriction in the doc for container port () where it explicitly says about the IANASVCNAME constraint
/triage accepted /assign

// ServiceBackendPort is the service port being referenced. type ServiceBackendPort struct { // Name is the name of the port on the Service. <ins> // This must be an IANA_SVC_NAME (following RFC6335). </ins> // This is a mutually exclusive setting with "Number". // +optional Name string


Question:
What happened: The latest release of the debian-base image, v2.1.3, contains a vulnerable version of the apt package. CVE-2020- impacts apt version 1.8.2.1. What you expected to happen: Image should be updated to use at least apt 1.8.2.2 to clear the vulnerability. How to reproduce it (as minimally and precisely as possible):
/sig release
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
kube-proxy is using debain as base image. /assign Let me check other components
checked the base image in and I seems that no new debian or ubuntu official image is built for this CVE ?
That is correct, the purpose of this issue is to get a new image built and released, one which contains a resolution for the CVE.
per comments of @ tianon in Will update it once debain releases it.

# # $1 - server architecture kube::build::get_docker_wrapped_binaries() { <del> local debian_iptables_version=buster-v1.3.0 </del> <ins> local debian_iptables_version=buster-v1.4.0 </ins> local go_runner_version=buster-v2.2.2 ### If you change any of these lists, please also update DOCKERIZED_BINARIES ### in build/BUILD. And kube::golang::server_image_targets


Question:
What happened: The latest release of the debian-base image, v2.1.3, contains a vulnerable version of the apt package. CVE-2020- impacts apt version 1.8.2.1. What you expected to happen: Image should be updated to use at least apt 1.8.2.2 to clear the vulnerability. How to reproduce it (as minimally and precisely as possible):
/sig release
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
kube-proxy is using debain as base image. /assign Let me check other components
checked the base image in and I seems that no new debian or ubuntu official image is built for this CVE ?
That is correct, the purpose of this issue is to get a new image built and released, one which contains a resolution for the CVE.
per comments of @ tianon in Will update it once debain releases it.

# Base images - name: "k8s.gcr.io/debian-base: dependents" <del> version: buster-v1.2.0 </del> <ins> version: buster-v1.3.0 </ins> refPaths: - path: build/workspace.bzl match: tag =


Question:
What happened: The latest release of the debian-base image, v2.1.3, contains a vulnerable version of the apt package. CVE-2020- impacts apt version 1.8.2.1. What you expected to happen: Image should be updated to use at least apt 1.8.2.2 to clear the vulnerability. How to reproduce it (as minimally and precisely as possible):
/sig release
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
kube-proxy is using debain as base image. /assign Let me check other components
checked the base image in and I seems that no new debian or ubuntu official image is built for this CVE ?
That is correct, the purpose of this issue is to get a new image built and released, one which contains a resolution for the CVE.
per comments of @ tianon in Will update it once debain releases it.

match: BASEIMAGE?=k8s.gcr.io/build-image/debian-base-s390x:[a-zA-Z]+-v((([0-9]+).([0-9]+).([0-9]+)(?:-([0-9a-zA-Z-]+(?:.[0-9a-zA-Z-]+)*))?)(?:+([0-9a-zA-Z-]+(?:.[0-9a-zA-Z-]+)*))?) - name: "k8s.gcr.io/debian-iptables: dependents" <del> version: buster-v1.3.0 </del> <ins> version: buster-v1.4.0 </ins> refPaths: - path: build/common.sh match: debian_iptables_version=


Question:
What happened: The latest release of the debian-base image, v2.1.3, contains a vulnerable version of the apt package. CVE-2020- impacts apt version 1.8.2.1. What you expected to happen: Image should be updated to use at least apt 1.8.2.2 to clear the vulnerability. How to reproduce it (as minimally and precisely as possible):
/sig release
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
kube-proxy is using debain as base image. /assign Let me check other components
checked the base image in and I seems that no new debian or ubuntu official image is built for this CVE ?
That is correct, the purpose of this issue is to get a new image built and released, one which contains a resolution for the CVE.
per comments of @ tianon in Will update it once debain releases it.

# Use skopeo to find these values: https://github.com/containers/skopeo # # Example <del> # Manifest: skopeo inspect docker://gcr.io/k8s-staging-build-image/debian-base:buster-v1.2.0 # Arches: skopeo inspect --raw docker://gcr.io/k8s-staging-build-image/debian-base:buster-v1.2.0 </del> <ins> # Manifest: skopeo inspect docker://gcr.io/k8s-staging-build-image/debian-base:buster-v1.3.0 # Arches: skopeo inspect --raw docker://gcr.io/k8s-staging-build-image/debian-base:buster-v1.3.0 </ins> _DEBIAN_BASE_DIGEST = { <del> "manifest": "sha256:ea668d3febd312e0edfbbdab6bd7d86448ddc8fddb484e20ec76b36a7aeac04c", "amd64": "sha256:2f3e61995bcd4b3a1a0aef49e4a7a6817c978031734b09df2aaaa28181898b0e", "arm": "sha256:d1073dcf8f1d55fbbd297e5b280375b6f276ea83a08a25fd59dc4f3ca43c6d50", "arm64": "sha256:dadcff1ab81177de4914f6db0e7d78a52e525daf7a1efb246cb3545de5e818d1", "ppc64le": "sha256:bfb24dc0d1e71e1deb0f04a078fadf2c94070266746b1b5acc4e739aa57d5601", "s390x": "sha256:cfe6a3508b7ee198cb5a0b3a62e0981676b1dfa4b3049f36398d03e6bd35a801", </del> <ins> "manifest": "sha256:d66137c7c362d1026dca670d1ff4c25e5b0770e8ace87ac3d008d52e4b0db338", "amd64": "sha256:a5ab028d9a730b78af9abb15b5db9b2e6f82448ab269d6f3a07d1834c571ccc6", "arm": "sha256:94e611363760607366ca1fed9375105b6c5fc922ab1249869b708690ca13733c", "arm64": "sha256:83512c52d44587271cd0f355c0a9a7e6c2412ddc66b8a8eb98f994277297a72f", "ppc64le": "sha256:9c8284b2797b114ebe8f3f1b2b5817a9c7f07f3f82513c49a30e6191a1acc1fc", "s390x": "sha256:d617637dd4df0bc1cfa524fae3b4892cfe57f7fec9402ad8dfa28e38e82ec688", </ins> } # Use skopeo to find these values: https://github.com/containers/skopeo # # Example <del> # Manifest: skopeo inspect docker://gcr.io/k8s-staging-build-image/debian-iptables:buster-v1.3.0 # Arches: skopeo inspect --raw docker://gcr.io/k8s-staging-build-image/debian-iptables:buster-v1.3.0 </del> <ins> # Manifest: skopeo inspect docker://gcr.io/k8s-staging-build-image/debian-iptables:buster-v1.4.0 # Arches: skopeo inspect --raw docker://gcr.io/k8s-staging-build-image/debian-iptables:buster-v1.4.0 </ins> _DEBIAN_IPTABLES_DIGEST = { <del> "manifest": "sha256:4c9410a4ee555dcb0e8b7bd6fc77c65ac400f7c5bd4555df8187630efaea6ea4", "amd64": "sha256:e30919918299988b318f0208e7fd264dee21a6be9d74bbd9f7fc15e78eade9b4", "arm": "sha256:bf59578f532bfd3378c4a713eeb14cf0dbed224d5ad03f549165f8d853997ca4", "arm64": "sha256:3d7ede6013b0516f1ec3852590895d4a7b6ec8f5e15bebc1a55237bba4538da2", "ppc64le": "sha256:ebd3bb280f8da8fc6a7158b7c2fc59b4552487bffd95d0b5ac1b190aff7b0fd9", "s390x": "sha256:a73b94aace7a571f36149aa917d4c7ee13453ed24a31f26549dd13b386dae4c1", </del> <ins> "manifest": "sha256:87f97cf2b62eb107871ee810f204ccde41affb70b29883aa898e93df85dea0f0", "amd64": "sha256:da837f39cf3af78adb796c0caa9733449ae99e51cf624590c328e4c9951ace7a", "arm": "sha256:bb6677337a4dbc3e578a3e87642d99be740dea391dc5e8987f04211c5e23abcd", "arm64": "sha256:6ad4717d69db2cc47bc2efc91cebb96ba736be1de49e62e0deffdbaf0fa2318c", "ppc64le": "sha256:168ccfeb861239536826a26da24ab5f68bb5349d7439424b7008b01e8f6534fc", "s390x": "sha256:5a88d4f4c29bac5b5c93195059b928f7346be11d0f0f7f6da0e14c0bfdbd1362", </ins> } # Use skopeo to find these values: https://github.com/containers/skopeo


Question:
What happened: The latest release of the debian-base image, v2.1.3, contains a vulnerable version of the apt package. CVE-2020- impacts apt version 1.8.2.1. What you expected to happen: Image should be updated to use at least apt 1.8.2.2 to clear the vulnerability. How to reproduce it (as minimally and precisely as possible):
/sig release
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
kube-proxy is using debain as base image. /assign Let me check other components
checked the base image in and I seems that no new debian or ubuntu official image is built for this CVE ?
That is correct, the purpose of this issue is to get a new image built and released, one which contains a resolution for the CVE.
per comments of @ tianon in Will update it once debain releases it.

registry = "k8s.gcr.io/build-image", repository = "debian-base", # Ensure the digests above are updated to match a new tag <del> tag = "buster-v1.2.0",  # ignored, but kept here for documentation </del> <ins> tag = "buster-v1.3.0",  # ignored, but kept here for documentation </ins> ) container_pull(


Question:
What happened: The latest release of the debian-base image, v2.1.3, contains a vulnerable version of the apt package. CVE-2020- impacts apt version 1.8.2.1. What you expected to happen: Image should be updated to use at least apt 1.8.2.2 to clear the vulnerability. How to reproduce it (as minimally and precisely as possible):
/sig release
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
kube-proxy is using debain as base image. /assign Let me check other components
checked the base image in and I seems that no new debian or ubuntu official image is built for this CVE ?
That is correct, the purpose of this issue is to get a new image built and released, one which contains a resolution for the CVE.
per comments of @ tianon in Will update it once debain releases it.

registry = "k8s.gcr.io/build-image", repository = "debian-iptables", # Ensure the digests above are updated to match a new tag <del> tag = "buster-v1.3.0",  # ignored, but kept here for documentation </del> <ins> tag = "buster-v1.4.0",  # ignored, but kept here for documentation </ins> ) def etcd_tarballs():


Question:
What happened: The latest release of the debian-base image, v2.1.3, contains a vulnerable version of the apt package. CVE-2020- impacts apt version 1.8.2.1. What you expected to happen: Image should be updated to use at least apt 1.8.2.2 to clear the vulnerability. How to reproduce it (as minimally and precisely as possible):
/sig release
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
kube-proxy is using debain as base image. /assign Let me check other components
checked the base image in and I seems that no new debian or ubuntu official image is built for this CVE ?
That is correct, the purpose of this issue is to get a new image built and released, one which contains a resolution for the CVE.
per comments of @ tianon in Will update it once debain releases it.

# REVISION provides a version number fo this image and all it's bundled # artifacts. It should start at zero for each LATEST_ETCD_VERSION and increment # for each revision of this image at that etcd version. <del> REVISION?=1 </del> <ins> REVISION?=2 </ins> # IMAGE_TAG Uniquely identifies k8s.gcr.io/etcd docker image with a tag of the form "<etcd-version>-<revision>". IMAGE_TAG=$(LATEST_ETCD_VERSION)-$(REVISION)


Question:
What happened: The latest release of the debian-base image, v2.1.3, contains a vulnerable version of the apt package. CVE-2020- impacts apt version 1.8.2.1. What you expected to happen: Image should be updated to use at least apt 1.8.2.2 to clear the vulnerability. How to reproduce it (as minimally and precisely as possible):
/sig release
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
kube-proxy is using debain as base image. /assign Let me check other components
checked the base image in and I seems that no new debian or ubuntu official image is built for this CVE ?
That is correct, the purpose of this issue is to get a new image built and released, one which contains a resolution for the CVE.
per comments of @ tianon in Will update it once debain releases it.

TEMP_DIR:=$(shell mktemp -d) ifeq ($(ARCH),amd64) <del> BASEIMAGE?=k8s.gcr.io/build-image/debian-base:buster-v1.2.0 </del> <ins> BASEIMAGE?=k8s.gcr.io/build-image/debian-base:buster-v1.3.0 </ins> endif ifeq ($(ARCH),arm) <del> BASEIMAGE?=k8s.gcr.io/build-image/debian-base-arm:buster-v1.2.0 </del> <ins> BASEIMAGE?=k8s.gcr.io/build-image/debian-base-arm:buster-v1.3.0 </ins> endif ifeq ($(ARCH),arm64) <del> BASEIMAGE?=k8s.gcr.io/build-image/debian-base-arm64:buster-v1.2.0 </del> <ins> BASEIMAGE?=k8s.gcr.io/build-image/debian-base-arm64:buster-v1.3.0 </ins> endif ifeq ($(ARCH),ppc64le) <del> BASEIMAGE?=k8s.gcr.io/build-image/debian-base-ppc64le:buster-v1.2.0 </del> <ins> BASEIMAGE?=k8s.gcr.io/build-image/debian-base-ppc64le:buster-v1.3.0 </ins> endif ifeq ($(ARCH),s390x) <del> BASEIMAGE?=k8s.gcr.io/build-image/debian-base-s390x:buster-v1.2.0 </del> <ins> BASEIMAGE?=k8s.gcr.io/build-image/debian-base-s390x:buster-v1.3.0 </ins> endif RUNNERIMAGE?=gcr.io/distroless/static:latest


Question:
What happened: The latest release of the debian-base image, v2.1.3, contains a vulnerable version of the apt package. CVE-2020- impacts apt version 1.8.2.1. What you expected to happen: Image should be updated to use at least apt 1.8.2.2 to clear the vulnerability. How to reproduce it (as minimally and precisely as possible):
/sig release
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
kube-proxy is using debain as base image. /assign Let me check other components
checked the base image in and I seems that no new debian or ubuntu official image is built for this CVE ?
That is correct, the purpose of this issue is to get a new image built and released, one which contains a resolution for the CVE.
per comments of @ tianon in Will update it once debain releases it.

configs[CheckMetadataConcealment] = Config{e2eRegistry, "metadata-concealment", "1.2"} configs[CudaVectorAdd] = Config{e2eRegistry, "cuda-vector-add", "1.0"} configs[CudaVectorAdd2] = Config{e2eRegistry, "cuda-vector-add", "2.0"} <del> configs[DebianIptables] = Config{buildImageRegistry, "debian-iptables", "buster-v1.3.0"} </del> <ins> configs[DebianIptables] = Config{buildImageRegistry, "debian-iptables", "buster-v1.4.0"} </ins> configs[EchoServer] = Config{e2eRegistry, "echoserver", "2.2"} configs[Etcd] = Config{gcRegistry, "etcd", "3.4.13-0"} configs[GlusterDynamicProvisioner] = Config{dockerGluster, "glusterdynamic-provisioner", "v1.0"}


Question:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: ListByResourceGroup method in the azure diskClient mock makes a call to Delete What you expected to happen: ListByResourceGroup method in the azure diskClient mock should call ListByResourceGroup How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: azure OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign
/sig cloud-provider-azure
The label(s) cannot be applied, because the repository doesn't have them <detailsIn response to : /sig cloud-provider-azure Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig storage
/area provider/azure
fixed by /close
Closing this issue. <detailsIn response to : fixed by /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details

// ListByResourceGroup indicates an expected call of ListByResourceGroup func (mr *MockInterfaceMockRecorder) ListByResourceGroup(ctx, resourceGroupName interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() <del> return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListByResourceGroup", reflect.TypeOf((*MockInterface)(nil).Delete), ctx, resourceGroupName) </del> <ins> return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListByResourceGroup", reflect.TypeOf((*MockInterface)(nil).ListByResourceGroup), ctx, resourceGroupName) </ins> }


Question:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Cluster was upgraded from 1.19.5 to 1.20.1 using kubeadm. After that, kube-apiserver cpu usage went from 50% to 300%. What you expected to happen: Expected that apiserver has the same cpu usage. How to reproduce it (as minimally and precisely as possible): Update a cluster from 1.19.5 to 1.20.1. Anything else we need to know?: The cluster is running on sbc hardware, thats why the original 50% cpu usage, and thats why 300% cpu usage is noticeable and causes service disruption. Environment: Kubernetes version (use ): (the apiserver is rolled back to 1.19.5)Cloud provider or hardware configuration: bare metal on SBC hardware. OS (e.g: ):Kernel (e.g. ):Install tools: kubeadm Network plugin and version (if this is a network-related bug): kube-router Others: Please check profiling of each apiserver.
/sig api-machinery
please do a and show the contents here. (hide sensitive IPs if needed). this is the static pod for the kube-apiserver. also, anything interesting in the logs of the api server?
On another cluster I could not reproduce the issue. But, on the original one, it happens again if i switch to the newer apiserver image. So, now I am bisecting apiserver to find the commit since it behaves this way, hopefully that will help. I could not spot any interesting in the logs. I will be able to provide more after the bisecting procedure.
post the above output whenever possible.
The static pod manifest has been moved to avoid kubelet starting it: Still bisecting, trying to find the cause. May it relate to the go1.15 upgrade? Building the apiserver with go1.15.6 seems to be working, while building with 1.15.5 results in a wrong binary. But, still I am working to confirm this.
With apiserver built with go1.15.5 I get 300% cpu usage most of the time, and the following log entries appear:
unclear whether we have perf tests to catch similar problems: /sig scalability for other readers, go1.15.5 is the version used to build 1.20: /sig auth this and similar come when doing rest storage on various apis. e.g.
/retitle performance regression: 1.20 apiserver has high cpu usage
/kind regression
Definitely it seems that the binary built with go1.15.5 is broken. It ends up on 300% cpu usage and with log messages posted above. Note that I am building apiserver this way: Building with go1.15.6 is better, even if not all of the time. Perhaps I am on the wrong way?
this is a detail i've missed from the OP. since we don't have k8s ARM tests, it is likely why we are not catching this and it could be a golang compiler issue on ARM only. in the past we have seen some weird issues from ARM users, that have hinted about compiler bugs.
Sorry for that, I did not mention that explicitly. If it turns out to be a compiler bug, will it trigger a golang upgrade for k8s images?
I will make a build with golang 1.15.2 too and post the results later.
if we know that a certain golang version is buggy, we can update a specific k8s version to use a golang version that works. in this case the latest 1.20.x k8s can be updated to not use 1.15.5.
Simply compiling 1.20.1 with golang 1.15.2 still produces misbehaving binaries. So, I will do a bisecting again with building binaries as documented, and try to find the bad commit this way. This will take longer on my hardware.
So, apiserver behavior is a bit random regarding this issue. What is true that on my hardware (orange pi pc) the apiserver needs around 85 cpu seconds to boot up, after that it consumes ~10% cpu time. When a build is bad, apiserver passes the 85 cpu seconds and still uses more cpu than 10%, sometimes even 300%. So, based on this, bisecting shows is first bad commit.
Reverting that commit on 1.20.1 makes apiserver behaving as before. Consistently it finishes its bootup in 90 seconds, and then cpu usage gets to its normal 10-50%, it depends on load-balancer.
By the way, the problem is not just high cpu usage, but during bootup and high cpu usage, apiserver stays in not ready state, thus kubelet restarts it after failed liveness probes. So it may cause service outage.
we had long discussions in the past where RPI users complained about the kubeadm probe defaults. we claimed something else is not right if the probes are failing, so i think you are confirming that. i can ping the release team and api-machinery about this ticket, but note post the holidays responses will be slow.
/retitle performance regression on ARM: 1.20 apiserver has high cpu usage
I suspect, that the mentioned commit fixes things in a way that it actually places more cpu load, I mean expression evaluates to true more often. The attached profiling data may be a proof for this.
It seems that on slower hardware, operations are distributed into more sample periods (time.Millisecond), which makes more calls to the expensive metrics.Observe functions. Before the mentioned fix, due to quantization bug, those calls were not made every time.Milliseconds. Now they are, and profiling data proves this. Can we raise the samplePeriod, can it be made configurable?
sig api machinery can provide feedback on this.
is that accurate? my reading of the fix is that the quantize method returns results with the same granularity. The bug that was fixed was that quantization could return numbers that resulted in negative calculations if wall clock time changed, and now it respects the monotonic clock.
Perhaps not. That was just my assumption. But still, profiling data shows (at least for me) that metric.Observe calls take most of the time, while reverting the mentioned commit solves my problem. Somehow less calls are made to them.
/assign
We have numerous K3s users reporting perf issues (and crashes in the cronjob controller) on armv7l. I can reproduce it locally on my Pi nodes as well. I hadn't gotten around to digging into it yet, so I'm glad someone else has done so. We also had issues getting arm to build properly starting with Kubernetes 1.19.4; some builds would crash with what looked like a corrupted stack on armv7l unless we set for the build. See: Prior to doing that it was fairly random if a build would work or not; changing unrelated bits of code (ie bumping a vendored module version) would give us a clean build, and then changing something else would break it, and then changing a 3rd thing would give us a clean build again. It felt very much like golang arm toolchain issues. This sounds similar to what reported with good vs bad builds.
I've a little debugging: And I got strange results, a snippet from stdout: Check out the high on the 4th line. That causes a long-long loop to start. And there are more of such lines. If I refactor the code to remove the inline func() and do the Unlock() and the end of the block, this behavior goes away, apiserver goes back to normal. May it be a compiler bug?
earlier you've mentioned: Building the apiserver with go1.15.6 seems to be working, while building with 1.15.5 results in a wrong binary But, still I am working to confirm this. did you confirm that the newer toolchain does not produce code that exhibits the problem? seems likely. in the past i've seen at least one compiler bug-like report from a k8s user (on ARM)
Unfortunately, I think I got confused regarding compilers. Now it seems that I got the same results with go1.15.2, 1.15.5, 1.15.6. So, now I think it does not matter which toolchain I use.
I've applied the following patch on release-1.20: And this produces strange results: It can be observed that from the third line, the function receives strange values. Later normal values appear, but sometimes the calculated is a large positive integer, which triggers a long running loop. Unfortunately, I am not familiar with golang's internals, and could not reproduce this in a few lines of code. But indeed, it seems that something weird is happening. If I place a before the call to , then the function gets the correct value passed all the time. How to proceed?
I think this is an arm32 int64 division overflow/underflow issue? I'm currently testing a minor change to confirm/deny this hunch, but building for arm takes forever so it's still running.
It looks like the time value is being corrupted when being passed into quantize somehow, would be nice to try and reproduce with a very small executable. k3s has a long history of arm32 issues like this that I suspect is a core golang issue, but hard to reproduce and isolate, arbitrary code changes could change the result.
you could try running something like this, which exercises the local logic of the old and new quantize methods and checks that they output roughly similar values (some drift is expected because one uses wall clock and the other uses monotonic clock) I was able to run this for several minutes on linux/amd64 with no problematic values output
Yes, it looks like parameter corruption rather than a division problem. very strange.
Yeah my simple change to the quantize didn't make any difference. That combined with the strange output from the debug printing does indicate some sort of corruption to the when time struct that's being passed in, rather than bad math on the parameter.
Presumably all the unit tests are passing on the arch & OS you are using.
But there are no unit tests focused on that. It's on the to-do list.
If we created unit tests for this file, would somebody be running them on the arch & OS involved here?
I put in a check in quantize to grab a stack trace if when is before t0:
This seems like papering over a compiler problem, but this fixes it for me:
: that is interesting info. I would prefer to not merge that change, we have another mystery where the monotonic time seems to go backwards, and I would hate to lose half the debug info from that. I will add unit tests for the sample-and-watermark histograms; can somebody please run them on the arch & OS involved here?
The time going backwards stuff all seems to be related to VM time sync, isn't it? In this case the compiler seems to be doing something that's corrupting the time struct out from under the anonymous function; the fact that it appears to be going backwards is not (as far as I can tell) due to host time actually jumping backwards into the dark ages with a weird TZ offset. FWIW I considered moving the Before check-and-err into the quantize function, but it wasn't necessary for a minimal fix for this issue don't I left it alone. I have a couple armhf (Pi3b/Pi4b) systems I can run tests on, assuming it's not too difficult to set up.
I am not sure whether the monotonic time going backwards is the same bug. It was observed on Intel arch. I suspect another way to paper over the compiler bug would be to switch the argument to from pass-by-value to pass-by-reference. I suspect a compiler bug would be a valuable thing to find. I am working on unit tests for this package. They will involve dependency on the Prometheus go client library, but nothing more. Will that be simple enough?
The incidents I have personally seen or helped users with time sync issues have all been VMs on Intel. If there are non-virtualized platforms with this issue it'd be interesting to see. Simply preventing the sampling time bucket from going backwards seems reasonable, which appears to be what the fix was attempting to do. Changing it to a pointer is an interesting idea. I can give it a try later this evening. I was figuring I'd have to run the entire Kubernetes test suite, so anything less than that is great!
Your small program is running on arm for minutes now without any errors. It needs more to trigger the compiler bug.
I will be able, though I will need instructions how to do that.
Here is a typescript of me running the new unit test: (the test does exercise the case of the clock going backwards a little) That's using the branch in
On ARM, I get similar results:
/remove-sig auth
: dang. Can you try it a few times? If that does not provoke the problem, perhaps you can edit the test to do a lot more iterations? You can also cut down on the warning noise by making a much smaller fraction of the cases have time going backwards.
Running multiple times results in some failing. Sometimes there are cases when 3 consecutive runs are successful. So basically, the test triggers the issue, but not deterministically. Even with iterations set to 1000, there are still successful test runs.
Is that close enough to submit as a compiler bug report? BTW, what do the failures look like?
is the logic non-flaky on AMD64? for a compiler bug, the maintainers are likely to request the failing code disassembled, so that they don't have to clone k8s, or a minimal code example that fails.
I am getting non-flaky results on this: I think I have gotten the test to be about as simple as makes sense for a kube unit test. I will have to leave the compiler bug hunting to people more familiar with ARM.
I tried to make a stdlib-only file that exercises the same call patterns w.r.t. variable scope as the original at what does that do when run on ARM?
Interesting, running the tests by on an rpi4 (using arm32) I am able to trigger two different issues: clock is NOT monotonic and can go backwards time IS corrupted periodically when being passed into I am also not able to trigger any issues running those tests on Darwin (using x86_64). However, running the stand-alone go file by I am not able to trigger any problems on the rpi4
: sorry if it wasn't clear, but the test introduced by uses a fake clock that is set backwards in some cases just to test how the metrics code reacts to that.
Same here, on an OrangePI armv7 board, the single stdlib-only test does not produce any errors.
Later I will do a test on arm64 as well.
woops, ah thank you I had to add to use the flag with go test to see those messages in Darwin and missed that part of the test code. At least the arm32 corruption issue is reproducible with that unit test, but I suspect that attempts to reduce the code space for the problem may cause it to go away, so maybe an issue of memory management or executable code layout. Unfortunately seems like debugging tools can be hard to come by on arm32 (eg no delve), am open to any ideas or suggestions on how to dig into it more
BTW, I updated to always log an info message explaining what's up with the warnings.
i don't know how to handle go mod imports with and some of the golang intrinsics are weird, but you could instead: build the unit test as a binary with the binary and for the object code of interest log a ticket @ golang and share the dissasembly you might also be able to use gdb over the test binary - set some breakpoints and try to catch what instruction is stomping the register / stack address that holds the value that ends up being bogus:
Thanks a bunch Was just looking into the compiled test, that is a great guide & will submit an issue with golang.
On arm64 the tests succeeded multiple times, no failures.
Compiled the test code for i386 resulted in: On i386 the test code triggers the bug too.
Different GOGC settings results in different test failure rate:
I am able to reproduce that. ... and if I build without the variant GOARCH then the test passes all 300 times.
When I merge and , the problem is visibly gone on the 386 arch on linux.
should we merge the pass-by-reference bit of in the meantime until golang/go is resolved?
I would say that we could wait a few days for response on the bug report. On the other side, I dont know how fast a resolution will get into a stable golang release, and what is needed for k8s to be built using that. Can these take a long time?
yes, I would not expect it faster than O(weeks), potentially O(months) if the golang investigation takes a while
It would be great if the fix could be merged and backported in time for 1.20.2; as it currently stands 1.20 is basically unusable on arm.
BTW, contains only the pass-by-reference change and logging of an error if there is a wild jump anyway; I would recommend taking all of that PR.
the deadline for cherry-picks to be merged for 1.20.2 is tomorrow
i wonder if it's also related w/ , the latter issue seems to be stambling on the memory pressure tho. but cpu&memory consumption can be also connected.
I think yes, I've seen similar log messages.
For what it is worth this seems like a very high impact bug, as it can potentially effect 64-bit architectures also. The following example of the bug that was submitted as part of the fix also fails on Darwin/x86_64: It seems like the best mitigation until a new version of golang is released is to disable optimizations for the compiled binaries.
I created as a cut-down alternative to , containing only the work-around and not the checking and logging for the argument passing problem.
: What do you recommend for how to proceed on this? We should probably get something into 1.20.2
agree. I'm fine with a temporary commented pass-by-reference workaround.
/triage accepted
ping , On arm64 I got the same error like . and kube-api-server restart many times.
the error you are getting looks like a connection problem, so not clear if related. the bug discussed here is in golang. so if we update to the latest PATCH version golang in our k8s PATCH versions it will be resolved.
thanks. but In my 3 masters kubernetse clusters. Only one master got this Error . others works well. this confused me. this problem like connection working with ipv6 not ipv4?
you can try asking in the #sig-network channel on slack or log a new issue.

} func (saw *sampleAndWaterMarkHistograms) innerSet(updateXOrX1 func()) { <del> var when time.Time var whenInt int64 var acc sampleAndWaterMarkAccumulator var wellOrdered bool func() { </del> <ins> when, whenInt, acc, wellOrdered := func() (time.Time, int64, sampleAndWaterMarkAccumulator, bool) { </ins> saw.Lock() defer saw.Unlock() <del> when = saw.clock.Now() whenInt = saw.quantize(when) acc = saw.sampleAndWaterMarkAccumulator wellOrdered = !when.Before(acc.lastSet) </del> <ins> // Moved these variables here to tiptoe around https://github.com/golang/go/issues/43570 for #97685 when := saw.clock.Now() whenInt := saw.quantize(when) acc := saw.sampleAndWaterMarkAccumulator wellOrdered := !when.Before(acc.lastSet) </ins> updateXOrX1() saw.relX = saw.x / saw.x1 if wellOrdered {