Question: }) // TODO: add tests that cover deployment.Spec.MinReadySeconds once we solved clock-skew issues // See https://github.com/kubernetes/kubernetes/issues/29229 <ins> ginkgo.It("should run the lifecycle of a Deployment", func() { deploymentResource := schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} testNamespaceName := f.Namespace.Name testDeploymentName := "test-deployment" testDeploymentInitialImage := imageutils.GetE2EImage(imageutils.Agnhost) testDeploymentPatchImage := imageutils.GetE2EImage(imageutils.Pause) testDeploymentUpdateImage := imageutils.GetE2EImage(imageutils.Httpd) testDeploymentDefaultReplicas := int32(3) testDeploymentMinimumReplicas := int32(1) testDeploymentNoReplicas := int32(0) testDeploymentLabels := map[string]string{"test-deployment-static": "true"} testDeploymentLabelsFlat := "test-deployment-static=true" testDeploymentLabelSelectors := metav1.LabelSelector{ MatchLabels: testDeploymentLabels, } w := &cache.ListWatch{ WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { options.LabelSelector = testDeploymentLabelsFlat return f.ClientSet.AppsV1().Deployments(testNamespaceName).Watch(context.TODO(), options) }, } deploymentsList, err := f.ClientSet.AppsV1().Deployments("").List(context.TODO(), metav1.ListOptions{LabelSelector: testDeploymentLabelsFlat}) framework.ExpectNoError(err, "failed to list Deployments") ginkgo.By("creating a Deployment") testDeployment := appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name:   testDeploymentName, Labels: map[string]string{"test-deployment-static": "true"}, }, Spec: appsv1.DeploymentSpec{ Replicas: &testDeploymentDefaultReplicas, Selector: &testDeploymentLabelSelectors, Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: testDeploymentLabelSelectors.MatchLabels, }, Spec: v1.PodSpec{ Containers: []v1.Container{{ Name:  testDeploymentName, Image: testDeploymentInitialImage, }}, }, }, }, } _, err = f.ClientSet.AppsV1().Deployments(testNamespaceName).Create(context.TODO(), &testDeployment, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create Deployment %v in namespace %v", testDeploymentName, testNamespaceName) ginkgo.By("waiting for Deployment to be created") ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, err = watchtools.Until(ctx, deploymentsList.ResourceVersion, w, func(event watch.Event) (bool, error) { switch event.Type { case watch.Added: if deployment, ok := event.Object.(*appsv1.Deployment); ok { found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.Labels["test-deployment-static"] == "true" return found, nil } default: framework.Logf("observed event type %v", event.Type) } return false, nil }) framework.ExpectNoError(err, "failed to see %v event", watch.Added) ginkgo.By("waiting for all Replicas to be Ready") ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, err = watchtools.Until(ctx, deploymentsList.ResourceVersion, w, func(event watch.Event) (bool, error) { if deployment, ok := event.Object.(*appsv1.Deployment); ok { found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.Labels["test-deployment-static"] == "true" && deployment.Status.AvailableReplicas == testDeploymentDefaultReplicas && deployment.Status.ReadyReplicas == testDeploymentDefaultReplicas return found, nil } return false, nil }) framework.ExpectNoError(err, "failed to see replicas of %v in namespace %v scale to requested amount of %v", testDeployment.Name, testNamespaceName, testDeploymentDefaultReplicas) ginkgo.By("patching the Deployment") deploymentPatch, err := json.Marshal(map[string]interface{}{ "metadata": map[string]interface{}{ "labels": map[string]string{"test-deployment": "patched"}, }, "spec": map[string]interface{}{ "replicas": testDeploymentMinimumReplicas, "template": map[string]interface{}{ "spec": map[string]interface{}{ "containers": [1]map[string]interface{}{{ "name":    testDeploymentName, "image":   testDeploymentPatchImage, "command": []string{"/bin/sleep", "100000"}, }}, }, }, }, }) framework.ExpectNoError(err, "failed to Marshal Deployment JSON patch") _, err = f.ClientSet.AppsV1().Deployments(testNamespaceName).Patch(context.TODO(), testDeploymentName, types.StrategicMergePatchType, []byte(deploymentPatch), metav1.PatchOptions{}) framework.ExpectNoError(err, "failed to patch Deployment") ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, err = watchtools.Until(ctx, deploymentsList.ResourceVersion, w, func(event watch.Event) (bool, error) { switch event.Type { case watch.Modified: if deployment, ok := event.Object.(*appsv1.Deployment); ok { found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.Labels["test-deployment-static"] == "true" return found, nil } default: framework.Logf("observed event type %v", event.Type) } return false, nil }) framework.ExpectNoError(err, "failed to see %v event", watch.Modified) ginkgo.By("waiting for Replicas to scale") ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, err = watchtools.Until(ctx, deploymentsList.ResourceVersion, w, func(event watch.Event) (bool, error) { if deployment, ok := event.Object.(*appsv1.Deployment); ok { found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.Labels["test-deployment-static"] == "true" && deployment.Status.AvailableReplicas == testDeploymentMinimumReplicas && deployment.Status.ReadyReplicas == testDeploymentMinimumReplicas && deployment.Spec.Template.Spec.Containers[0].Image == testDeploymentPatchImage return found, nil } return false, nil }) framework.ExpectNoError(err, "failed to see replicas of %v in namespace %v scale to requested amount of %v", testDeployment.Name, testNamespaceName, testDeploymentMinimumReplicas) ginkgo.By("listing Deployments") deploymentsList, err = f.ClientSet.AppsV1().Deployments("").List(context.TODO(), metav1.ListOptions{LabelSelector: testDeploymentLabelsFlat}) framework.ExpectNoError(err, "failed to list Deployments") foundDeployment := false for _, deploymentItem := range deploymentsList.Items { if deploymentItem.ObjectMeta.Name == testDeploymentName && deploymentItem.ObjectMeta.Namespace == testNamespaceName && deploymentItem.ObjectMeta.Labels["test-deployment-static"] == "true" { foundDeployment = true break } } framework.ExpectEqual(foundDeployment, true, "unable to find the Deployment in list", deploymentsList) ginkgo.By("updating the Deployment") testDeploymentUpdate := testDeployment testDeploymentUpdate.ObjectMeta.Labels["test-deployment"] = "updated" testDeploymentUpdate.Spec.Template.Spec.Containers[0].Image = testDeploymentUpdateImage testDeploymentDefaultReplicasPointer := &testDeploymentDefaultReplicas testDeploymentUpdate.Spec.Replicas = testDeploymentDefaultReplicasPointer testDeploymentUpdateUnstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&testDeploymentUpdate) framework.ExpectNoError(err, "failed to convert to unstructured") testDeploymentUpdateUnstructured := unstructuredv1.Unstructured{ Object: testDeploymentUpdateUnstructuredMap, } // currently this hasn't been able to hit the endpoint replaceAppsV1NamespacedDeploymentStatus _, err = dc.Resource(deploymentResource).Namespace(testNamespaceName).Update(context.TODO(), &testDeploymentUpdateUnstructured, metav1.UpdateOptions{}) //, "status") framework.ExpectNoError(err, "failed to update the DeploymentStatus") ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, err = watchtools.Until(ctx, deploymentsList.ResourceVersion, w, func(event watch.Event) (bool, error) { switch event.Type { case watch.Modified: if deployment, ok := event.Object.(*appsv1.Deployment); ok { found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.Labels["test-deployment-static"] == "true" return found, nil } default: framework.Logf("observed event type %v", event.Type) } return false, nil }) framework.ExpectNoError(err, "failed to see %v event", watch.Modified) ginkgo.By("fetching the DeploymentStatus") deploymentGetUnstructured, err := dc.Resource(deploymentResource).Namespace(testNamespaceName).Get(context.TODO(), testDeploymentName, metav1.GetOptions{}, "status") framework.ExpectNoError(err, "failed to fetch the Deployment") deploymentGet := appsv1.Deployment{} err = runtime.DefaultUnstructuredConverter.FromUnstructured(deploymentGetUnstructured.Object, &deploymentGet) framework.ExpectNoError(err, "failed to convert the unstructured response to a Deployment") framework.ExpectEqual(deploymentGet.Spec.Template.Spec.Containers[0].Image, testDeploymentUpdateImage, "failed to update image") framework.ExpectEqual(deploymentGet.ObjectMeta.Labels["test-deployment"], "updated", "failed to update labels") ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, err = watchtools.Until(ctx, deploymentsList.ResourceVersion, w, func(event watch.Event) (bool, error) { if deployment, ok := event.Object.(*appsv1.Deployment); ok { found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.Labels["test-deployment-static"] == "true" && deployment.Status.AvailableReplicas == testDeploymentDefaultReplicas && deployment.Status.ReadyReplicas == testDeploymentDefaultReplicas return found, nil } return false, nil }) framework.ExpectNoError(err, "failed to see replicas of %v in namespace %v scale to requested amount of %v", testDeployment.Name, testNamespaceName, testDeploymentDefaultReplicas) ginkgo.By("patching the DeploymentStatus") deploymentStatusPatch, err := json.Marshal(map[string]interface{}{ "metadata": map[string]interface{}{ "labels": map[string]string{"test-deployment": "patched-status"}, }, "status": map[string]interface{}{ "readyReplicas": testDeploymentNoReplicas, }, }) framework.ExpectNoError(err, "failed to Marshal Deployment JSON patch") dc.Resource(deploymentResource).Namespace(testNamespaceName).Patch(context.TODO(), testDeploymentName, types.StrategicMergePatchType, []byte(deploymentStatusPatch), metav1.PatchOptions{}, "status") ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, err = watchtools.Until(ctx, deploymentsList.ResourceVersion, w, func(event watch.Event) (bool, error) { switch event.Type { case watch.Modified: if deployment, ok := event.Object.(*appsv1.Deployment); ok { found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.Labels["test-deployment-static"] == "true" return found, nil } default: framework.Logf("observed event type %v", event.Type) } return false, nil }) framework.ExpectNoError(err, "failed to see %v event", watch.Modified) ginkgo.By("fetching the DeploymentStatus") deploymentGetUnstructured, err = dc.Resource(deploymentResource).Namespace(testNamespaceName).Get(context.TODO(), testDeploymentName, metav1.GetOptions{}, "status") framework.ExpectNoError(err, "failed to fetch the DeploymentStatus") deploymentGet = appsv1.Deployment{} err = runtime.DefaultUnstructuredConverter.FromUnstructured(deploymentGetUnstructured.Object, &deploymentGet) framework.ExpectNoError(err, "failed to convert the unstructured response to a Deployment") framework.ExpectEqual(deploymentGet.Spec.Template.Spec.Containers[0].Image, testDeploymentUpdateImage, "failed to update image") framework.ExpectEqual(deploymentGet.ObjectMeta.Labels["test-deployment"], "updated", "failed to update labels") ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, err = watchtools.Until(ctx, deploymentsList.ResourceVersion, w, func(event watch.Event) (bool, error) { if deployment, ok := event.Object.(*appsv1.Deployment); ok { found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.Labels["test-deployment-static"] == "true" && deployment.Status.AvailableReplicas == testDeploymentDefaultReplicas && deployment.Status.ReadyReplicas == testDeploymentDefaultReplicas && deployment.Spec.Template.Spec.Containers[0].Image == testDeploymentUpdateImage return found, nil } return false, nil }) framework.ExpectNoError(err, "failed to see replicas of %v in namespace %v scale to requested amount of %v", testDeployment.Name, testNamespaceName, testDeploymentDefaultReplicas) ginkgo.By("deleting the Deployment") err = f.ClientSet.AppsV1().Deployments(testNamespaceName).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: testDeploymentLabelsFlat}) framework.ExpectNoError(err, "failed to delete Deployment via collection") ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, err = watchtools.Until(ctx, deploymentsList.ResourceVersion, w, func(event watch.Event) (bool, error) { switch event.Type { case watch.Deleted: if deployment, ok := event.Object.(*appsv1.Deployment); ok { found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.Labels["test-deployment-static"] == "true" return found, nil } default: framework.Logf("observed event type %v", event.Type) } return false, nil }) framework.ExpectNoError(err, "failed to see %v event", watch.Deleted) }) </ins> }) func failureTrap(c clientset.Interface, ns string) {

[x] Original approval issue: [x] Test pr: [x] Two weeks soak end date: 19 November 2020 This issue is created to allow edit by /wip /hold According to this APIsnoop query, there are still some remaining Deployment endpoints which are untested. a Deployment with a static label the Deployment with a new Label and updated data the Deployment to ensure it's patched all Deployments in all Namespaces find the Deployment(1) ensure that the Deployment is found and is patched Namespaced Deployment(1) via a Collection with a LabelSelector Discover useragents: List endpoints hit by the test: Display endpoint coverage change: If a test with these calls gets merged, test coverage will go up by 5 points* This test is also created with the goal of conformance promotion.
/sig testing /sig architecture /area conformance
BobyMcbobs commented in original Issue: Unsure if the endpoint replaceAppsV1NamespacedDeploymentStatus is able to be hit. When this endpoint is attempted to be updated the data after retrieving the update resource doesn't reflect the values in the update
/assign
When using the appv1.Deployment structure instead of maps of interfaces to build the structure in a patch of the Deployment, this error comes up. Still in progress yment-static":"true"}: does not match template , spec.selector: Invalid value: "null": field is immutable]
replaceAppsV1NamespacedDeploymentStatus to test. Increase cover by 1 Endpoint
/reopen PR reverted by due to flaky test
Reopened this issue. <detailsIn response to : /reopen PR reverted by due to flaky test Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/reopen Soaking, ready for promotion 19 November 2020
Reopened this issue. <detailsIn response to : /reopen Soaking, ready for promotion 19 November 2020 Instructions for interacting with me using PR comments are available .


Question: }) // TODO: add tests that cover deployment.Spec.MinReadySeconds once we solved clock-skew issues // See https://github.com/kubernetes/kubernetes/issues/29229 <ins> ginkgo.It("should run the lifecycle of a Deployment", func() { deploymentResource := schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} testNamespaceName := f.Namespace.Name testDeploymentName := "test-deployment" testDeploymentInitialImage := imageutils.GetE2EImage(imageutils.Agnhost) testDeploymentPatchImage := imageutils.GetE2EImage(imageutils.Pause) testDeploymentUpdateImage := imageutils.GetE2EImage(imageutils.Httpd) testDeploymentDefaultReplicas := int32(3) testDeploymentMinimumReplicas := int32(1) testDeploymentNoReplicas := int32(0) testDeploymentLabels := map[string]string{"test-deployment-static": "true"} testDeploymentLabelsFlat := "test-deployment-static=true" testDeploymentLabelSelectors := metav1.LabelSelector{ MatchLabels: testDeploymentLabels, } w := &cache.ListWatch{ WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { options.LabelSelector = testDeploymentLabelsFlat return f.ClientSet.AppsV1().Deployments(testNamespaceName).Watch(context.TODO(), options) }, } deploymentsList, err := f.ClientSet.AppsV1().Deployments("").List(context.TODO(), metav1.ListOptions{LabelSelector: testDeploymentLabelsFlat}) framework.ExpectNoError(err, "failed to list Deployments") ginkgo.By("creating a Deployment") testDeployment := appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name:   testDeploymentName, Labels: map[string]string{"test-deployment-static": "true"}, }, Spec: appsv1.DeploymentSpec{ Replicas: &testDeploymentDefaultReplicas, Selector: &testDeploymentLabelSelectors, Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: testDeploymentLabelSelectors.MatchLabels, }, Spec: v1.PodSpec{ Containers: []v1.Container{{ Name:  testDeploymentName, Image: testDeploymentInitialImage, }}, }, }, }, } _, err = f.ClientSet.AppsV1().Deployments(testNamespaceName).Create(context.TODO(), &testDeployment, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create Deployment %v in namespace %v", testDeploymentName, testNamespaceName) ginkgo.By("waiting for Deployment to be created") ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, err = watchtools.Until(ctx, deploymentsList.ResourceVersion, w, func(event watch.Event) (bool, error) { switch event.Type { case watch.Added: if deployment, ok := event.Object.(*appsv1.Deployment); ok { found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.Labels["test-deployment-static"] == "true" return found, nil } default: framework.Logf("observed event type %v", event.Type) } return false, nil }) framework.ExpectNoError(err, "failed to see %v event", watch.Added) ginkgo.By("waiting for all Replicas to be Ready") ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, err = watchtools.Until(ctx, deploymentsList.ResourceVersion, w, func(event watch.Event) (bool, error) { if deployment, ok := event.Object.(*appsv1.Deployment); ok { found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.Labels["test-deployment-static"] == "true" && deployment.Status.AvailableReplicas == testDeploymentDefaultReplicas && deployment.Status.ReadyReplicas == testDeploymentDefaultReplicas return found, nil } return false, nil }) framework.ExpectNoError(err, "failed to see replicas of %v in namespace %v scale to requested amount of %v", testDeployment.Name, testNamespaceName, testDeploymentDefaultReplicas) ginkgo.By("patching the Deployment") deploymentPatch, err := json.Marshal(map[string]interface{}{ "metadata": map[string]interface{}{ "labels": map[string]string{"test-deployment": "patched"}, }, "spec": map[string]interface{}{ "replicas": testDeploymentMinimumReplicas, "template": map[string]interface{}{ "spec": map[string]interface{}{ "containers": [1]map[string]interface{}{{ "name":    testDeploymentName, "image":   testDeploymentPatchImage, "command": []string{"/bin/sleep", "100000"}, }}, }, }, }, }) framework.ExpectNoError(err, "failed to Marshal Deployment JSON patch") _, err = f.ClientSet.AppsV1().Deployments(testNamespaceName).Patch(context.TODO(), testDeploymentName, types.StrategicMergePatchType, []byte(deploymentPatch), metav1.PatchOptions{}) framework.ExpectNoError(err, "failed to patch Deployment") ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, err = watchtools.Until(ctx, deploymentsList.ResourceVersion, w, func(event watch.Event) (bool, error) { switch event.Type { case watch.Modified: if deployment, ok := event.Object.(*appsv1.Deployment); ok { found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.Labels["test-deployment-static"] == "true" return found, nil } default: framework.Logf("observed event type %v", event.Type) } return false, nil }) framework.ExpectNoError(err, "failed to see %v event", watch.Modified) ginkgo.By("waiting for Replicas to scale") ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, err = watchtools.Until(ctx, deploymentsList.ResourceVersion, w, func(event watch.Event) (bool, error) { if deployment, ok := event.Object.(*appsv1.Deployment); ok { found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.Labels["test-deployment-static"] == "true" && deployment.Status.AvailableReplicas == testDeploymentMinimumReplicas && deployment.Status.ReadyReplicas == testDeploymentMinimumReplicas && deployment.Spec.Template.Spec.Containers[0].Image == testDeploymentPatchImage return found, nil } return false, nil }) framework.ExpectNoError(err, "failed to see replicas of %v in namespace %v scale to requested amount of %v", testDeployment.Name, testNamespaceName, testDeploymentMinimumReplicas) ginkgo.By("listing Deployments") deploymentsList, err = f.ClientSet.AppsV1().Deployments("").List(context.TODO(), metav1.ListOptions{LabelSelector: testDeploymentLabelsFlat}) framework.ExpectNoError(err, "failed to list Deployments") foundDeployment := false for _, deploymentItem := range deploymentsList.Items { if deploymentItem.ObjectMeta.Name == testDeploymentName && deploymentItem.ObjectMeta.Namespace == testNamespaceName && deploymentItem.ObjectMeta.Labels["test-deployment-static"] == "true" { foundDeployment = true break } } framework.ExpectEqual(foundDeployment, true, "unable to find the Deployment in list", deploymentsList) ginkgo.By("updating the Deployment") testDeploymentUpdate := testDeployment testDeploymentUpdate.ObjectMeta.Labels["test-deployment"] = "updated" testDeploymentUpdate.Spec.Template.Spec.Containers[0].Image = testDeploymentUpdateImage testDeploymentDefaultReplicasPointer := &testDeploymentDefaultReplicas testDeploymentUpdate.Spec.Replicas = testDeploymentDefaultReplicasPointer testDeploymentUpdateUnstructuredMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&testDeploymentUpdate) framework.ExpectNoError(err, "failed to convert to unstructured") testDeploymentUpdateUnstructured := unstructuredv1.Unstructured{ Object: testDeploymentUpdateUnstructuredMap, } // currently this hasn't been able to hit the endpoint replaceAppsV1NamespacedDeploymentStatus _, err = dc.Resource(deploymentResource).Namespace(testNamespaceName).Update(context.TODO(), &testDeploymentUpdateUnstructured, metav1.UpdateOptions{}) //, "status") framework.ExpectNoError(err, "failed to update the DeploymentStatus") ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, err = watchtools.Until(ctx, deploymentsList.ResourceVersion, w, func(event watch.Event) (bool, error) { switch event.Type { case watch.Modified: if deployment, ok := event.Object.(*appsv1.Deployment); ok { found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.Labels["test-deployment-static"] == "true" return found, nil } default: framework.Logf("observed event type %v", event.Type) } return false, nil }) framework.ExpectNoError(err, "failed to see %v event", watch.Modified) ginkgo.By("fetching the DeploymentStatus") deploymentGetUnstructured, err := dc.Resource(deploymentResource).Namespace(testNamespaceName).Get(context.TODO(), testDeploymentName, metav1.GetOptions{}, "status") framework.ExpectNoError(err, "failed to fetch the Deployment") deploymentGet := appsv1.Deployment{} err = runtime.DefaultUnstructuredConverter.FromUnstructured(deploymentGetUnstructured.Object, &deploymentGet) framework.ExpectNoError(err, "failed to convert the unstructured response to a Deployment") framework.ExpectEqual(deploymentGet.Spec.Template.Spec.Containers[0].Image, testDeploymentUpdateImage, "failed to update image") framework.ExpectEqual(deploymentGet.ObjectMeta.Labels["test-deployment"], "updated", "failed to update labels") ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, err = watchtools.Until(ctx, deploymentsList.ResourceVersion, w, func(event watch.Event) (bool, error) { if deployment, ok := event.Object.(*appsv1.Deployment); ok { found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.Labels["test-deployment-static"] == "true" && deployment.Status.AvailableReplicas == testDeploymentDefaultReplicas && deployment.Status.ReadyReplicas == testDeploymentDefaultReplicas return found, nil } return false, nil }) framework.ExpectNoError(err, "failed to see replicas of %v in namespace %v scale to requested amount of %v", testDeployment.Name, testNamespaceName, testDeploymentDefaultReplicas) ginkgo.By("patching the DeploymentStatus") deploymentStatusPatch, err := json.Marshal(map[string]interface{}{ "metadata": map[string]interface{}{ "labels": map[string]string{"test-deployment": "patched-status"}, }, "status": map[string]interface{}{ "readyReplicas": testDeploymentNoReplicas, }, }) framework.ExpectNoError(err, "failed to Marshal Deployment JSON patch") dc.Resource(deploymentResource).Namespace(testNamespaceName).Patch(context.TODO(), testDeploymentName, types.StrategicMergePatchType, []byte(deploymentStatusPatch), metav1.PatchOptions{}, "status") ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, err = watchtools.Until(ctx, deploymentsList.ResourceVersion, w, func(event watch.Event) (bool, error) { switch event.Type { case watch.Modified: if deployment, ok := event.Object.(*appsv1.Deployment); ok { found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.Labels["test-deployment-static"] == "true" return found, nil } default: framework.Logf("observed event type %v", event.Type) } return false, nil }) framework.ExpectNoError(err, "failed to see %v event", watch.Modified) ginkgo.By("fetching the DeploymentStatus") deploymentGetUnstructured, err = dc.Resource(deploymentResource).Namespace(testNamespaceName).Get(context.TODO(), testDeploymentName, metav1.GetOptions{}, "status") framework.ExpectNoError(err, "failed to fetch the DeploymentStatus") deploymentGet = appsv1.Deployment{} err = runtime.DefaultUnstructuredConverter.FromUnstructured(deploymentGetUnstructured.Object, &deploymentGet) framework.ExpectNoError(err, "failed to convert the unstructured response to a Deployment") framework.ExpectEqual(deploymentGet.Spec.Template.Spec.Containers[0].Image, testDeploymentUpdateImage, "failed to update image") framework.ExpectEqual(deploymentGet.ObjectMeta.Labels["test-deployment"], "updated", "failed to update labels") ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, err = watchtools.Until(ctx, deploymentsList.ResourceVersion, w, func(event watch.Event) (bool, error) { if deployment, ok := event.Object.(*appsv1.Deployment); ok { found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.Labels["test-deployment-static"] == "true" && deployment.Status.AvailableReplicas == testDeploymentDefaultReplicas && deployment.Status.ReadyReplicas == testDeploymentDefaultReplicas && deployment.Spec.Template.Spec.Containers[0].Image == testDeploymentUpdateImage return found, nil } return false, nil }) framework.ExpectNoError(err, "failed to see replicas of %v in namespace %v scale to requested amount of %v", testDeployment.Name, testNamespaceName, testDeploymentDefaultReplicas) ginkgo.By("deleting the Deployment") err = f.ClientSet.AppsV1().Deployments(testNamespaceName).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: testDeploymentLabelsFlat}) framework.ExpectNoError(err, "failed to delete Deployment via collection") ctx, cancel = context.WithTimeout(context.Background(), 30*time.Second) defer cancel() _, err = watchtools.Until(ctx, deploymentsList.ResourceVersion, w, func(event watch.Event) (bool, error) { switch event.Type { case watch.Deleted: if deployment, ok := event.Object.(*appsv1.Deployment); ok { found := deployment.ObjectMeta.Name == testDeployment.Name && deployment.Labels["test-deployment-static"] == "true" return found, nil } default: framework.Logf("observed event type %v", event.Type) } return false, nil }) framework.ExpectNoError(err, "failed to see %v event", watch.Deleted) }) </ins> }) func failureTrap(c clientset.Interface, ns string) {

If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/remove-lifecycle stale
/triage accepted
/merged


Question: // Makes sure the security group exists. // For multi-cluster isolation, name must be globally unique, for example derived from the service UUID. <ins> // Additional tags can be specified </ins> // Returns the security group id or error <del> func (c *Cloud) ensureSecurityGroup(name string, description string) (string, error) { </del> <ins> func (c *Cloud) ensureSecurityGroup(name string, description string, additionalTags map[string]string) (string, error) { </ins> groupID := "" attempt := 0 for {

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: /kind feature /sig aws What happened: In k8s 1.7, when creating a service of type LoadBalancer, K8S creates the ELB and a Security Group for it. For these resources, it adds a tag, constructed dynamically, containing the cluster id: "${cluster_id}" with the value "true" What you expected to happen: We'd like to have also a statically named tag , as it was in 1.5 ("KubernetesCluster") with the value of the cluster id. How to reproduce it (as minimally and precisely as possible): Running on AWS cloud provider, create a service of type LoadBalancer. Anything else we need to know?: The reason for what we expect is that we have tight security constraints that require us to fine tune the IAM policies for the Instance profiles that the controllers use. With a statically named tag we were able, in 1.5, to restrict the permissions in the IAM policy to act only on the k8s managed resources (ELBs, security groups). Here is the relevant part of the IAM policy: With a dynamically named tag we are not able to do this any more as AWS IAM doesn't support something like: We are sharing the Instance Profiles among different k8s clusters, hence the "" in the value of the tag. Environment: Kubernetes version (use ):Cloud provider or hardware configuration*: AWS OS (e.g. from /etc/os-release):Kernel (e.g. ):Install tools: kube-aws Others:
Reiterating the mentions to trigger a notification: <detailsIn response to : <!-- This form is for bug reports and feature requests ONLY! -- /sig aws In k8s 1.7, when creating a service of type LoadBalancer, K8S creates the ELB and a Security Group for it. For these resources, it adds a tag, constructed dynamically, containing the cluster id: "${cluster_id}" with the value "true" We'd like to have also a statically named tag , as it was in 1.5 ("KubernetesCluster") with the value of the cluster id. Running on AWS cloud provider, create a service of type LoadBalancer.


Question: // Makes sure the security group exists. // For multi-cluster isolation, name must be globally unique, for example derived from the service UUID. <ins> // Additional tags can be specified </ins> // Returns the security group id or error <del> func (c *Cloud) ensureSecurityGroup(name string, description string) (string, error) { </del> <ins> func (c *Cloud) ensureSecurityGroup(name string, description string, additionalTags map[string]string) (string, error) { </ins> groupID := "" attempt := 0 for {

The reason for what we expect is that we have tight security constraints that require us to fine tune the IAM policies for the Instance profiles that the controllers use. With a statically named tag we were able, in 1.5, to restrict the permissions in the IAM policy to act only on the k8s managed resources (ELBs, security groups). Here is the relevant part of the IAM policy: Environment: Kubernetes version (use ):OS (e.g. from /etc/os-release):Others: </details
I believe they are moving away from the tag since there are many resources that can be shared by multiple clusters. could you use an annotation?
We removed the condition in roles in the end. Thank you for your suggestion.
That annotation affects the ELB but not any security groups that are created. can you please reopen this? You reported a real issue, and even if it no longer affects you, it needs to be tracked and fixed.
Indeed, this might be a feature that someone else would need.
In fact, this is a feature I need!
Food for thought: should we propagate the tags on the ELB to the SG, or should we introduce an annotation for the SG specifically?


Question: err := esController.syncService(fmt.Sprintf("%s/%s", namespace, missingServiceName)) <del> // Since the service doesn't exist, we should get a not found error assert.NotNil(t, err, "Expected no error syncing service") assert.Equal(t, err.Error(), "service "notthere" not found") </del> <ins> // nil should be returned when the service doesn't exist assert.Nil(t, err, "Expected no error syncing service") </ins> // That should mean no client actions were performed assert.Len(t, client.Actions(), 0)

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: This requires 1.16 alpha feature: Endpoint Slices being enabled. After deleting a , EndpointSliceController kept enqueuing the service after syncing it successfully, until its retry budget exceeded. And the logs are misleading by indicating something wrong happened while the cleanup has been executed successfully: What you expected to happen: The deletion should be triggered only once and there should be no warning and error logs if the cleanup is successfully. How to reproduce it (as minimally and precisely as possible): Endpoint Slices feature a then delete it. kube-controller-manager logs, there would be repeated logs. Anything else we need to know?: Environment: Kubernetes version (use ):Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
There are no sig labels on this issue. Please add a sig label by either: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. See the . The in method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign
/assign


Question: } } <del> if klog.V(4).Enabled() { </del> <ins> if klog.V(10).Enabled() { </ins> for i := range result { klog.InfoS("Calculated node's final score for pod", "pod", klog.KObj(pod), "node", result[i].Name, "score", result[i].Score) }

Scheduler throughput drops significantly in http://perf- ! After , seems to be the curlspirit.
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/priority critical-urgent /sig scheduling /sig scalability


Question: ginkgo.Context("once the node is setup", func() { ginkgo.It("container runtime's oom-score-adj should be -999", func() { runtimePids, err := getPidsForProcess(framework.TestContext.ContainerRuntimeProcessName, framework.TestContext.ContainerRuntimePidFile) <del> gomega.Expect(err).To(gomega.BeNil(), "failed to get list of container runtime pids") </del> <ins> framework.ExpectNoError(err, "failed to get list of container runtime pids") </ins> for _, pid := range runtimePids { gomega.Eventually(func() error { return validateOOMScoreAdjSetting(pid, -999)

<!-- Please only use this template for submitting reports about failing tests in Kubernetes CI jobs --Which jobs are failing: kubernetes/node-problem-detector/node-problem-detector-ci: ci-npd-e2e-node See the Which test(s) are failing: [] NodeProblemDetector [NodeFeature:NodeProblemDetector] [] SystemLogMonitor should generate node condition and events for corresponding errors Since when has it been failing: Dec 03. was at 19:50 PST, was at 21:51 PST. Testgrid link: https://k8s- Reason for failure: Apparently is the culprit. Anything else we need to know: This is likely caused by some library upgrade. Nothing big.
/sig node
/assign
Seems like the failure is caused by . /cc /cc /cc /cc
/priority critical-urgent
The particular bug I care about has been fixed in . I'm gonna still work on submitting , as the same error should also be affecting a few other tests.


Question: } // Status indicates the result of running a plugin. It consists of a code, a <del> // message, (optionally) an error and an plugin name it fails by. When the status // code is not `Success`, the reasons should explain why. </del> <ins> // message, (optionally) an error and an plugin name it fails by. // When the status code is not Success, the reasons should explain why. // And, when code is Success, all the other fields should be empty. </ins> // NOTE: A nil Status is also considered as Success. type Status struct { code    Code

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --AppendReason panic when status is nil. Some plugins return nil to mean success. (Or, shouldn't be used on success status? I use field as a reason for why it is success.) not panic or describe this behaviour on the document
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig scheduling /assign I think there are some solutions we can take. just add this behaviour to AppendReason document. (most easy solution) Status when status is nil, like below. all plugins not to return nil when success (hard way..) I support the second idea.
I'd expect (and in fact it is) the caller to be responsible to ensure the status is not nil: The 2nd option may work, but semantically should be called by a non-success status.
Hi OK, I see :) I created the small PR to describe that reason field is for non-success status. Please take a look


Question: return false, output, nil } } <ins> // getRbdImageInfo try to get rbdImageInfo from deviceMountPath. func getRbdImageInfo(deviceMountPath string) (*rbdImageInfo, error) { deviceMountedPathSeps := strings.Split(filepath.Base(deviceMountPath), "-image-") if len(deviceMountedPathSeps) != 2 { return nil, fmt.Errorf("Can't found devicePath for %s ", deviceMountPath) } return &rbdImageInfo{ pool: deviceMountedPathSeps[0], name: deviceMountedPathSeps[1], }, nil } </ins>

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: What happened: Pod was deleted successfully but there is still a lock on the ceph image. I find two logs,one for that lock not removed and another for that lock removed. I find that there are little difference between them. the log that lock not removed: and the log that lock removed: What you expected to happen: How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): v1.6.7 Cloud provider or hardware configuration**: OS (e.g. from /etc/os-release): centos7 Kernel (e.g. ): 3.10.0-327.el7.x86_64 Install tools: Others:
/sig storage
Error 16 is EBUSY. similar to
were you running kubelet with cadvisor on?
/assign
yes, every kubelet run with cadvisor,and i know what code 16 represent for,but i still don't know the reason, is this a bug? How can i avoid this kind of situation from happening,thank you
It is probably related to
We see some other case may cause , kubelet run with nodeexporter and nodeexporter is deployed by k8s daemonset with -v /:/rootfs:ro (mount kubelet host rootfs into nodeexporter container). rbd device is mounted before nodeexporter started, because k8s 1.6.7 not support mount propagation, when the rbd plugin unmount /dev/rbd0 on the host namespace, the mount info will still leave over in node_exporter container namespace, may cause . hope this would help
thanks, does the problem persist if node_exporter stops collecting filesystem metrics?
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with .


Question: return false, output, nil } } <ins> // getRbdImageInfo try to get rbdImageInfo from deviceMountPath. func getRbdImageInfo(deviceMountPath string) (*rbdImageInfo, error) { deviceMountedPathSeps := strings.Split(filepath.Base(deviceMountPath), "-image-") if len(deviceMountedPathSeps) != 2 { return nil, fmt.Errorf("Can't found devicePath for %s ", deviceMountPath) } return &rbdImageInfo{ pool: deviceMountedPathSeps[0], name: deviceMountedPathSeps[1], }, nil } </ins>

Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten /remove-lifecycle stale
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close
I'm experiencing this issue. From what I can read above, if the rbd mount happens before the node_exporter starts; we'll always experience status 16?
We're getting this issue. This is because the mounted volume is mounted on Cadvisor container too. So the /dev/rbdX file is still in use. But still didn't find any solution to fix.
rbd volume will stuck when failed as will always returen
My solution: container use a volumeMount. make a script file to be executed every a minute:


Question: return "", fmt.Errorf("created security group, but id was not returned: %s", name) } <del> err := c.tagging.createTags(c.ec2, groupID, ResourceLifecycleOwned, nil) </del> <ins> err := c.tagging.createTags(c.ec2, groupID, ResourceLifecycleOwned, additionalTags) </ins> if err != nil { // If we retry, ensureClusterTags will recover from this - it // will add the missing tags.  We could delete the security


Question: return "", fmt.Errorf("created security group, but id was not returned: %s", name) } <del> err := c.tagging.createTags(c.ec2, groupID, ResourceLifecycleOwned, nil) </del> <ins> err := c.tagging.createTags(c.ec2, groupID, ResourceLifecycleOwned, additionalTags) </ins> if err != nil { // If we retry, ensureClusterTags will recover from this - it // will add the missing tags.  We could delete the security


Question: "context" "fmt" "math/rand" <del> "sort" </del> "sync" "sync/atomic" "time"


Question: } }) }) <ins> ginkgo.Context("PriorityClass endpoints", func() { var cs clientset.Interface f := framework.NewDefaultFramework("sched-preemption-path") var pcs []*schedulingv1.PriorityClass ginkgo.BeforeEach(func() { cs = f.ClientSet // Create 2 PriorityClass: p1, p2. for i := 1; i <= 2; i++ { name, val := fmt.Sprintf("p%d", i), int32(i) pc, err := cs.SchedulingV1().PriorityClasses().Create(context.TODO(), &schedulingv1.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: name}, Value: val}, metav1.CreateOptions{}) if err != nil { framework.Logf("Failed to create priority '%v/%v'. Reason: %v. Msg: %v", name, val, apierrors.ReasonForError(err), err) } framework.ExpectEqual(err == nil || apierrors.IsAlreadyExists(err), true) pcs = append(pcs, pc) } }) ginkgo.AfterEach(func() { // Print out additional info if tests failed. if ginkgo.CurrentGinkgoTestDescription().Failed { // List existing PriorityClasses. priorityList, err := cs.SchedulingV1().PriorityClasses().List(context.TODO(), metav1.ListOptions{}) if err != nil { framework.Logf("Unable to list PriorityClasses: %v", err) } else { framework.Logf("List existing PriorityClasses:") for _, p := range priorityList.Items { framework.Logf("%v/%v created at %v", p.Name, p.Value, p.CreationTimestamp) } } } // Cannot run collection deletion which would delete all system level priority classes. for _, pc := range pcs { err := cs.SchedulingV1().PriorityClasses().Delete(context.TODO(), pc.Name, *metav1.NewDeleteOptions(0)) framework.ExpectNoError(err) } }) ginkgo.It("verify PriorityClass endpoints can be operated with different HTTP methods", func() { // 1. Patch/Update on immutable fields will fail. pcCopy := pcs[0].DeepCopy() pcCopy.Value = pcCopy.Value * 10 err := patchPriorityClass(cs, pcs[0], pcCopy) framework.ExpectError(err, "expect a patch error on an immutable field") framework.Logf("%v", err) pcCopy = pcs[1].DeepCopy() pcCopy.Value = pcCopy.Value * 10 _, err = cs.SchedulingV1().PriorityClasses().Update(context.TODO(), pcCopy, metav1.UpdateOptions{}) framework.ExpectError(err, "expect an update error on an immutable field") framework.Logf("%v", err) // 2. Patch/Update on mutable fields will succeed. newDesc := "updated description" pcCopy = pcs[0].DeepCopy() pcCopy.Description = newDesc err = patchPriorityClass(cs, pcs[0], pcCopy) framework.ExpectNoError(err) pcCopy = pcs[1].DeepCopy() pcCopy.Description = newDesc _, err = cs.SchedulingV1().PriorityClasses().Update(context.TODO(), pcCopy, metav1.UpdateOptions{}) framework.ExpectNoError(err) // 3. List existing PriorityClasses. _, err = cs.SchedulingV1().PriorityClasses().List(context.TODO(), metav1.ListOptions{}) framework.ExpectNoError(err) // 4. Verify fields of updated PriorityClasses. for _, pc := range pcs { livePC, err := cs.SchedulingV1().PriorityClasses().Get(context.TODO(), pc.Name, metav1.GetOptions{}) framework.ExpectNoError(err) framework.ExpectEqual(livePC.Value, pc.Value) framework.ExpectEqual(livePC.Description, newDesc) } }) }) </ins> }) type pauseRSConfig struct {

The following PriorityClass endpoints was Promoted it 1.14 without Conformance tests, and remain untested: replace-, read-, patch-, list-, delete-collection- SchedulingV1PriorityClass According to this APIsnoop query, there are still some remaining PriorityClass endpoints which are untested. with this query you can filter untested endpoints by their category and eligiblity for conformance. e.g below shows a query to find all conformance eligible untested,stable,core endpoints ! createSchedulingV1PriorityClass deleteSchedulingV1PriorityClass Promoted to Conformance 1.19 with /sig testing /sig architecture /area conformance
Is the ultimiate goal to use conformance tests to explicitly issue all CRUD requests against PriorityClass? And, is equivalent to "update PriorityClass"?
all APIs must have Conformance tests unless they are not eligible for Conformance due to the There is other APIGroups that have both Replace and Update as endpoints. That is the prerogative of the SIG to decide.
can you please point us to similar conformance tests used for core objects such as pods and nodes?
This is a life cycle test for Create, List, Update, Patch, DeleteCollection More can be found in the test/e2e folder. Also look for examples on the Conformance at previously Promoted Conformance test Reach out if we can help with more information on our approach to Conformance testing.
I remembered about this PR. Last week a few lines of code was to cover the last of the "Lease" endpoint group for conformance. The one file contain the Conformance tests for the complete life cycle of Lease endpoints.
/assign I will work on it.
/reopen Will monitor in test grid to move to Conformance after 2 weeks soak.
Reopened this issue. <detailsIn response to : /reopen Will monitor in test grid to move to Conformance after 2 weeks soak. </details
/triage accepted


Question: "enableHttps": false, "nodeCacheCapable": false } <del> ] </del> <ins> ], "hardPodAffinitySymmetricWeight" : 10 </ins> }

( noticed this) HardPodAffinitySymmetricWeight should be in the , not in KubeSchedulerConfiguration If/when fixing this, we need to be careful to do it in a backward-compatilble way.
Do you mean ? It seems scheduler policy config's args can not passed to FitPredicates; it need to update related interface. For the : mark to be deprecated and set scheduler policy config to HardPodAffinitySymmetricWeight if policy config is empty.
Are you working on this? if no, I can take it.
, nop; please go ahead.
/ / / , When reviewing the PR, there's one question on : the default value of is 1, which means the weight (1) will always add to priorities if term matched (), does it our expectation? And the user has to set it to 0 to disable it. IMO, the default value of should be 0.
The value 1 (instead of 0) was chosen so that if you say A must run with B, there is an automatic preference that B should run with A. This seems like reasonable behavior.
Thanks very much; reasonable case to me. But it seems we can not disable it: if set to 0, the default funcs will set it to default (1); similar result to un-set. Anyway, no user complain this now :).


Question: // StoreCertsInSecrets is alpha in v1.8 StoreCertsInSecrets = "StoreCertsInSecrets" <ins> // SupportIPVSProxyMode is alpha in v1.8 SupportIPVSProxyMode = "SupportIPVSProxyMode" </ins> ) <del> var v190 = version.MustParseSemantic("v1.9.0") </del> <ins> var v190 = version.MustParseSemantic("v1.9.0-alpha.1") </ins> // InitFeatureGates are the default feature gates for the init command var InitFeatureGates = FeatureList{ <del> SelfHosting:         {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Beta}}, StoreCertsInSecrets: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}}, HighAvailability:    {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190}, </del> <ins> SelfHosting:          {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Beta}}, StoreCertsInSecrets:  {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}}, HighAvailability:     {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190}, SupportIPVSProxyMode: {FeatureSpec: utilfeature.FeatureSpec{Default: false, PreRelease: utilfeature.Alpha}, MinimumVersion: v190}, </ins> } // Feature represents a feature being gated

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: /kind feature What happened: Currently, IPVS proxier does not run any e2e test cases. What you expected to happen: I think it's important to create test suite for IPVS in upstream CI which will help a lot in moving IPVS existing alpha. Anything else we need to know?: Let's work together. /sig network
Self /assign
There are to ways to do this work: , but doesn't support any kube-proxy config right now, so I had this pr , and also, need support it too, see this pr: kubernetes/kubernetes-anywhere build cluster. See this pr: kubernetes/test-infra The second way is much easier and faster, but I was told they gonna deprecated the , so in the long term, we still need and 's support. If anybody have better way or have some suggestion about this, please let me know, I'll be really appreciate, Thanks.


Question: } } <ins> // CloneTLSConfig returns a tls.Config with all exported fields except SessionTicketsDisabled and SessionTicketKey copied. // This makes it safe to call CloneTLSConfig on a config in active use by a server. // TODO: replace with tls.Config#Clone when we move to go1.8 func CloneTLSConfig(cfg *tls.Config) *tls.Config { if cfg == nil { return &tls.Config{} } return &tls.Config{ Rand:                     cfg.Rand, Time:                     cfg.Time, Certificates:             cfg.Certificates, NameToCertificate:        cfg.NameToCertificate, GetCertificate:           cfg.GetCertificate, RootCAs:                  cfg.RootCAs, NextProtos:               cfg.NextProtos, ServerName:               cfg.ServerName, ClientAuth:               cfg.ClientAuth, ClientCAs:                cfg.ClientCAs, InsecureSkipVerify:       cfg.InsecureSkipVerify, CipherSuites:             cfg.CipherSuites, PreferServerCipherSuites: cfg.PreferServerCipherSuites, ClientSessionCache:       cfg.ClientSessionCache, MinVersion:               cfg.MinVersion, MaxVersion:               cfg.MaxVersion, CurvePreferences:         cfg.CurvePreferences, } } </ins> func TLSClientConfig(transport http.RoundTripper) (*tls.Config, error) { if transport == nil { return nil, nil

When the master dials nodes, it is mutating the tlsConfig held by the master kubelet client, to add a server name. That means if two different nodes are dialed from a master started with , x509 verification will fail for the second one. Introduced in
this is a regression from 1.3 and breaks pod logs/exec/attach and node proxying on installations that use
We probably need to review that PR and look to see if any other deliberate copies were flagged incorrectly by govet.
Installations that depend on secure trust from externally provided kubelet server certs are the highest security rings. If we don't fix this we need to tell anyone using this flag not to upgrade until we release 1.4.1.
the rest of the changes in the govet cleanup PR looked fine to me. fix for this issue is in
Since we are telling folks they have to upgrade to kubectl 1.4.0 for rollingupdate to work it would definitely be a confusing message to say other folks should not upgrade to 1.4.0 at all.
If the fix is merged into master by this afternoon, lets discuss whether or not we should cherrypick it - e.g. risks involved, how we can manually qa to verify nothing else was broken, etc.
Risk involved: If you are using a secure cluster (defined as ensuring no one can MITM your nodes and masters) and you upgrade to 1.4.0 and it does not include this fix - you are broken.
What is the likelihood of interactions with other code? Is it possible this fix could introduce a breaking change to something else? Has the fix been manually verified? Are we confident that this will fully fixes the issue (vs other edge cases exist for secure clusters that this doesn't catch). Ideally I would like to see this cherrypicked into 1.4 this evening and then cut a new beta that includes it.
None. The single line that is currently broken was changed. The rest is a new function (with no side effects), and tests. No, it restores the original intent of the TLS config copy, and tests cover the cases we are using this with (secure, insecure, custom dialer) Yes. I swept for the same issue elsewhere and didn't find any issues. We test with a fully secured master-kubelet setup in Openshift and got a clean test run with this fix applied.
SGTM. Thanks for fixing this.


Question: fs.DurationVar(&s.ShutdownDelayDuration, "shutdown-delay-duration", s.ShutdownDelayDuration, ""+ "Time to delay the termination. During that time the server keeps serving requests normally and /healthz "+ <del> "returns success, but /readzy immediately returns failure. Graceful termination starts after this delay "+ </del> <ins> "returns success, but /readyz immediately returns failure. Graceful termination starts after this delay "+ </ins> "has elapsed. This can be used to allow load balancer to stop sending traffic to this server.") utilfeature.DefaultMutableFeatureGate.AddFlag(fs)

What happened: has a typo, (should be ) This also shows up in the rendered documentation. What you expected to happen: Documentation has correct path for readyz endpoint. How to reproduce it (as minimally and precisely as possible): Easiest to see this wrong in the source code, I think. Anything else we need to know?: Environment: Kubernetes version (use ): v1.16 Cloud provider or hardware configuration: not applicable OS (e.g: ): not applicable Kernel (e.g. ): not applicable Install tools: Network plugin and version (if this is a network-related bug): Others: /sig docs /kind bug /priority backlog


Question: // unicode code point can be up to 4 bytes long) strWithMaxLength.MaxElements = zeroIfNegative(*s.MaxLength()) * 4 } else { <del> strWithMaxLength.MaxElements = estimateMaxStringLengthPerRequest(s) </del> <ins> if len(s.Enum()) > 0 { strWithMaxLength.MaxElements = estimateMaxStringEnumLength(s) } else { strWithMaxLength.MaxElements = estimateMaxStringLengthPerRequest(s) } </ins> } return strWithMaxLength case "boolean":

pointed out to me that a CEL x-kubernetes-validations rule in a CRD was hitting the estimated cost limit when performing a simple "string has prefix" check on an en enum field. Putting a limit in the OpenAPI of the enum string field fixed the problem, but having to add a limit like this is counter-intuitive and unnecessary, since the allowed enum values already make the max allowed length obvious. The apiserver automatically computes the max length of an enum as the length of the longest allowed enum value. I believe we already refuse evaluate CEL rules on values that fail OpenAPI value validation, since we rely in the value validations to establish all length limits. I haven't tested it locally, but something like this should work (given enough nested arrays): As a workaround, one can manually set on the string enum, e.g.: 25+ All No response No response No response No response
/sig api-machinery /triage accepted /assign
/assign
I wasn't sure if we special-cased the maxLength / maxItems / maxProperties failures, or if all openapi validation had to pass before we started messing with CEL evaluation
it looks like we currently special-case specific openapi errors that short-circuit CEL validation, I'm not sure an enum validation failure would currently prevent evaluation:
The enum err is converted to () error which is not short-circuit here. But for defaulting case, all error would be short-circuit. I would suggest to add error into the blockingErr list to short-circuit CEL validation. Would this make sense?
yeah, that sounds good if we're going to rely on it to catch things that are longer than the maximum length enum value


Question: "fmt" "reflect" "strings" <ins> "testing" </ins> "time" apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"

reported that conformance testing with the latest version of sonobuoy against a version 1.29.0-alpha.3 cluster fails on the support the 1.17 Sample API Server using the current Aggregator test: , related to I found the reason why the test is working fine in Kubernetes CI jobs but fails with sonobuoy is because the service account used by sonobuoy "sonobuoy-serviceaccount" doesn't have read permission of the non-resource url "/", which is used by the test to confirm the removal of group path: The permission of "sonobuoy-serviceaccount" is as below: There may be two ways to fix it: the RBAC rule of sonobuoy to include "get" of "/". However, could other test runners encounter this issue? Is there a doc listing the required permissions for running e2e tests? the e2e to skip checking the removal of the group path if the service account doesn't have permission to access the root path. Which way do we usually prefer? Running e2e with sonobuoy should succeed. a v1.29.0-alpha.3 cluster. No response <details</details<details</details<details</details<details</details<details</details<details</details
implemented this solution.
/sig api-machinery
thank you for opening this issue and working on a solution. I'm probably not the right person to decide on which solution is preferred. From my perspective, either solution is fine.
Note: Sonobuoy is not owned by any Kubernetes SIG. At most it is between SIG Architecture and the CNCF to choose whether to recommend it for running tests. SIG Testing owns the e2e image that sonobuoy can be used to schedule but not sonobuoy. IMHO this is a bug in running the tests via a pod, which is not how we develop the tests (they're run outside the cluster, by a cluster admin account typically). I would say 1) is the correct answer. xref: /sig architecture /sig testing /triage accepted cc
thanks for your comment. I guess 1) means non of the existing sonobuoy versions can pass K8s conformance test against a cluster 1.29.0-alpha3, as the RBAC rule is hardcoded.


Question: "fmt" "reflect" "strings" <ins> "testing" </ins> "time" apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"

I will file an issue to sonobuoy first.
question here is what are the set of client permissions required to run the conformance suite? perhaps a level of access or a more granular recommendation should be documented. what is your take on the root path requirement as approver for: EDIT: slack thread:
I agree, we already require e.g. privileged pods for a long time in conformance tests and we should make it clear that we expect a highly privileged client.
+1 for what is saying, that we should expect that the user running the tests has required permissions. These tests are not built for being run unprivileged.
i wanted to PR the admin note in our conformance docs under sig-arch, but then i saw this: so broke this guarantee. sounds like SIG Arch needs to comment.
This statement surprises me. Root discovery is one of the least-privileged permissions you can have on a cluster. For years this was even open for unauthorized users on the internet (if your api endpoint has been exposed). In other words, just add the permission. While thinking about it, it is surprising that the user in question does not have this access already. Don't we allow that for every authorized user?
In contrast, I hope that lots of privileged operations are tested by conformance today. If we don't test things like admission webhooks for example, we have much bigger problems. And admission webhooks are basically equivalent to cluster-admin.
that is outdated, we use hostNetwork pods one some of the tests IIRC
looks like we don't. before this ticket, i was under the impression that the conformance suite already requires admin and that all testers (like Sonobuoy) use admin level credentials. it is non-privileged (e.g., does not require root on nodes, access to raw network interfaces, or cluster admin permissions) looks like we may have to change that sentence completely.
Would have expected it here: The PR in question explicitly fixes the result returned by . If that is part of conformance (IMO it should), every conformance test needs access it obviously. Whether we require admin, or we add it manually, either would work.


Question: "fmt" "reflect" "strings" <ins> "testing" </ins> "time" apiextensions "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"

adding to milestone and making release blocking ... we should not release 1.29 in a state that the most commonly used conformance tool will fail
IMO we should clarify the state and go fix the tool, but we should keep the test and we should NOT add skip. Conformance docs are not a release artifact, and sonobuoy release lifecycle is not a conformance artifact. If we can't get sonobuoy working in a reasonable time, we already know the tests can be run by scheduling the image (which is ours) directly with kubectl instead and we can document that in the upstream conformance docs. We need to fix the for this image: ,%20master%20(dev) And it should probably be to release-blocking in the future.
it looks like the e2e change made in was not consciously made part of conformance, it was just trying to functionally test the bugfix being made. For 1.29, I think we should revert the change to and make an integration test that validates the fix instead. We can revisit whether content of is part of conformance or not for 1.30
I opened to revert the e2e change and move the functional test to an integration test
Filed to try to get the image job passing again.
We probably need another issue to track 1.30 follow-up?
If no one has done it, I could create another issue and follow up with the items that have been discussed here: outdated privilege requirement in commonly used conformance tools like sonobuoy aware of the required priviledge to avoid similar situations in the future. permission of GET "/" to authorized users via the role for root discovery.


Question: // createClients creates a kube client and an event client from the given config and masterOverride. // TODO remove masterOverride when CLI flags are removed. <del> func createClients(config componentconfig.ClientConnectionConfiguration, masterOverride string) (clientset.Interface, clientset.Interface, v1core.EventsGetter, error) { </del> <ins> func createClients(config componentconfig.ClientConnectionConfiguration, masterOverride string, timeout time.Duration) (clientset.Interface, clientset.Interface, v1core.EventsGetter, error) { </ins> if len(config.KubeConfigFile) == 0 && len(masterOverride) == 0 { glog.Warningf("Neither --kubeconfig nor --master was specified. Using default API client. This might not work.") }

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . If the matter is security related, please disclose it privately via --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: kube-cm kube-scheduler does not exit when kube-apiserver suspended What you expected to happen: kube-cm kube-scheduler should exit after LeaderElector stops leading How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others:
/sig api-machinery /kind bug
/assign
/cc


Question: currentBucketID := t.clock.Now().Unix() // There should be one or two elements in almost all cases expiredWatchers := make([][]*cacheWatcher, 0, 2) <ins> t.lock.Lock() defer t.lock.Unlock() </ins> for ; t.startBucketID <= currentBucketID; t.startBucketID++ { if watchers, ok := t.watchersBuckets[t.startBucketID]; ok { delete(t.watchersBuckets, t.startBucketID)

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: I try to backport bookmark feature to our internal branch. When I enable bookmark feature, apiserver leaks memory. I spend some time to dig the root cause and find the bookmarkWatchers of service cacher is keep increasing. I am not sure if it exists at upstream code. Please let me know if it is ok at latest code. What you expected to happen: Avoid memory leaking How to reproduce it (as minimally and precisely as possible): a cluster. not update service. will find kube-apiserver's memory keep increasing. Anything else we need to know?: Kubelet will watch service with bookmark, and add a watcher to bookmarkWatchers. and service init resource version is 0 Without updating service, we cannot update "lastProcessedResourceVersion", it will keep 0, bookmarkTimer branch will always be continued. As a result, it cannot trigger popExpiredWatchers, so the service watcher will keep increasing every watching. Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig scalability PLAT :)
How about making this change ?
- can you also please take a look?
Agree with u. That's same way I fix in our branch. But I wanna hear the suggestion of and .
! With the same bugfix wrote above, we can see the memory not increasing any more.
Given that watch-bookmarks doesn't create any additional go-routines (well, creates 1, but cannot leak them), it has to leak the data. The only data that can be leaked is exactly bookmarkWatchers. So I looked into the code and got into exactly the same conclusion - that we must be leaking watchers for resources that aren't changing. The fix as above makes sense (though we also need a unit test for it).
Thanks a lot.
has the fix by
Thanks for reporting the bug, I will look at this day later. In addition, also need to think carefully about whether need to send a bookmark when there are no events at all.
cc


Question: }, }, }, <ins> "invalid extended resource requirement without limit": { expectedError: "Limit must be set", spec: core.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "123", Namespace: "ns"}, Spec: core.PodSpec{ Containers: []core.Container{ { Name:            "invalid", Image:           "image", ImagePullPolicy: "IfNotPresent", Resources: core.ResourceRequirements{ Requests: core.ResourceList{ core.ResourceName("example.com/a"): resource.MustParse("2"), }, }, }, }, RestartPolicy: core.RestartPolicyAlways, DNSPolicy:     core.DNSClusterFirst, }, }, }, </ins> "invalid fractional extended resource in container request": { expectedError: "must be an integer", spec: core.Pod{

/kind bug /sig node /area hw-accelerators What happened: In devices are allocated from a function called . This function inspects only container resource limits, ignoring container resource requests. This is a usability problem because it is valid to omit limits for extended resources. If limit is omitted for a device resource, then the scheduler properly places the pod with respect to available node resources. However, the device manager never dispatches to the device plugin for that resource. The result is that the user application cannot access the requested device from inside the container. What you expected to happen: Following the documented behavior of extended resources, I expected that specifying a container resource request for a device would cause the device subsystem to properly configure access for my workload. How to reproduce it (as minimally and precisely as possible): Launch an alpha GKE cluster with GPU support enabled. Attempt to run any program that accesses the GPU from a container where the resource is specified in requests but omitted from limits. Environment: Kubernetes version (use ): v1.8+ Cloud provider or hardware configuration: device plugins enabled, devices available.
Related issue:
cc
/assign
FYI. The said issue has been fixed in . There are several valid cases when requesting extended resources: The user can specify 'limits' only. The API server will automatically set 'requests' to be equal to 'limits'. The user specify 'limits' and 'requests', and they are equal. The user specify 'requests' only. We can either reject the request saying that 'limits' must be set, or implicitly treat 'limits' as equal to 'requests'.
Hi went with to begin since it was ready to go and smaller (easier to revert if necessary.) Hopefully it's easy to rebase your work on top of that. Sorry for the inconvenience.
I feel the documented behavior is incorrect. The original intention was to always require for extended resources until we have better APIs to express bursting scenarios. I vote for fixing the documentation and fixing the validation logic and having kubelet use limits only.
/reopen


Question: e26fc85d14a1d3dc25569831acc06919673c545a kube::util::go_install_from_commit  github.com/bazelbuild/rules_go/go/tools/gazelle/gazelle  <del> a280fbac1a0a4c67b0eee660b4fd1b3db7c9f058 </del> <ins> c72631a220406c4fae276861ee286aaec82c5af2 </ins> touch "${KUBE_ROOT}/vendor/BUILD"

Is this a BUG REPORT or FEATURE REQUEST?: /kind feature This is a followup on , but for Go 1.9 We should do the same kind of testing now to possibly catch bugs in either Go or Kubernetes when doing this kind of dependency version bump. Ideally we'd have automated tests against Go tip (), but we don't have that at the moment, so we have to do it kind of manually. We should also decide which k8s version we should target. k8s v1.8 or v1.9? What Go versions have been used so far? Kubernetes v1.7.0 - go1.8.3 Kubernetes v1.6.0 - go1.7.5 Kubernetes v1.5.0 - go1.7.4 Kubernetes v1.4.0 - go1.6.3 Kubernetes v1.3.0 - go1.6.2 Kubernetes v1.2.0 - go1.4.2 Seems like the trend has been to stay on the same Go version for two releases and then bump. That'd mean we'd switch to Go1.9 in k8s v1.9 Everyone happy with that? Or is there any reason for us to go start using go1.9 in k8s v1.8? In any case, we should indeed start testing go1.9 out now... cc
Let me know if you find any bugs
FWIW pretty tight to code-freeze I would prefer punting out a cycle fwiw.
I don't think we're that close to code freeze given that we have 5 (of 8) weeks left in this coding cycle But I'm happy to defer this to v1.9 if others think that. I guess the main reason for a bump would be scalability goals, but I'm not sure we have the need to expand above 5000 (and that Go would be the bottleneck in that case) for v1.8
made the Go benchmarks 9% faster with 15% larger binaries. I'm interested to see how that affects our numbers.
mid-stack inlining was not included in Go 1.9. It made the compiler too slow. It's being targeted to Go 1.10 now instead, after some compiler optimization work.
Oh, thanks for the correction!


Question: e26fc85d14a1d3dc25569831acc06919673c545a kube::util::go_install_from_commit  github.com/bazelbuild/rules_go/go/tools/gazelle/gazelle  <del> a280fbac1a0a4c67b0eee660b4fd1b3db7c9f058 </del> <ins> c72631a220406c4fae276861ee286aaec82c5af2 </ins> touch "${KUBE_ROOT}/vendor/BUILD"

Regardless, we need to run the scalability tests against the RC so we can report issues earlier-- the last upgrade was particularly painful because of an HTTP regression that we caught way too late.
yeah. I filed for that now
yeah, my recollection is that the Go releases usually occur too late in our release cadence and end up getting bumped to the next Kubernetes release. (Also we inevitably find something wrong, though testing RCs would help with that. Though I'm not sure anyone has cycles to test RCs and fix all of the issues that do come up.)
Based on our current time frame, it looks like go1.9 support will have to slip to k8s 1.9. is open to dig into and start things testing, but there's a number of things that have to move.. I'll try to document those this time around to help with future go version bumps. So far: bazel build rules needed to be updated to support go1.9 () bazel version possibly needs to be updated? some of our custom bazel rules broke
When done, we should update this:
/priority important-soon
Updating the dependency is blocked pending bazel 0.6.0 - .
Can we upgrade to 0.6.0 now? Does bazel 0.6.0 have multiarch support btw?
We can't update to depend on bazel 0.6.0+ until we can figure out how to support different versions of the bazelbuild and kubekins-e2e images in PR jobs in prow based on what branch is being tested. cc
Noting that we're working on branching jobs, is testing one option here:
[MILESTONENOTIFIER] Milestone Issue Current <details open<summaryIssue Labels</summary- : Issue will be escalated to these SIGs if needed. : Escalate to the issue owners and SIG owner; move out of milestone after several unsuccessful escalation attempts. : New functionality.


Question: e26fc85d14a1d3dc25569831acc06919673c545a kube::util::go_install_from_commit  github.com/bazelbuild/rules_go/go/tools/gazelle/gazelle  <del> a280fbac1a0a4c67b0eee660b4fd1b3db7c9f058 </del> <ins> c72631a220406c4fae276861ee286aaec82c5af2 </ins> touch "${KUBE_ROOT}/vendor/BUILD"

</details<details<summaryHelp</summary<ul<li<a href=""Additional instructions</a</li<li<a href=""Commands for setting labels</a</li</ul</details
I approved this for v1.9, this is something we definitely want in order to keep up with Go (and for improved performance)
we're starting to use the forked images now /cc this should be more feasible very soon Edit: test-infra jobs have also been using bazel 0.6 (now 0.6.1) / a recent rules_go to soak things a bit
We've identified a failing test as a result of the WIP PR to upgrade the codebase to go1.9: The admission webhook test fails with certificate errors under go1.9.1, but runs successfully under go1.8.3. cc: who wanted to be alerted to potential bugs
- please see for a possible fix


Question: if err := p.Unmarshal(data); err != nil { return err } <del> m.Time = time.Unix(p.Seconds, int64(p.Nanos)).Local() </del> <ins> // truncate precision to microseconds to match JSON marshaling/unmarshaling truncatedNanoseconds := time.Duration(p.Nanos).Truncate(time.Microsecond) m.Time = time.Unix(p.Seconds, int64(truncatedNanoseconds)).Local() </ins> return nil }

Found some error logs when sending series events to apiserver: seems the later eventTime differed from the first one in series events After investigation, I found the problem was in the marshal. MicroTime is supposed to work on microsecond level precision, but init with nanosecond level precision, in this case, generates microsecond level precision data and generates nanosecond level precision, so when unmarshalled, the new time is not considered the exactly same as the original time (they are different at nanosecond level) MicroTime marshal should work at same precision for json and proto, so unmarshal can generate the exactly same MicroTime result: This causes the series events won't be updated on api-server after created master <details</details<details</details<details</details<details</details<details</details
/sig api-machinery
/assign
Good catch... we'd probably need to truncate to microseconds when marshaling and unmarshaling proto to ensure any already persisted data is fixed up when read. Something like this:
Thanks for advice, going to send a PR soon


Question: podName := "statscollectiontest-" + string(uuid.NewUUID()) pod := v1.Pod{ ObjectMeta: metav1.ObjectMeta{ <del> Name: podName, </del> <ins> Name: fmt.Sprintf("%s-%d", podName, i), </ins> Labels: map[string]string{ "name":    podName, "testapp": "stats-collection",

<!-- Please only use this template for submitting reports about continuously failing tests or jobs in Kubernetes CI --Which jobs are failing: aks-engine-azure-windows-master-containerd-serial-slow Which test(s) are failing: [sig-windows] [Feature:Windows] Kubelet-Stats [Serial] Kubelet stats collection for Windows nodes when running 10 pods should return within 10 seconds Since when has it been failing: Unknown. Testgrid link: Reason for failure: The test spawns 10 pods with the same pod name, which contains multiple containers with the same container name. Because of this, the test fails. Logs: Anything else we need to know:
/sig windows These SIGs are my best guesses for this issue. Please comment if I am incorrect about one. I am a bot run by vllry.


Question: w = crlf.NewCRLFWriter(w) } <del> results.header.writeTo(w) </del> <ins> if addHeader { results.header.writeTo(w) } </ins> if !containsError { if err := printer.PrintObj(objToEdit, w); err != nil {

Recently I was experimenting with changing for a Golang pre-purposed faux editor when editing resources when I noticed that it couldn't be marshaled. Upon inspecting the output JSON I noticed it had the same comments that the default yaml version has. Valid JSON cannot have comments. Since the comment also gives warning / error information upon saving it must be decided where to put that in the JSON output. Perhaps a new JSON property?
Maybe just not save the comments as part of the json to get rid of the warning, but still should them?
Is this a 1.2 thing? I do not see any errors when running the two commands. I made an edit in one run, and did not make an edit in a separate run. Maybe I don't understand the problem that is seen.
there isn't a kubernetes error that is shown, the issue is that the output JSON that kubernetes generates is actually invalid JSON due to the comments at the top. This caused errors when I was trying to unmarshal that JSON in another project.
Okay. Got it. The editor is being very strict about json having no comments.
It's not only about editor, it's simply part of language specification. Explanation for this: IMO if we want to use format, we should comply with it's standard. Also currently used sign for comments is unusual in javascript world, from which json evolved. In this world comments are marked in "C++ like" style (what in early days of JSON was also supported), i mean using and . So IMO in time of converting into JSON - we should strip such comments (I would prefer that), or they should be in some documented and specific to kubernetes way into JSON object.


Checked in: Not checked in: Can anyone explain why?
ones published outside k/k that are in libraries/modules expected to be usable without running custom code generation steps are checked in
example issue: there were similar issues with other instances
/triage accepted
I guess I am asking why not check in
oh, I'd be in favor of that :)
downstream users very often end up realizing that we warn folks not to use ( ) because they go looking for why this file is not checked in. This paper cut makes people realize that they are doing something that we don't want them to do. (not arguing for or against, just stating where this was effective before)
I think the v0.0.0 local replace directives in accomplish similar things w.r.t. making people go digging about friction using k/k as a library


Question: } func (m *NsenterMounter) IsNotMountPoint(dir string) (bool, error) { <del> return IsNotMountPoint(m, dir) </del> <ins> return isNotMountPoint(m, dir) </ins> } func (*NsenterMounter) IsMountPointMatch(mp MountPoint, dir string) bool {

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!--What happened: There's , and . The former is used by local and nfs volume plugins, while the latter is used by , which is also used by local and nfs volume plugins. What you expected to happen: Can we consolidate these? This is really confusing and could easily be the source of bugs. <!-- DO NOT EDIT BELOW THIS LINE --/kind cleanup
cc
/assign I'd like to clean up code.


Question: func (c *FakeNamespaces) Create(namespace *api.Namespace) (*api.Namespace, error) { c.Fake.Actions = append(c.Fake.Actions, FakeAction{Action: "create-namespace"}) <del> return &api.Namespace{}, nil </del> <ins> return &api.Namespace{}, c.Fake.Err </ins> } func (c *FakeNamespaces) Update(namespace *api.Namespace) (*api.Namespace, error) {

While running one of my soak tests I noticed the error message: I don't see why I should get this error message. Why should it matter if the namespace already exists? It was created as part of the service creation call just beforehand. And the second attempt to create this pod in exactly the same namespace works. Is this a bug?
Running a second time did not produce this error.
I agree it should not matter. The error is produced there, and I suspect there were multiple concurrent calls being made that try to create a resource in a namespace that had not yet existed. We need to update the code in question to swallow the error if the namespace already exists.
FYI - longer term, I would like to eliminate the use of autoprovision of namespace and require you to first create a namespace manually. what is there now is more to preserve existing behavior prior to namespace as object was introduced.


Question: if !skipAnyOf { for i := range v.AnyOf { <del> allErrs = append(allErrs, validateNestedValueValidation(&v.AnyOf[i], false, false, fldPath.Child("anyOf").Index(i))...) </del> <ins> allErrs = append(allErrs, validateNestedValueValidation(&v.AnyOf[i], false, false, lvl, fldPath.Child("anyOf").Index(i))...) </ins> } }

We allow CRD authors to write . Currently we don't restrict writing validation for , which means people can have . For example, people can have additive name constraints (require CR to be ) with the following schema: This brings us two questions: do we if "metadata" is included in the schema: simplest option is to publish standard metav1.ObjectMeta schema info, and use any CRD-specified "metadata" properties as additive validation (just like in-tree types do, publishing standard ObjectMeta schema info, and implementing type-specific name validation internally) alternatively, we can try merging user-specified metadata schema with the standard one. Technically people should only add extra validation to existing metadata properties (adding new fields to metadata won't work since they will be pruned by built-in validation first), but it's unclear to me how arbitrary people may want the metadata validation to be term question is whether to allow people specifying metadata validation at all, or just allow a subset (like name validation) in CRD v1 /sig api-machinery /area custom-resources cc
What is the difference between these two? I mean the end result is the same right? The only thing that can be is validation. Nothing else works unless you are considering extensions?
The former means we drop/hide user-specified metadata schema when we publish CRD OpenAPI. We still validate user-specified constraints on server side, but client-side validation won't see them. The latter enables client-side user-specified metadata validation.
If it's possible to merge with standard metadata schema info, or at the cost of not publishing the standard schema. My inclination would be to publish the complete standard schema for metadata/apiVersion/kind, which we know is complete and accurate, and use and user-specified schema for those fields as addition validations server side


Question: return err } <del> if pod.Status.Phase != api.PodRunning { return fmt.Errorf("pod %s is not running and cannot execute commands; current phase is %s", p.PodName, pod.Status.Phase) } </del> containerName := p.ContainerName if len(containerName) == 0 { glog.V(4).Infof("defaulting container name to %s", pod.Spec.Containers[0].Name)

Need to finish so the client allows you to select an init container, and verify the kubelet works correctly.
(this is a nice intersection of new in 1.3, node and usability, if anyone wants to give it a shot. Pretty sure Clayton is busier than I am and I don't think I'll get to it)
Will be happy to review any changes - should be fairly straightforward.
Yes, init containers are "run to completion" always. , harryz wrote:
IIUC when the init containers of pod have not complete, the pod should be in status, which means will fail, if init containers run to completion, they will exit, still fail. I don't know what you mean exec should work on init containers?
the init container is still up and running even when the pod is in pending, we should be able to exec and debug why it's hung, for example.
add a straightforward exec implementation ptal.


Question: cadvisorapi "github.com/google/cadvisor/info/v1" cadvisorapiv2 "github.com/google/cadvisor/info/v2" <ins> "github.com/stretchr/testify/assert" </ins> "k8s.io/kubernetes/pkg/api" apierrors "k8s.io/kubernetes/pkg/api/errors" "k8s.io/kubernetes/pkg/api/resource"

is this a Node bug? cc as next oncall to follow up on this.
This is a node bug due to init containers
I may need help debugging - I see:
So I think the problem is in the error case - we're not properly recording that the start failed, and the next time we come in we somehow think the first container succeeded. I'll add some debugging to see if we can't recreate it.
Let's discuss on instead.


Question: cadvisorapi "github.com/google/cadvisor/info/v1" cadvisorapiv2 "github.com/google/cadvisor/info/v2" <ins> "github.com/stretchr/testify/assert" </ins> "k8s.io/kubernetes/pkg/api" apierrors "k8s.io/kubernetes/pkg/api/errors" "k8s.io/kubernetes/pkg/api/resource"

Kubelet issues a docker inspect call after creating a container, but for very short lived containers there is a window in which they could exit before being inspected in the docker manager. If the container PID is 0, we should check the ExitCode and other relevant fields and handle them appropriately. [1] This is the suspected cause of several flakes: /cc
looking at the logs 2 weird things: race described second container ran, which suggests inconsistent reporting of initialized status of init1 and choosing init2 as the next container ( vs ). I think 1 got it right and 2 got it wrong. From logs: I'd expect the second one to not have run in this case? Node logs:
Init containers have an implicit assumption that container statuses include results of previous SyncPod invocations. If that's not the case we may need to fix that (or at least mitigate it). On May 18, 2016, at 8:28 PM, Prashanth B wrote: looking at the logs 2 weird things: race described second container ran, which suggests inconsistent reporting of initialized status of init1 and choosing init2 as the next container ( vs ). I think 1 got it right and 2 got it wrong. From logs: May 18 15:58:33.380: INFO: pod-init--1d4b-11e6-80a4- started at 2016-05-18T15:58:04-07:00 (2+1 container statuses recorded) May 18 15:58:33.380: INFO: Init container init1 ready: false, restart count 0 May 18 15:58:33.380: INFO: Init container init2 ready: true, restart count 0 I'd expect the second one to not have run in this case? Node logs: — You are receiving this because you were mentioned. Reply to this email directly or view it on GitHub
This should be true. Is there any evidence that suggests the opposite? only shows conflicting view of whether init1 has run.
If the container exits already and it's not an infra container, kubelet should not error out. Although even if it does, I failed to see how this could affect the readiness of the init container without checking the code further.


Question: cadvisorapi "github.com/google/cadvisor/info/v1" cadvisorapiv2 "github.com/google/cadvisor/info/v2" <ins> "github.com/stretchr/testify/assert" </ins> "k8s.io/kubernetes/pkg/api" apierrors "k8s.io/kubernetes/pkg/api/errors" "k8s.io/kubernetes/pkg/api/resource"

Each syncPod invocation, we inspect the list of init container in order looking for lastTerminationState. There could be a bug in that logic, or if we didn't get the full status, we might make a bad decision. When determining which container to run next, we always look for the last init container in the chain that has success, then run the next container (unless the pod is RestartNever). , Yu-Ju Hong wrote:
Saw this flake again. How do we verify that container status is not lost between syn pods?
... between calls to syncPods
The second sync clearly saw the init1 container, and decided to launch init2. See "InitContainersToKeep" in the message below.
containers to keep can include failed containers. , Yu-Ju Hong wrote:
Discussed on slack. In summary, the problem has to do with how we convert the kubecontainer.PodStatus to the api.PodStatus. If a container is terminated, the function makes the decisions: the pod restart policy is "Always", and there is a "failure reason" recorded for the given container, this means the pod worker tried to (re)start the container but failed. Set the state to "Waiting" and populate the reason. , the container state is simply "terminated". Init containers don't obey the restart policy like regular containers, and the logic will need to be amended to work with them. is working on a PR. In addition to that, kubelet should not error out if a user container exited before it could apply the oom score (which is best effort in this case). fixes that.
Ahh, thanks for digging into the issue. This is the exact the concern I had when I reviewed the initial implementation of init container, which blocks pr for a while. I even asked to help me verify this case, unfortunately we didn't reproduce the issue back then. :-(
I wish I'd seen the flakes in the runs prior to merge.
We didn't notice this until we started running the tests against docker v1.11.1, which is much faster and hit the race condition differently.
I suppose that's a good thing...
It is hard to reproduce this.


Question: cadvisorapi "github.com/google/cadvisor/info/v1" cadvisorapiv2 "github.com/google/cadvisor/info/v2" <ins> "github.com/stretchr/testify/assert" </ins> "k8s.io/kubernetes/pkg/api" apierrors "k8s.io/kubernetes/pkg/api/errors" "k8s.io/kubernetes/pkg/api/resource"

The extra check of last terminated container in your initial code makes me nervous, but we couldn't reproduce it due to that small race windows until we have a faster docker running on the node. :-)
I got "Pods should invoke init containers on a RestartAlways pod" flake (referenced from ): Logs:
All the fixes went in. Please reopen this issue if you see any more flakes.


Question: container_image( name = "image", <del> base = "@debian_jessie//image", </del> <ins> base = "@distroless_base//image", </ins> entrypoint = ["/kubemark"], files = ["//cmd/kubemark"], stamp = True,

There are CVEs happening in different images - we already migrated Kubmeark from busybox in , but we should go one step further and migrate the base image to distroless. There are some things that would need to be changed mentioned here: /sig scalability
Can I work on this? /assign
/remove-help
- sure that would be great.
Hi based on the discussion in , the kubemark image has dependency on and maybe more other utilities. So it seems impossible to migrate the base image to distroless? Or do we need something like debian-hyperkube-base?
The mount one we may get rid of - basically we should be able to switch to FakeMounter:


Question: "k8s.io/component-base/metrics" "k8s.io/klog/v2" podutil "k8s.io/kubernetes/pkg/api/v1/pod" <del> "k8s.io/kubernetes/pkg/apis/apps" </del> kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/prober/results" )

The metric used to report acutal pod in the label , but now I see the podname prefix only. The full pod name in the label, so I can trace the metric to the failing pod. Check raw api and verify the actual pod name with uids. <details</detailsAzure AKS <details</detailsNo response No response No response_
/sig node
/triage accepted /priority important-soon /cc As discussed this is a change in functionaloty that wasn't highlighted in release notes. Need to decide on what do do with it. I understand it's a change of functionality. what is the side effect of this for you? Does the label without last part not informational enough?
I've been using the metric in my custom grafana dashboard that shows CPU, RAM, IO usage of pods with graph of increases in probes. If the apps are stable, the probe graph looks steadily. I can see if some irregularities in system metrics are influencing probes of respective pod. Also, if readiness probe is failing, the pod is removed from service, but is not restarted so I must have knowledge of pods with failing readiness probes. I attach the screenshot of my graph. ! So I'd welcome the possibility to pair the probes with exact pod and thank you for looking into it.
are we thinking that this resulted in an undesirable change? Spologies for being a bit late to reply, I'm returning from a brief paternity leave staycation.
This issue is labeled with but has not been updated in over 90 days, and should be re-triaged. Important-soon issues must be staffed and worked on either currently, or very soon, ideally in time for the next release. You can: Confirm that this issue is still relevant with (org members only) Deprioritize it with or Close this issue with For more details on the triage process, see /remove-triage accepted
Well this is awkward... I still believe the actual behavior does not respect the intended idea...
This issue makes it hard to link probe metrics with right pods. The value of pod label in and are also inconsistent with how it is used in other kubelet metrics, see: vs Definitely a bug, regression since 1.25.
I can't get anyone to look at the fix/revert: you may have better luck
/milestone v1.28
discussion in
You've pushed it through, congrats!


Question: legacyregistry.MustRegister(DeprecatedEvictionStatsAge) legacyregistry.MustRegister(DeprecatedDevicePluginRegistrationCount) legacyregistry.MustRegister(DeprecatedDevicePluginAllocationLatency) <ins> legacyregistry.MustRegister(RunningContainerCount) legacyregistry.MustRegister(RunningPodCount) </ins> if utilfeature.DefaultFeatureGate.Enabled(features.DynamicKubeletConfig) { legacyregistry.MustRegister(AssignedConfig) legacyregistry.MustRegister(ActiveConfig)

Some kubelet metrics use non-standard prometheus collectors unnecessarily. Specifically, we have which can be simple gauges. Since we cannot migrate non-standard prometheus collectors using the internal metrics stability framework, we will need to convert these metrics into standard prometheus variants. /sig instrumentation /assign
GitHub didn't allow me to assign the following users: solodov. Note that only and repo collaborators can be assigned and that issues/PRs can only have 10 assignees at the same time. For more information please see <detailsIn response to : Some kubelet metrics use non-standard prometheus collectors unnecessarily. Specifically, we have which can be simple gauges. Since we cannot migrate non-standard prometheus collectors using the internal metrics stability framework, we will need to convert these metrics into standard prometheus variants. /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
recommends that we generate those metrics in the GenericPLEG during the relist. We can generate the data directly from the pods call there ().
I would be interested to work on this issue if no one else is working on it currently
Go for it!
/assign


Question: apiVersion: v1 metadata: name: etcd <del> creationTimestamp: </del> labels: name: etcd spec:

Kubernetes version (use ): Environment: Cloud provider or hardware configuration: Vagrant OS (e.g. from /etc/os-release): CoreOS Kernel (e.g. ):Install tools: Vagrant & Others: What happened: What you expected to happen: Success How to reproduce it (as minimally and precisely as possible): Followed instructions at Tried to run Received: Anything else do we need to know: From googling, it seems several of the fields in the config are for when using OpenShift origin. But creating the replication controller is a prerequisite to launching OpenShift Origin. Chicken & egg problem.