Question: fi } <ins> create_csi_crd() { echo "create_csi_crd $1" YAML_FILE=${KUBE_ROOT}/cluster/addons/storage-crds/$1.yaml if [ -e $YAML_FILE ]; then echo "Create $1 crd" ${KUBECTL} --kubeconfig="${CERT_DIR}/admin.kubeconfig" create -f $YAML_FILE else echo "No $1 available." fi } </ins> function print_success { if [[ "${START_MODE}" != "kubeletonly" ]]; then if [[ "${ENABLE_DAEMON}" = false ]]; then

<!-- Please only use this template for submitting enhancement requests --What would you like to be : a new addon to addon manager "that automatically installs CSI CRDs if CSIDriverRegistry or CSINodeInfo feature gates are true". But this is either broken or this addon manager isn't active in a cluster brough up with: This should work automatically also for such local clusters. Why is this needed: The CSI E2E tests are now expecting that the cluster has the CRDs and fail when that isn't the case. /kind feature (or perhaps a bug, not sure) /cc
/assign
Is this expected to be installed in other clusters? A lot of clusters are not going to be using cluster/* scripts.


Question: framework.Failf("Expected `not found` error, got: %v", err) } }) <ins> ginkgo.It("should apply changes to a resourcequota status", func() { ns := f.Namespace.Name rqClient := f.ClientSet.CoreV1().ResourceQuotas(ns) rqName := "e2e-rq-status-" + utilrand.String(5) label := map[string]string{"e2e-rq-label": rqName} labelSelector := labels.SelectorFromSet(label).String() w := &cache.ListWatch{ WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { options.LabelSelector = labelSelector return rqClient.Watch(context.TODO(), options) }, } rqList, err := f.ClientSet.CoreV1().ResourceQuotas("").List(context.TODO(), metav1.ListOptions{LabelSelector: labelSelector}) framework.ExpectNoError(err, "failed to list Services") ginkgo.By(fmt.Sprintf("Creating resourceQuota %q", rqName)) resourceQuota := &v1.ResourceQuota{ ObjectMeta: metav1.ObjectMeta{ Name:   rqName, Labels: label, }, Spec: v1.ResourceQuotaSpec{ Hard: v1.ResourceList{ v1.ResourceCPU:    resource.MustParse("500m"), v1.ResourceMemory: resource.MustParse("500Mi"), }, }, } _, err = createResourceQuota(f.ClientSet, ns, resourceQuota) framework.ExpectNoError(err) initialResourceQuota, err := rqClient.Get(context.TODO(), rqName, metav1.GetOptions{}) framework.ExpectNoError(err) framework.ExpectEqual(*initialResourceQuota.Spec.Hard.Cpu(), resource.MustParse("500m"), "Hard cpu value for ResourceQuota %q is %s not 500m.", initialResourceQuota.Name, initialResourceQuota.Spec.Hard.Cpu().String()) framework.Logf("Resource quota %q reports spec: hard cpu limit of %s", rqName, initialResourceQuota.Spec.Hard.Cpu()) framework.ExpectEqual(*initialResourceQuota.Spec.Hard.Memory(), resource.MustParse("500Mi"), "Hard memory value for ResourceQuota %q is %s not 500Mi.", initialResourceQuota.Name, initialResourceQuota.Spec.Hard.Memory().String()) framework.Logf("Resource quota %q reports spec: hard memory limit of %s", rqName, initialResourceQuota.Spec.Hard.Memory()) ginkgo.By(fmt.Sprintf("Updating resourceQuota %q /status", rqName)) var updatedResourceQuota *v1.ResourceQuota hardLimits := quota.Add(v1.ResourceList{}, initialResourceQuota.Spec.Hard) err = retry.RetryOnConflict(retry.DefaultRetry, func() error { updateStatus, err := rqClient.Get(context.TODO(), rqName, metav1.GetOptions{}) framework.ExpectNoError(err, "Unable to get ResourceQuota %q", rqName) updateStatus.Status = v1.ResourceQuotaStatus{ Hard: hardLimits, } updatedResourceQuota, err = rqClient.UpdateStatus(context.TODO(), updateStatus, metav1.UpdateOptions{}) return err }) framework.ExpectNoError(err, "Failed to update resourceQuota") ginkgo.By(fmt.Sprintf("Confirm /status for %q resourceQuota via watch", rqName)) ctx, cancel := context.WithTimeout(context.Background(), f.Timeouts.PodStartShort) defer cancel() _, err = watchtools.Until(ctx, rqList.ResourceVersion, w, func(event watch.Event) (bool, error) { if rq, ok := event.Object.(*v1.ResourceQuota); ok { found := rq.Name == updatedResourceQuota.Name && rq.Namespace == ns && apiequality.Semantic.DeepEqual(rq.Status.Hard, updatedResourceQuota.Spec.Hard) if !found { framework.Logf("observed resourceQuota %q in namespace %q with hard status: %#v", rq.Name, rq.Namespace, rq.Status.Hard) return false, nil } framework.Logf("Found resourceQuota %q in namespace %q with hard status: %#v", rq.Name, rq.Namespace, rq.Status.Hard) return found, nil } framework.Logf("Observed event: %+v", event.Object) return false, nil }) framework.ExpectNoError(err, "failed to locate ResourceQuota %q in namespace %q", updatedResourceQuota.Name, ns) framework.Logf("ResourceQuota %q /status was updated", updatedResourceQuota.Name) // Sync resourceQuota list before patching /status rqList, err = f.ClientSet.CoreV1().ResourceQuotas("").List(context.TODO(), metav1.ListOptions{LabelSelector: labelSelector}) framework.ExpectNoError(err, "failed to list Services") ginkgo.By("Patching hard spec values for cpu & memory") xResourceQuota, err := rqClient.Patch(context.TODO(), updatedResourceQuota.Name, types.StrategicMergePatchType, []byte(`{"spec":{"hard":{"cpu":"1","memory":"1Gi"}}}`), metav1.PatchOptions{}) framework.ExpectNoError(err, "Could not patch resourcequota %q. Error: %v", xResourceQuota.Name, err) framework.Logf("Resource quota %q reports spec: hard cpu limit of %s", rqName, xResourceQuota.Spec.Hard.Cpu()) framework.Logf("Resource quota %q reports spec: hard memory limit of %s", rqName, xResourceQuota.Spec.Hard.Memory()) ginkgo.By(fmt.Sprintf("Patching %q /status", rqName)) hardLimits = quota.Add(v1.ResourceList{}, xResourceQuota.Spec.Hard) rqStatusJSON, err := json.Marshal(hardLimits) framework.ExpectNoError(err) patchedResourceQuota, err := rqClient.Patch(context.TODO(), rqName, types.StrategicMergePatchType, []byte(`{"status": {"hard": `+string(rqStatusJSON)+`}}`), metav1.PatchOptions{}, "status") framework.ExpectNoError(err) ginkgo.By(fmt.Sprintf("Confirm /status for %q resourceQuota via watch", rqName)) ctx, cancel = context.WithTimeout(context.Background(), f.Timeouts.PodStartShort) defer cancel() _, err = watchtools.Until(ctx, rqList.ResourceVersion, w, func(event watch.Event) (bool, error) { if rq, ok := event.Object.(*v1.ResourceQuota); ok { found := rq.Name == patchedResourceQuota.Name && rq.Namespace == ns && apiequality.Semantic.DeepEqual(rq.Status.Hard, patchedResourceQuota.Spec.Hard) if !found { framework.Logf("observed resourceQuota %q in namespace %q with hard status: %#v", rq.Name, rq.Namespace, rq.Status.Hard) return false, nil } framework.Logf("Found resourceQuota %q in namespace %q with hard status: %#v", rq.Name, rq.Namespace, rq.Status.Hard) return found, nil } framework.Logf("Observed event: %+v", event.Object) return false, nil }) framework.ExpectNoError(err, "failed to locate ResourceQuota %q in namespace %q", patchedResourceQuota.Name, ns) framework.Logf("ResourceQuota %q /status was patched", patchedResourceQuota.Name) ginkgo.By(fmt.Sprintf("Get %q /status", rqName)) rqResource := schema.GroupVersionResource{Group: "", Version: "v1", Resource: "resourcequotas"} unstruct, err := f.DynamicClient.Resource(rqResource).Namespace(ns).Get(context.TODO(), resourceQuota.Name, metav1.GetOptions{}, "status") framework.ExpectNoError(err) rq, err := unstructuredToResourceQuota(unstruct) framework.ExpectNoError(err, "Getting the status of the resource quota %q", rq.Name) framework.ExpectEqual(*rq.Status.Hard.Cpu(), resource.MustParse("1"), "Hard cpu value for ResourceQuota %q is %s not 1.", rq.Name, rq.Status.Hard.Cpu().String()) framework.Logf("Resourcequota %q reports status: hard cpu of %s", rqName, rq.Status.Hard.Cpu()) framework.ExpectEqual(*rq.Status.Hard.Memory(), resource.MustParse("1Gi"), "Hard memory value for ResourceQuota %q is %s not 1Gi.", rq.Name, rq.Status.Hard.Memory().String()) framework.Logf("Resourcequota %q reports status: hard memory of %s", rqName, rq.Status.Hard.Memory()) // Sync resourceQuota list before repatching /status rqList, err = f.ClientSet.CoreV1().ResourceQuotas("").List(context.TODO(), metav1.ListOptions{LabelSelector: labelSelector}) framework.ExpectNoError(err, "failed to list Services") ginkgo.By(fmt.Sprintf("Repatching %q /status before checking Spec is unchanged", rqName)) newHardLimits := v1.ResourceList{ v1.ResourceCPU:    resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("2Gi"), } rqStatusJSON, err = json.Marshal(newHardLimits) framework.ExpectNoError(err) repatchedResourceQuota, err := rqClient.Patch(context.TODO(), rqName, types.StrategicMergePatchType, []byte(`{"status": {"hard": `+string(rqStatusJSON)+`}}`), metav1.PatchOptions{}, "status") framework.ExpectNoError(err) framework.ExpectEqual(*repatchedResourceQuota.Status.Hard.Cpu(), resource.MustParse("2"), "Hard cpu value for ResourceQuota %q is %s not 2.", repatchedResourceQuota.Name, repatchedResourceQuota.Status.Hard.Cpu().String()) framework.Logf("Resourcequota %q reports status: hard cpu of %s", repatchedResourceQuota.Name, repatchedResourceQuota.Status.Hard.Cpu()) framework.ExpectEqual(*repatchedResourceQuota.Status.Hard.Memory(), resource.MustParse("2Gi"), "Hard memory value for ResourceQuota %q is %s not 2Gi.", repatchedResourceQuota.Name, repatchedResourceQuota.Status.Hard.Memory().String()) framework.Logf("Resourcequota %q reports status: hard memory of %s", repatchedResourceQuota.Name, repatchedResourceQuota.Status.Hard.Memory()) _, err = watchtools.Until(ctx, rqList.ResourceVersion, w, func(event watch.Event) (bool, error) { if rq, ok := event.Object.(*v1.ResourceQuota); ok { found := rq.Name == patchedResourceQuota.Name && rq.Namespace == ns && *rq.Status.Hard.Cpu() == resource.MustParse("2") && *rq.Status.Hard.Memory() == resource.MustParse("2Gi") if !found { framework.Logf("observed resourceQuota %q in namespace %q with hard status: %#v", rq.Name, rq.Namespace, rq.Status.Hard) return false, nil } framework.Logf("Found resourceQuota %q in namespace %q with hard status: %#v", rq.Name, rq.Namespace, rq.Status.Hard) return found, nil } framework.Logf("Observed event: %+v", event.Object) return false, nil }) framework.ExpectNoError(err, "failed to locate ResourceQuota %q in namespace %q", patchedResourceQuota.Name, ns) err = wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) { resourceQuotaResult, err := rqClient.Get(context.TODO(), rqName, metav1.GetOptions{}) framework.ExpectNoError(err) if apiequality.Semantic.DeepEqual(resourceQuotaResult.Spec.Hard.Cpu(), resourceQuotaResult.Status.Hard.Cpu()) { framework.ExpectEqual(*resourceQuotaResult.Status.Hard.Cpu(), resource.MustParse("1"), "Hard cpu value for ResourceQuota %q is %s not 1.", repatchedResourceQuota.Name, repatchedResourceQuota.Status.Hard.Cpu().String()) framework.ExpectEqual(*resourceQuotaResult.Status.Hard.Memory(), resource.MustParse("1Gi"), "Hard memory value for ResourceQuota %q is %s not 1Gi.", repatchedResourceQuota.Name, repatchedResourceQuota.Status.Hard.Memory().String()) framework.Logf("ResourceQuota %q Spec was unchanged and /status reset", resourceQuotaResult.Name) return true, nil } return false, nil }) framework.ExpectNoError(err) }) </ins> }) var _ = SIGDescribe("ResourceQuota [Feature:ScopeSelectors]", func() {

[X] Test approval issue: [X] Test PR: 19 Oct 2022 [x] Two weeks soak end date: 2 Nov 2022 [x] Test promotion PR: According to this APIsnoop query, there are still three remaining ResourceQuota status endpoints which are untested. patchCoreV1NamespacedResourceQuotaStatusreplaceCoreV1NamespacedResourceQuotaStatusreadCoreV1NamespacedResourceQuotaStatus Using a number of existing e2e test practices a new has been created for 3 ResourceQuota Status endpoints. The e2e logs for this test are listed below. This query shows which Resource Status endpoints are hit within a short period of running this e2e test If a test with these calls gets merged, test coverage will go up by 3 point This test is also created with the goal of conformance promotion. /sig testing /sig architecture /area conformance
/triage accepted
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: func(client kubeclientset.Interface, obj pkgruntime.Object) error { daemonset := obj.(*extensionsv1.DaemonSet) glog.V(4).Infof("Attempting to delete daemonset: %s/%s", daemonset.Namespace, daemonset.Name) <del> err := client.Extensions().DaemonSets(daemonset.Namespace).Delete(daemonset.Name, &metav1.DeleteOptions{}) </del> <ins> orphanDependents := false err := client.Extensions().DaemonSets(daemonset.Namespace).Delete(daemonset.Name, &metav1.DeleteOptions{OrphanDependents: &orphanDependents}) </ins> if err != nil { glog.Errorf("Error deleting daemonset %s/%s/: %v", daemonset.Namespace, daemonset.Name, err) } else {

Problem: When users sends a cascading deletion request to federation i.e they set orphanDependents=false, then federation should also set orphanDependents=false, when it sends the request to underlying clusters.
Fixed by


Question: return &ConfigFlags{ Insecure:   &insecure, <del> Timeout:    stringptr("0"), KubeConfig: stringptr(""), CacheDir:         stringptr(defaultCacheDir), ClusterName:      stringptr(""), AuthInfoName:     stringptr(""), Context:          stringptr(""), Namespace:        stringptr(""), APIServer:        stringptr(""), TLSServerName:    stringptr(""), CertFile:         stringptr(""), KeyFile:          stringptr(""), CAFile:           stringptr(""), BearerToken:      stringptr(""), Impersonate:      stringptr(""), ImpersonateUID:   stringptr(""), </del> <ins> Timeout:    utilpointer.String("0"), KubeConfig: utilpointer.String(""), CacheDir:         utilpointer.String(defaultCacheDir), ClusterName:      utilpointer.String(""), AuthInfoName:     utilpointer.String(""), Context:          utilpointer.String(""), Namespace:        utilpointer.String(""), APIServer:        utilpointer.String(""), TLSServerName:    utilpointer.String(""), CertFile:         utilpointer.String(""), KeyFile:          utilpointer.String(""), CAFile:           utilpointer.String(""), BearerToken:      utilpointer.String(""), Impersonate:      utilpointer.String(""), ImpersonateUID:   utilpointer.String(""), </ins> ImpersonateGroup: &impersonateGroup, usePersistentConfig: usePersistentConfig,

The discovery cache for kubectl is defaulted to 10 minutes: That means that every 10 minutes of using against a cluster, it runs API requests against all group versions, which can take quite some time on clusters with many operators installed (like , , , etc.). I previously wrote that shows that every 10 minutes a simple makes 170 API requests against a test cluster, vs 4 API requests when the cache exists. That results in significant productivity implications against such clusters. Here is a time comparison between a request that runs the discovery cache vs one that doesn't: The discovery cache doesn't really have to run every 10 minutes, as CRDs don't change that often. A lot of unnecessary load is created on clients and servers. This issue is raised to change the discovery cache default to 24 hours and also to make the discovery cache ttl configurable. Running the discovery cache every 10 minutes has a significant productivity impact on using on clusters with many CRDs as it is taking time to run these unnecessary requests. The discovery cache doesn't really have to run every 10 minutes, as CRDs don't change that often. A lot of unnecessary load is created on clients and servers.
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig cli
I would like to work on this issue.
/assign


Question: if sha1checker.CheckRoundTripError(err) { sha1checker.IncreaseMetricsCounter(req) <ins> if len(auditCtx.Event.Annotations["insecure-sha1.invalid-cert.kubernetes.io/"+req.URL.Hostname()]) == 0 { t.Errorf("expected audit annotations, got %#v", auditCtx.Event.Annotations) } </ins> } errorCounterVal := getSingleCounterValueFromRegistry(t, registry, "Test_checkForInsecureAlgorithmError")

go1.18 dropped support for certificates signed with the SHA-1 hash function - While this does improve security, and publicly trusted Certificate Authorities no longer issue such certificates, it is possible that user-provided certificates issued by private Certificate Authorities and used in webhooks or aggregated API servers do still use this. Similar to removal of support for certificates without subjectAltNames in go1.17, we need to: [x] add metrics when webhook/conversion/aggregated certificates with this hash are encountered - [x] document use of and the metric from 1 as a mitigation for clusters which still need to identify and replace use of such certificates - to release note and known issues [x] add audit annotation when SHA-1 cert is encountered - [x] update docs at and - /sig api-machinery instrumentation docs /priority important-soon /milestone v1.24 /kind bug documentation Upgraded 1.23 clusters to continue working as-is Configure an admission webhook, conversion webhook, or aggregated server with a serving certificate signed with the SHA-1 hash function. Upgrade to HEAD of 1.24. The certificate will no longer work by default No response 24 HEAD (go1.18-based) n/a n/a No response No response No response
it looks like NewMissingSANRoundTripperWrapperConstructor already has all the plumbing, and instead of missing SANs we could check the SignatureAlgorithm and output a distinct metric
had a not-yet-tested WIP branch at with a sketch of what it might look like
/assign who offered to pick this up
GitHub didn't allow me to assign the following users: stlaz. Note that only , repo collaborators and people who have commented on this issue/PR can be assigned. Additionally, issues/PRs can only have 10 assignees at the same time. For more information please see <detailsIn response to : /assign who offered to pick this up Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Indeed, please, you should trust Jordan :slightlysmilingface:
/assign
/triage accepted
These restrictions are on the GitHub side, this message is posted when it receives a specific error from the GitHub API while attempting to assign It is an annoying GitHub limitation, and one that seemingly does not exist in the web client vs the API.


Question: if sha1checker.CheckRoundTripError(err) { sha1checker.IncreaseMetricsCounter(req) <ins> if len(auditCtx.Event.Annotations["insecure-sha1.invalid-cert.kubernetes.io/"+req.URL.Hostname()]) == 0 { t.Errorf("expected audit annotations, got %#v", auditCtx.Event.Annotations) } </ins> } errorCounterVal := getSingleCounterValueFromRegistry(t, registry, "Test_checkForInsecureAlgorithmError")

adds the metrics. the release note on that issue describes the go change, links to go docs, describes use of the option, the new metrics, and the kube-apiserver logs what would you expect kubeadm to do with this? GODEBUG off by default and let users override GODEBUG on their own (is this possible)? GODEBUG by default on all clusters? GODEBUG by default only on upgrade? The risk of 1 is that a user upgrades without reading release notes and breaks a cluster that has SHA-1 certs in webhooks, etc The risk of 2 is that we propagate accepting insecure certificates longer, even in new clusters that we know don't have any pre-existing bad certs The risk of 3 is that it makes an upgraded 1.23 → 1.24 cluster and a new 1.24 cluster inconsistent
the majority of kubeadm users use kubeadm managed CA and certificates. there are less users that use external CA to sign their own certificates. sha1 attacks surfaced more than a before kubeadm's creation and Git hash collusion made the problem more known around 2017 - that was at a time when kubeadm did not support external CA. given these facts, an estimate on my side is that there are currently no kubeadm users that use sha1 certificates, but i would not exclude the possibility entirely. if such users exists, they are on they own and my vote goes for option 1, until they migrate their PKI to something secure. if the same users log tickets for us, our response would be "please use GODEBUG and migrate ASAP". all users should be able to see the notes in IMO, the PR needs an "action required" to appear on top of the release notes.
that's helpful, thanks agreed. weird, I set , but the label didn't get . the label manually.
and there's a way for them to set that envvar in kube-apiserver manifests when using kubeadm?
There's been a lot of intermittent github webhook delivery issues in the past few weeks or so, slash commands are going to be somewhat unreliable :/
they could use kubeadm patches to override the YAML manifest defaults - i.e. the contents of v1.Container -Env:


Question: }() // If leader election is enabled, run via LeaderElector until done and exit. <del> if c.LeaderElection != nil { c.LeaderElection.Callbacks = leaderelection.LeaderCallbacks{ </del> <ins> if cc.LeaderElection != nil { cc.LeaderElection.Callbacks = leaderelection.LeaderCallbacks{ </ins> OnStartedLeading: run, OnStoppedLeading: func() { utilruntime.HandleError(fmt.Errorf("lost master")) }, } <del> leaderElector, err := leaderelection.NewLeaderElector(*c.LeaderElection) </del> <ins> leaderElector, err := leaderelection.NewLeaderElector(*cc.LeaderElection) </ins> if err != nil { return fmt.Errorf("couldn't create leader elector: %v", err) }

PLEASE COORDINATE WITH SIG-SCHEDULING BEFORE WORKING ON THIS ISSUE. We want to avoid the situation where two people accidentally start work on the same fix. Main tracking issue: In the function creates a object and populates the field with an anonymous function. At the end of that function, is called, which contains the rest of our startup logic. Rather than split this across two functions, we should put all the startup logic in one named function called (not ) so that constructing the object is much more plain. /sig scheduling /kind cleanup /help /good-first-issue
This request has been marked as suitable for new contributors. Please ensure the request meets the requirements listed . If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : PLEASE COORDINATE WITH SIG-SCHEDULING BEFORE WORKING ON THIS ISSUE. We want to avoid the situation where two people accidentally start work on the same fix. Main tracking issue: Rather than split this across two functions, we should put all the startup logic in one named function called (not ) so that constructing the object is much more plain. /kind cleanup /help /good-first-issue </details
I'll pick this up! /assign


Question: package lifecycle import ( <ins> "fmt" </ins> "testing" "k8s.io/kubernetes/pkg/admission" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/client/unversioned/cache" "k8s.io/kubernetes/pkg/client/unversioned/testclient" <ins> "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/watch" </ins> ) // TestAdmission

We should have a single NamespaceLifecycle plugin that enforces the Namespace rules now that all providers are off NamespaceAutoProvision. We may also want to make this no longer a user choice to configure and hard-wire the server to always run this check first to simplify configuration errors. For an example


Question: pullQPS float32 // Optional, maximum burst QPS from the docker registry, must be positive if QPS is > 0.0 pullBurst int <ins> // Optional, no statistics will be available if omitted cadvisorClient CadvisorInterface cadvisorLock   sync.RWMutex } // SetCadvisorClient sets the cadvisor client in a thread-safe way. func (kl *Kubelet) SetCadvisorClient(c CadvisorInterface) { kl.cadvisorLock.Lock() defer kl.cadvisorLock.Unlock() kl.cadvisorClient = c } // GetCadvisorClient gets the cadvisor client. func (kl *Kubelet) GetCadvisorClient() CadvisorInterface { kl.cadvisorLock.RLock() defer kl.cadvisorLock.RUnlock() return kl.cadvisorClient </ins> } // Run starts the kubelet reacting to config updates

So today, cadvisor became wedged and just stopped paying attention to its socket. Not kubelet's fault. But when I would launch all I would get would be: It took me a bit of trolling through netstat and lsof to figure out that the problem was cadvisor being completely locked up. kill -9 cadvisor and everything started working well. I wonder if we should emit more messages during startup about what operation we are about to attempt, not just the results of operations, as I was blaming docker for a while....
kubelet shouldn't block on this. I'll take a look.
Any ideas what was locking cAdvisor? and was it just not responding to requests?
I don't have any idea why it was locked. If I find another one locked I'll hang onto it to debug. Journal had: obviously the 'kill' came from me. It would not respond to systemctl cadvisor stop. netstat looking at port 4194 showed me At which point I realized cadvisor wasn't handling its sockets right since so many long dead kubelet sockets hadn't been cleaned up, so I kill -9'd it.
I cut some interesting info from the cadvisor journal log (trying to cut hostnames) You can see the last message about failed to update stats for container ... happened Oct 08 01:06:01. It was then silent until I killed it Oct 08 19:20:13, even though I'm pretty sure dnf was still being launched every hour on the hour. So it was wedged for about 18 hours.
What kernel is this on? We've seen there is a kernel bug fixed in 3.12+ that sometimes hangs us forever on exit. We're not quite sure how it gets triggered...
16.3


Question: Bind(ctx context.Context, binding *v1.Binding, opts metav1.CreateOptions) error Evict(ctx context.Context, eviction *policy.Eviction) error GetLogs(name string, opts *v1.PodLogOptions) *restclient.Request <ins> ProxyGet(scheme, name, port, path string, params map[string]string) restclient.ResponseWrapper </ins> } // Bind applies the provided binding to the named pod in the current namespace (binding.Namespace is ignored).

Currently services/proxy is available in client-go, but pods/proxy resource is not. This requires manual usage via the rest client like This can be seen in Kubernetes e2e tests as well
/kind feature /sig api-machinery


Question: if newRS != nil && newRS.UID == rsUID { continue } <del> if len(podList.Items) > 0 { return true </del> <ins> for _, pod := range podList.Items { switch pod.Status.Phase { case v1.PodFailed, v1.PodSucceeded: // Don't count pods in terminal state. continue case v1.PodUnknown: // This happens in situation like when the node is temporarily disconnected from the cluster. // If we can't be sure that the pod is not running, we have to count it. return true default: // Pod is not in terminal phase. return true } </ins> } } return false

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: The deployment controller for a deployment with a single replica failed to create a new ReplicaSet when the old ReplicaSet included an Evicted pod. Typically, the eviction was caused by "node low on resource: nodefs". When we manually deleted the evicted pod, the new RS and accompanying pod were created immediately. We have seen this happen at least a dozen times. Here are the audit logs for a recent case: ! What you expected to happen: The deployment controller ignores the evicted pod and creates the updated replicaset/pod. How to reproduce it (as minimally and precisely as possible): To reproduce, you would need a way to trigger a kubelet-driven eviction on a node that is currently running a single-replica Deployment with . Wait for the pod to be evicted and the new pod to be created, confirm that the evicted pod still exists, and then make a trivial change to the deployment spec. The running pod will be deleted, and no new replicaset/pod will be created. This issue does not seem to occur for manual evictions (via ); pods I evict that way disappear from the API server immediately. Anything else we need to know?: Possibly related to and/or Environment: Kubernetes version (use ): v1.8.5-gke.0 Cloud provider or hardware configuration: GKE OS (e.g. from /etc/os-release): Container-Optimized OS
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Thanks for the detailed bug report!


Question: if newRS != nil && newRS.UID == rsUID { continue } <del> if len(podList.Items) > 0 { return true </del> <ins> for _, pod := range podList.Items { switch pod.Status.Phase { case v1.PodFailed, v1.PodSucceeded: // Don't count pods in terminal state. continue case v1.PodUnknown: // This happens in situation like when the node is temporarily disconnected from the cluster. // If we can't be sure that the pod is not running, we have to count it. return true default: // Pod is not in terminal phase. return true } </ins> } } return false

From Here's what happened: Deployment was updated by the user, so it scaled all old ReplicaSets to 0 old ReplicaSet was scaled to 0 (ReplicaSet's == 0), but still owned the evicted pod, which wouldn't be killed automatically because of the above comment (ReplicaSet's 0) Deployment found that there were still old Pods, and kept waiting for them to be gone ~(wait for old ReplicaSet's to be 0)~ (Edit: Deployment also checks those old Pods by listing them to make sure there are none, not just by checking ReplicaSet's status) before starting the new rollout. We can either modify Deployment controller (to ignore evicted old pods) or ReplicaSet controller (to kill evicted pods ~when it gets scaled down~), if this is not the intended behavior. WDYT?
I am actually debugging likely the same production issue with Recreate strategy in two clusters right now. (We run on patched 1.9.1.) I have traced it down to broken RS as well. in 1 cluster RS leaves around pod failed matching the selector in the other one those are evicted pods. all replicas are scaled down to 0 and report 0 in status but pods with valid controllerRefs to them are still present. Same is 0 for me but pod is still there I think the correct behaviour is for replicaSet scaled to 0 to delete all of its pod, not to leave them laying around
xref - this should fix it


Question: BalanceAttachedNodeVolumes:                  {Default: false, PreRelease: utilfeature.Alpha}, DynamicProvisioningScheduling:               {Default: false, PreRelease: utilfeature.Alpha}, VolumeSubpathEnvExpansion:                   {Default: false, PreRelease: utilfeature.Alpha}, <ins> KubeletPluginsWatcher:                       {Default: false, PreRelease: utilfeature.Alpha}, </ins> // inherited features from generic apiserver, relisted here to get a conflict if it is changed // unintentionally on either side:


Question: return } } <del> // Listing pods shouldn't really fail, as we are just querying the informer cache. selector, err := metav1.LabelSelectorAsSelector(jobObj.Spec.Selector) if err != nil { utilruntime.HandleError(fmt.Errorf("parsing deleted job selector: %v", err)) return } pods, _ := jm.podStore.Pods(jobObj.Namespace).List(selector) for _, pod := range pods { if metav1.IsControlledBy(pod, jobObj) && hasJobTrackingFinalizer(pod) { jm.enqueueOrphanPod(pod) } } </del> <ins> jm.backupRemovePodFinalizers(jobObj) </ins> } // enqueueSyncJobImmediately tells the Job controller to invoke syncJob

When a Job is deleted (via Background cascade), we have a backup logic to remove finalizers from pods This logic could also be applied for any finished Job. While investigating a case similar to , we observed that the job controller didn't delete finalizers from Pods where the Job was deleted with Foreground cascading policy. Since the Job was already marked as Finished, it only makes sense to trigger the backup logic to remove finalizers, so that we can guard against any possible bug where a Job is marked as Finished but not all pod finalizers are removed. Note that I'm not aware of any other bug in tracking with finalizers.
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig apps
/remove-kind feature /kind cleanup
/wg batch
I am happy to give this a go.
/assign
Just to check if I have the logic chain right: We are talking about adding this logic If so, I should probably turn the backup-logic code into a function that then apply that in both places? I also would be unsure of where in the unit tests this would be tested, are you able to point me to how this is tested in the deletePod function? I assume we add a pod that is never included in the "uncounted" lists.
I would put it in the event handler, to be more of a "catch all" Yes, that's a good idea. I think you would have to force it quite a bit, as we have fixed all the bugs that we have found. For example, from the test: the controller, but do not start the workers (this will setup the event handlers) a job the pods the job the workers some inspiration here
Alright, thank you :))
Should the issue title be: Remove finalizers from Pods belonging to finished Jobs ?
If someone has set a custom finalizer on a Pod, even one owned by a Job, (when) would we remove that finalizer?
/retitle Remove tracking finalizers from Pods belonging to finished Jobs No, only the finalizer specific to job tracking


Question: message ResourceSliceList { // Standard list metadata // +optional <del> optional .k8s.io.apimachinery.pkg.apis.meta.v1.ListMeta listMeta = 1; </del> <ins> optional .k8s.io.apimachinery.pkg.apis.meta.v1.ListMeta metadata = 1; </ins> // Items is the list of resource ResourceSlices. repeated ResourceSlice items = 2;

Listing s on v1.31.0 returns an object with a field instead of the standard field for (list) objects: Listing s on v1.30.3 behaves correctly: This is also propagated to the Swagger spec (which is how I noticed it, since I maintain the Rust bindings for the client API). v1.31.0: v1.30.3: As explained above. v1.31.0 with feature gate enabled and runtime config . With kind, this is: the API server request as in the above curl command in any way you like as long as it lets you see the raw JSON response. No response <details</details<detailskind 0.23.0 </details<details</details<details</details<details</details<details</details
/sig api-machinery
It seems there may be a discrepancy in the API response format between v1.30.3 and v1.31.0. The object is expected to follow standard Kubernetes API conventions, which typically use the field for list objects.
This deviation might cause compatibility issues for API clients that rely on this structure.
If this deviation from the expected API structure wasn't intentional, it should be aligned with the standard conventions.
The presence of instead of could disrupt client integrations, as you've encountered with the Swagger spec and Rust bindings.
cc
/triage accepted
/assign
this must be fixed to json:"metadata" protobuf:"bytes,1,opt,name=metadata"`` the serialization will not work with standard clients
/priority critical-urgent
Shoot, that's clearly a mistake.


Question: package gce import ( <ins> "flag" </ins> "fmt" <ins> "net" </ins> "net/http" "sort" "strconv"

From GCE email today: Current behavior: You are not required to add firewall rules for Network LB health checks today Action required: We are introducing three new ranges for health checking: 209.85.152.0/22, 209.85.204.0/22, and 35.191.0.0/16. You need to ensure that your firewall allows traffic from 209.85.152.0/22, 209.85.204.0/22, and 35.191.0.0/16. The health check probes to your backend instances will come from one of these ranges. We have until Jan 31, 2018 - 1 year.
/assign this issue explains the confusion I had several weeks ago when speaking with and you about L4 health checks. I didn't realize a firewall rule isn't currently required.
/assign
Is this done now?
Not yet, there is a WIP on .
/close


Question: kl.sourcesReady.AddSource(u.Source) case e := <-plegCh: <ins> if e.Type == pleg.ContainerStarted { // record the most recent time we observed a container start for this pod. // this lets us selectively invalidate the runtimeCache when processing a delete for this pod // to make sure we don't miss handling graceful termination for containers we reported as having started. kl.lastContainerStartedTime.Add(e.ID, time.Now()) } </ins> if isSyncPodWorthy(e) { // PLEG event for a pod; sync it. if pod, ok := kl.podManager.GetPodByUID(e.ID); ok {

These tests have flaked consistently since merged: in many cases, they are the only failure in a test run: Either the functionality introduced by that PR is not reliable, or the test is faulty. /sig node /priority critical-urgent /assign
/milestone v1.19
/assign
Looks like this is failing ~5% of pull-kubernetes-node-e2e runs
FYI
Looks like there might still be a race somewhere looking at the kubelet logs Success case: Fail case:
Looking at it more, it seems like is unexpectedly empty here Success: Fail:
Stacks of interest I think we might be dealing with a stale If there is a call before the container is to the Pod in the runtime and the is called within the 2 second default cache period, it seems possible that the in the could have no containers.
what do you think of this analysis? If so, any obvious solutions occur to you?
If I'm correct, didn't introduce this; just the e2e test exposed an existing bug in the kubelet.
That all looks correct. I'm taking a look at the proposed solutions


Question: TimeoutSeconds:      5, // default 1s can be pretty aggressive in CI environments with low resources FailureThreshold:    1, } <del> pod := gRPCServerPodSpec(nil, livenessProbe, "etcd") </del> <ins> pod := gRPCServerPodSpec(nil, livenessProbe, "agnhost") </ins> RunLivenessTest(ctx, f, pod, 1, defaultObservationTimeout) })

Tests in are using image for gRPC probe tests: should not be restarted with a GRPC liveness probe should be restarted with a GRPC liveness probe We should use image for these tests as it is much smaller size. /sig node /kind cleanup /priority important-soon /area tests /help-wanted /good-first-issue KEP:
This request has been marked as suitable for new contributors. Please ensure that the issue body includes answers to the following questions: Why are we solving this issue? To address this issue, are there any code changes? If there are code changes, what needs to be done in the code and what places can the assignee treat as reference points? Does this issue have zero to low barrier of entry? How can the assignee reach out to you for help? For more details on the requirements of such an issue, please see and ensure that they are met. If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : Tests in are using image for gRPC probe tests: /kind cleanup /priority important-soon /area tests /help-wanted /good-first-issue Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
The label(s) cannot be applied, because the repository doesn't have them. <detailsIn response to : Tests in are using image for gRPC probe tests: /kind cleanup /priority important-soon /area tests /help-wanted /good-first-issue Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Hi can I work on this issue? I'm new to Kubernetes and want to contribute!
Should I change to in this file?
Yes.


Question: TimeoutSeconds:      5, // default 1s can be pretty aggressive in CI environments with low resources FailureThreshold:    1, } <del> pod := gRPCServerPodSpec(nil, livenessProbe, "etcd") </del> <ins> pod := gRPCServerPodSpec(nil, livenessProbe, "agnhost") </ins> RunLivenessTest(ctx, f, pod, 1, defaultObservationTimeout) })

Also pass proper parameters to agnhost to enable gRPC endpoint, change ports, and run tests to ensure that this worked.
Thanks for your reply! Will let you know if I have more questions as I work
/assign
/assign


Question: } } <ins> func TestCheckUmountError(t *testing.T) { target := "/test/path" withSafeNotMountedBehavior := true command := exec.Command("uname", "-r") // dummy command return status 0 if err := command.Run(); err != nil { t.Errorf("Faild to exec dummy command. err: %s", err) } testcases := []struct { output   []byte err      error expected bool }{ { err:      errors.New("wait: no child processes"), expected: true, }, { output:   []byte("umount: /test/path: not mounted."), err:      errors.New("exit status 1"), expected: true, }, { output:   []byte("umount: /test/path: No such file or directory"), err:      errors.New("exit status 1"), expected: false, }, } for _, v := range testcases { if err := checkUmountError(target, command, v.output, v.err, withSafeNotMountedBehavior); (err == nil) != v.expected { if v.expected { t.Errorf("Expected to return nil, but did not. err: %s", err) } else { t.Errorf("Expected to return error, but did not.") } } } } </ins> func TestFormat(t *testing.T) { const ( formatCount    = 5

A status of Pod mounted NFS PV remain as terminating when a node shutdown and restart. A Pod is completely terminated. Deploy a Pod mounted NFS PV Shutdown the node where the Pod is running Wait for 5 min and the Pod status become Restart the node The Pod status is still and the Pod is never terminated completely <detailsPrepare a NFS Server.(I used ) Deploy a Pod mounted NFS PV like this. Shutdown the node where the Pod is running.(In this case ) And the status of node02 becomes . After 5 min later(by default), the Pod status becomes . (The time a Pod becomes status depends on .) Then, restart the node and the status of node02 becomes . But the Pod status is still and the Pod is never terminated completely. It seems that a faild to delete Pod because of trying to unmount a not mounted directory. Force delete Pod. Delete Pod's directory on the node. </detailsNo response <details</details<detailsnone(on premise) </details<details</details<details</details<details</details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig storage
/sig node
It dose not happen using , so it seems to depend on NFS volume plugin.
/assign
I try to build and run kubelet with comment out this logic. Then it seems to get well. related: This change checks the result of instead of the mount point check. However, check logic is not implemented in and (these functions are used in NFS volume plugin), despite it is done in .
/remove-sig node /sig storage
Reopening, fix had to be reverted due to unit tests requiring root permissions ()
I am terribly sorry... Please wait a moment. I'll try to fix it.
no worries, just get to review the rework, making sure the units tests don't require root permissions
Thank you! Fixed in .
Are there any plans to backport this to older Kubernetes versions? This seems like a critical fix.


Question: Write-Output $c throw "Contents of /etc/secret/foo.txt are not as expected" } <del> if ($env:NODE_NAME_TEST -ne $env:COMPUTERNAME) { throw "NODE_NAME_TEST env var ($env:NODE_NAME_TEST) does not equal COMPUTERNAME ($env:COMPUTERNAME)" </del> <ins> # Windows ComputerNames cannot exceed 15 characters, which means that the $env:COMPUTERNAME # can only be a substring of $env:NODE_NAME_TEST. We compare it with the hostname instead. # The following comparison is case insensitive. if ($env:NODE_NAME_TEST -ine "$(hostname)") { throw "NODE_NAME_TEST env var ($env:NODE_NAME_TEST) does not equal hostname" </ins> } Write-Output "SUCCESS"` )

Windows GCE jobs: [sig-windows] [Feature:WindowsHostProcessContainers] [MinimumKubeletVersion:1.22] HostProcess containers should run as a process on the host/node [sig-windows] [Feature:WindowsHostProcessContainers] [MinimumKubeletVersion:1.22] HostProcess containers should support various volume mount types Since the jobs were reintroduced. Windows COMPUTERNAMEs are typically limited to 15 characters [1], while the kubelet nodes spawned in the GCE test jobs are much longer (e.g.: `). The names might be truncated, resulting in the test failure. [1] No response /sig windows /sig testing
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
dupe of /close
Closing this issue. <detailsIn response to : dupe of /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: Write-Output $c throw "Contents of /etc/secret/foo.txt are not as expected" } <del> if ($env:NODE_NAME_TEST -ne $env:COMPUTERNAME) { throw "NODE_NAME_TEST env var ($env:NODE_NAME_TEST) does not equal COMPUTERNAME ($env:COMPUTERNAME)" </del> <ins> # Windows ComputerNames cannot exceed 15 characters, which means that the $env:COMPUTERNAME # can only be a substring of $env:NODE_NAME_TEST. We compare it with the hostname instead. # The following comparison is case insensitive. if ($env:NODE_NAME_TEST -ine "$(hostname)") { throw "NODE_NAME_TEST env var ($env:NODE_NAME_TEST) does not equal hostname" </ins> } Write-Output "SUCCESS"` )

I want to run the e2e test but it failed. Reason: Node name and hostname is longer than 15 characters. But the ComputerName only used the first 15 characters. So they don't equal and the test failed. The e2etest should pass regardless of the hostname length. Deploy a Windows with long nodename (longer than 15 characters). Then, run e2e test. No response <details</details<details</details<details</details<details</details<details</details<details</details
/sig testing
Can you show the error message?
STEP: Destroying namespace "host-process-test-windows-2574" for this suite. • Failure [6.334 seconds] [sig-windows] [Feature:WindowsHostProcessContainers] [Excluded:WindowsDocker] [MinimumKubeletVersion:1.22] HostProcess containers should run as a process on the host/node [It] Nov 21 22:12:49.003: Expected <v1.PodPhase: Failed to equal <v1.PodPhase: Succeeded {"msg":"FAILED [sig-windows] [Feature:WindowsHostProcessContainers] [Excluded:WindowsDocker] [MinimumKubeletVersion:1.22] HostProcess containers should run as a process on the host/node","total":1,"completed":0,"skipped":1147,"failed":1,"failures":["[sig-windows] [Feature:WindowsHostProcessContainers] [Excluded:WindowsDocker] [MinimumKubeletVersion:1.22] HostProcess containers should run as a process on the host/node"]}
/sig windows does this hostname error ring a bell?
We discussed this a bit on slack, the test is using %COMPUTERNAME% to validate we are running on the host. This value is truncated to 15 chars on windows due to netbios. We could use the hostname, which isn't truncated, with a fallback to computer name for this validation to make the test validate more scenarios. Important to note that some scenarios might not work if in older environments that use netbios but for our test purposes this will work. Long discussion on netbios here:
We'll be working on this as part of a broader windows operational readiness work. Thanks wenli for digging into it.


Question: Write-Output $c throw "Contents of /etc/secret/foo.txt are not as expected" } <del> if ($env:NODE_NAME_TEST -ne $env:COMPUTERNAME) { throw "NODE_NAME_TEST env var ($env:NODE_NAME_TEST) does not equal COMPUTERNAME ($env:COMPUTERNAME)" </del> <ins> # Windows ComputerNames cannot exceed 15 characters, which means that the $env:COMPUTERNAME # can only be a substring of $env:NODE_NAME_TEST. We compare it with the hostname instead. # The following comparison is case insensitive. if ($env:NODE_NAME_TEST -ine "$(hostname)") { throw "NODE_NAME_TEST env var ($env:NODE_NAME_TEST) does not equal hostname" </ins> } Write-Output "SUCCESS"` )

/assign
Just for broader context... the reason wenli discovered this is, bc we heavily test cluster API over here :), where the dynamically generated host name appends machine-set + cluster characters, so the machines have names like this: to the machines initial name, and it's hardcoded. We want to try to improve this upstream , and have a proposal to do this in flight also:
/sig windows /remove-sig testing
The Kubernetes project currently lacks enough contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Mark this issue or PR as rotten with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle stale [1]:
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle rotten [1]:
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Reopen this issue or PR with Mark this issue or PR as fresh with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /close [1]:
Closing this issue. <detailsIn response to : The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs.


Question: Write-Output $c throw "Contents of /etc/secret/foo.txt are not as expected" } <del> if ($env:NODE_NAME_TEST -ne $env:COMPUTERNAME) { throw "NODE_NAME_TEST env var ($env:NODE_NAME_TEST) does not equal COMPUTERNAME ($env:COMPUTERNAME)" </del> <ins> # Windows ComputerNames cannot exceed 15 characters, which means that the $env:COMPUTERNAME # can only be a substring of $env:NODE_NAME_TEST. We compare it with the hostname instead. # The following comparison is case insensitive. if ($env:NODE_NAME_TEST -ine "$(hostname)") { throw "NODE_NAME_TEST env var ($env:NODE_NAME_TEST) does not equal hostname" </ins> } Write-Output "SUCCESS"` )

After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closedReopen this issue or PR with Mark this issue or PR as fresh with Offer to help out with [Issue Triage][1] Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/reopen being fixed by
Reopened this issue. <detailsIn response to : /reopen being fixed by Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
The Kubernetes project currently lacks enough contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Mark this issue or PR as rotten with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle stale [1]:
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle rotten [1]:
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs.


Question: Write-Output $c throw "Contents of /etc/secret/foo.txt are not as expected" } <del> if ($env:NODE_NAME_TEST -ne $env:COMPUTERNAME) { throw "NODE_NAME_TEST env var ($env:NODE_NAME_TEST) does not equal COMPUTERNAME ($env:COMPUTERNAME)" </del> <ins> # Windows ComputerNames cannot exceed 15 characters, which means that the $env:COMPUTERNAME # can only be a substring of $env:NODE_NAME_TEST. We compare it with the hostname instead. # The following comparison is case insensitive. if ($env:NODE_NAME_TEST -ine "$(hostname)") { throw "NODE_NAME_TEST env var ($env:NODE_NAME_TEST) does not equal hostname" </ins> } Write-Output "SUCCESS"` )

This bot triages issues according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Reopen this issue with Mark this issue as fresh with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /close not-planned [1]:
Closing this issue, marking it as "Not Planned". <detailsIn response to : The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closedReopen this issue with Mark this issue as fresh with Offer to help out with [Issue Triage][1] Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: } else { // Trigger immediate sync when spec is changed. jm.enqueueSyncJobImmediately(logger, curJob) <ins> } // if curJob is finished, remove the finalizer as a backup check. if IsJobFinished(curJob) { jm.backupRemovePodFinalizers(curJob) </ins> } // check if need to add a new rsync for ActiveDeadlineSeconds


Question: } templates.ActsAsRootCommand(cmds, filters, groups...) <del> cmds.AddCommand(kubectl.NewCmdVersion(f, out)) cmds.AddCommand(kubectl.NewCmdOptions(out)) </del> <ins> cmdVersion := kubectl.NewCmdVersion(f, out) cmdVersion.Example = kubefedVersionExample cmds.AddCommand(cmdVersion) cmdOptions := kubectl.NewCmdOptions(out) cmdOptions.Example = kubefedOptionsExample cmds.AddCommand(cmdOptions) </ins> return cmds }

Referring and ; both have string in place of in section Example. This needs to be corrected in kubefed docs autogeneration. cc also see
/assign


Question: Expect(err).NotTo(HaveOccurred()) }() <del> // Waiting for service to expose endpoint </del> <ins> // Waiting for service to expose endpoint. </ins> validateEndpointsOrFail(c, ns, serviceName, PortsByPodName{serverPodName: {servicePort}}) By("Retrieve sourceip from a pod on the same node")

https://k8s- Failed: [] Services should preserve source pod IP for traffic thru service cluster IP {Kubernetes e2e suite} Happened on a presubmit run in .
Sorry, it is a new e2e test I am still working on. It is not merged yet.
Don't worry about it, our flake detection isn't quite perfect yet :)
[FLAKE-PING] This flaky-test issue would love to have more attention...
[FLAKE-PING] This flaky-test issue would love to have more attention.
This new e2e test has been merged. Let's observe it few more days and close this issue if nothing bad happen.
[FLAKE-PING] This flaky-test issue would love to have more attention.
This does't look like a failure in new test, rather its a generic failure with serviceaccounts.
Thanks for the investigation. Yeah I think it is the serviceaccounts issue. I wonder when could we close this flaky issue. Seems like it is stable if serviceaccounts issue is fixed.
You can close it with a reference to the serviceaccounts issue - do you know the # for that one ?
[FLAKE-PING] This flaky-test issue would love to have more attention.
https://k8s- Failed: [] Services should preserve source pod IP for traffic thru service cluster IP {Kubernetes e2e suite}
https://k8s- Failed: [] Services should preserve source pod IP for traffic thru service cluster IP {Kubernetes e2e suite}
https://k8s- Failed: [] Services should preserve source pod IP for traffic thru service cluster IP {Kubernetes e2e suite}
https://k8s- Failed: [] Services should preserve source pod IP for traffic thru service cluster IP {Kubernetes e2e suite}
[FLAKE-PING] This flaky-test issue would love to have more attention.
Have a fix for this, I'm now keeping running tests on my own cluster. Will send out PR once verify the fix works.


Question: return converted, nil } <ins> // Deep copy the list before we invoke the converter to ensure that if the converter does mutate the // list (which it shouldn't, but you never know), it doesn't have any impact. convertedList := list.DeepCopy() convertedList.SetAPIVersion(desiredAPIVersion) </ins> convertedObjects, err := c.converter.Convert(list, toGVK.GroupVersion()) if err != nil { return nil, fmt.Errorf("conversion for %v failed: %w", in.GetObjectKind().GroupVersionKind(), err)

If a CR converter is wired in and it modifies the input list, the delegating CR converter does not protect against this. A converter cannot effect the input list Not really possible externally - it's a coding issue that the current converters don't hit. No response <details</details<detailsN/A </details<details</details<details</details<details</details<details</details
/sig api-machinery /assign
/triage accepted


Question: <g> <rect style="fill: #ffffff" x="2.35" y="476.5" width="91.3" height="54"/> <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="2.35" y="476.5" width="91.3" height="54"/> <del> <text font-size="12.7998" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="48" y="499.4"> </del> <ins> <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="48" y="499.4"> </ins> <tspan x="48" y="499.4">scheduling</tspan> <tspan x="48" y="515.4">actuator</tspan> </text> </g> <del> <text font-size="12.7998" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="-13" y="436.75"> </del> <ins> <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="-13" y="436.75"> </ins> <tspan x="-13" y="436.75">APIs</tspan> </text> <g>

Architecture picture: says kubecfg is deprecated. The architecture picture should be updated to show kubectl instead of kubecfg.
ref /cc


Question: <ins> { "ImportPath": "k8s.io/api", "GoVersion": "go1.8", "GodepVersion": "v79", "Packages": [ "./..." ], "Deps": [ { "ImportPath": "github.com/PuerkitoBio/purell", "Rev": "8a290539e2e8629dbc4e6bad948158f790ec31f4" }, { "ImportPath": "github.com/PuerkitoBio/urlesc", "Rev": "5bd2802263f21d8788851d5305584c82a5c75d7e" }, { "ImportPath": "github.com/emicklei/go-restful", "Rev": "ff4f55a206334ef123e4f79bbf348980da81ca46" }, { "ImportPath": "github.com/emicklei/go-restful/log", "Rev": "ff4f55a206334ef123e4f79bbf348980da81ca46" }, { "ImportPath": "github.com/go-openapi/jsonpointer", "Rev": "46af16f9f7b149af66e5d1bd010e3574dc06de98" }, { "ImportPath": "github.com/go-openapi/jsonreference", "Rev": "13c6e3589ad90f49bd3e3bbe2c2cb3d7a4142272" }, { "ImportPath": "github.com/go-openapi/spec", "Rev": "6aced65f8501fe1217321abf0749d354824ba2ff" }, { "ImportPath": "github.com/go-openapi/swag", "Rev": "1d0bd113de87027671077d3c71eb3ac5d7dbba72" }, { "ImportPath": "github.com/gogo/protobuf/proto", "Rev": "c0656edd0d9eab7c66d1eb0c568f9039345796f7" }, { "ImportPath": "github.com/gogo/protobuf/sortkeys", "Rev": "c0656edd0d9eab7c66d1eb0c568f9039345796f7" }, { "ImportPath": "github.com/golang/glog", "Rev": "44145f04b68cf362d9c4df2182967c2275eaefed" }, { "ImportPath": "github.com/google/gofuzz", "Rev": "44d81051d367757e1c7c6a5a86423ece9afcf63c" }, { "ImportPath": "github.com/mailru/easyjson/buffer", "Rev": "d5b7844b561a7bc640052f1b935f7b800330d7e0" }, { "ImportPath": "github.com/mailru/easyjson/jlexer", "Rev": "d5b7844b561a7bc640052f1b935f7b800330d7e0" }, { "ImportPath": "github.com/mailru/easyjson/jwriter", "Rev": "d5b7844b561a7bc640052f1b935f7b800330d7e0" }, { "ImportPath": "github.com/spf13/pflag", "Rev": "9ff6c6923cfffbcd502984b8e0c80539a94968b7" }, { "ImportPath": "github.com/ugorji/go/codec", "Rev": "ded73eae5db7e7a0ef6f55aace87a2873c5d2b74" }, { "ImportPath": "golang.org/x/net/http2", "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" }, { "ImportPath": "golang.org/x/net/http2/hpack", "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" }, { "ImportPath": "golang.org/x/net/idna", "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" }, { "ImportPath": "golang.org/x/net/lex/httplex", "Rev": "f2499483f923065a842d38eb4c7f1927e6fc6e6d" }, { "ImportPath": "golang.org/x/text/cases", "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" }, { "ImportPath": "golang.org/x/text/internal/tag", "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" }, { "ImportPath": "golang.org/x/text/language", "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" }, { "ImportPath": "golang.org/x/text/runes", "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" }, { "ImportPath": "golang.org/x/text/secure/bidirule", "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" }, { "ImportPath": "golang.org/x/text/secure/precis", "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" }, { "ImportPath": "golang.org/x/text/transform", "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" }, { "ImportPath": "golang.org/x/text/unicode/bidi", "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" }, { "ImportPath": "golang.org/x/text/unicode/norm", "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" }, { "ImportPath": "golang.org/x/text/width", "Rev": "2910a502d2bf9e43193af9d68ca516529614eed3" }, { "ImportPath": "gopkg.in/inf.v0", "Rev": "3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4" } ] } </ins>

But it doesn't have the Godeps/ folder. add the Godeps/. Ideally, we should not let the IntOrString pull in so much dependency to
There are no sig labels on this issue. Please by:<br(1) mentioning a sig: <bre.g., for API Machinery<br(2) specifying the label manually: <bre.g., for sig/scalability<br<brNote: method (1) will trigger a notification to the team. You can find the team list and label list


Question: Expect(err).NotTo(HaveOccurred()) } <ins> func pollReadWithTimeout(pet petTester, petNumber int, key, expectedVal string) error { err := wait.PollImmediate(time.Second, readTimeout, func() (bool, error) { val := pet.read(petNumber, key) if val == "" { return false, nil } else if val != expectedVal { return false, fmt.Errorf("expected value %v, found %v", expectedVal, val) } return true, nil }) if err == wait.ErrWaitTimeout { return fmt.Errorf("timed out when trying to read value for key %v from pet %d", key, petNumber) } return err } </ins> func isInitialized(pod api.Pod) bool { initialized, ok := pod.Annotations[petset.PetSetInitAnnotation] if !ok {

https://k8s- Failed: [] PetSet [Slow] [Feature:PetSet] [] Deploy clustered applications [Slow] [Feature:PetSet] should creating a working redis cluster [Feature:PetSet] {Kubernetes e2e suite}
The test failed because it read nothing back from the key for the "member with index 2". Looking at , I notice that it's writing to the "member with index 0" and expecting to read it back from the "member with index 2". I'm curious, is each index supposed to represent a separate redis instance? What is the consistency model here?
/cc
https://k8s- Failed: [] PetSet [Slow] [Feature:PetSet] [] Deploy clustered applications [Slow] [Feature:PetSet] should creating a working redis cluster [Feature:PetSet] {Kubernetes e2e suite}
https://k8s- Failed: [] PetSet [Slow] [Feature:PetSet] [] Deploy clustered applications [Slow] [Feature:PetSet] should creating a working redis cluster [Feature:PetSet] {Kubernetes e2e suite}
[FLAKE-PING] This flaky-test issue would love to have more attention.
It's master slave redis, you should be able to create the same petset with this: , fyi
https://k8s- Failed: [] PetSet [Slow] [Feature:PetSet] [] Deploy clustered applications [Slow] [Feature:PetSet] should creating a working redis cluster [Feature:PetSet] {Kubernetes e2e suite}
https://k8s- Failed: [] PetSet [Slow] [Feature:PetSet] [] Deploy clustered applications [Slow] [Feature:PetSet] should creating a working redis cluster [Feature:PetSet] {Kubernetes e2e suite}
[FLAKE-PING] This flaky-test issue would love to have more attention.
https://k8s- Failed: [] PetSet [Slow] [Feature:PetSet] [] Deploy clustered applications [Slow] [Feature:PetSet] should creating a working redis cluster [Feature:PetSet] {Kubernetes e2e suite}
[FLAKE-PING] This flaky-test issue would love to have more attention.


Question: Expect(err).NotTo(HaveOccurred()) } <ins> func pollReadWithTimeout(pet petTester, petNumber int, key, expectedVal string) error { err := wait.PollImmediate(time.Second, readTimeout, func() (bool, error) { val := pet.read(petNumber, key) if val == "" { return false, nil } else if val != expectedVal { return false, fmt.Errorf("expected value %v, found %v", expectedVal, val) } return true, nil }) if err == wait.ErrWaitTimeout { return fmt.Errorf("timed out when trying to read value for key %v from pet %d", key, petNumber) } return err } </ins> func isInitialized(pod api.Pod) bool { initialized, ok := pod.Annotations[petset.PetSetInitAnnotation] if !ok {

https://k8s- Failed: [] PetSet [Slow] [Feature:PetSet] [] Deploy clustered applications [Slow] [Feature:PetSet] should creating a working redis cluster [Feature:PetSet] {Kubernetes e2e suite}
https://k8s- Failed: [] PetSet [Slow] [Feature:PetSet] [] Deploy clustered applications [Slow] [Feature:PetSet] should creating a working redis cluster [Feature:PetSet] {Kubernetes e2e suite}
https://k8s- Failed: [] PetSet [Slow] [Feature:PetSet] [] Deploy clustered applications [Slow] [Feature:PetSet] should creating a working redis cluster [Feature:PetSet] {Kubernetes e2e suite}
[FLAKE-PING] This flaky-test issue would love to have more attention.
[FLAKE-PING] This flaky-test issue would love to have more attention.


Question: var clusterIDs []string capacity := p.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)] volSizeBytes := capacity.Value() <del> sz := int(volume.RoundUpSize(volSizeBytes, 1024*1024*1024)) </del> <ins> // Glusterfs creates volumes in units of GBs sz := int(volume.RoundUpSize(volSizeBytes, 1000*1000*1000)) </ins> glog.V(2).Infof("create volume of size: %d bytes and configuration %+v", volSizeBytes, p.provisionerConfig) if p.url == "" { glog.Errorf("REST server endpoint is empty")

Glusterfs+Heketi use GB units for expanding volume. Using GiB incorrectly leads to expanding the volume to smaller size than user requested. /sig storage cc


Question: // randBytes returns a random string consisting of the characters in // validBootstrapTokenChars, with the length customized by the parameter func randBytes(length int) (string, error) { <del> // len("0123456789abcdefghijklmnopqrstuvwxyz") = 36 which doesn't evenly divide // the possible values of a byte: 256 mod 36 = 4. Discard any random bytes we // read that are >= 252 so the bytes we evenly divide the character set. const maxByteValue = 252 </del> var ( <del> b     byte err   error </del> token = make([]byte, length) <ins> max   = new(big.Int).SetUint64(uint64(len(validBootstrapTokenChars))) </ins> ) <del> reader := bufio.NewReaderSize(rand.Reader, length*2) </del> for i := range token { <del> for { if b, err = reader.ReadByte(); err != nil { return "", err } if b < maxByteValue { break } </del> <ins> val, err := rand.Int(rand.Reader, max) if err != nil { return "", fmt.Errorf("could not generate random integer: %w", err) </ins> } <del> token[i] = validBootstrapTokenChars[int(b)%len(validBootstrapTokenChars)] </del> <ins> // Use simple operations in constant-time to obtain a byte in the a-z,0-9 // character range x := val.Uint64() res := x + 48 + (39 & ((9 - x) >> 8)) token[i] = byte(res) </ins> } return string(token), nil

This issue was reported in the Impact Attackers may determine Bootstrap Token values based on timing-based side channels. This may in turn allow them to establish further attacks against a Kubernetes cluster, including joining nodes to an existing cluster. Feasibility of this attack is contingent on a number of factors including but not limited to the underlying hardware platform running the Kubernetes system, and the deployment landscape. Description Kubernetes Bootstrap Tokens are bearer token credentials employed to authenticate requests against the API server, when creating new clusters or joining new nodes to an existing cluster. NCC Group identified a number of functions that handle the Bootstrap Token, and whose implementations are not constant-time. The function , in package , generates Bootstrap Token secret values; it does so by sampling alphanumeric characters from a table, based on a random secret index. The access time of a table element, and resulting cache memory contents, may vary with the access index, which may assist attackers with inferring partial or full information about a token value. Recommendation When generating a secret token value, ensure that the selection process from the alphanumeric table is done in constant-time. The problem is akin to base64 encoding and decoding in constant-time, as implemented by the BearSSL TLS library. Given a byte in the 0..35 range, which we want to map to a digit range (ASCII values 48 to 57) and lower case character range (ASCII values 97 to 122), the constant-time selection process can be achieved as follows: . In order to increase the level of assurance that the modulo operation is performed in constant-time on various platforms, and despite changes to compilers, consider modeling the modulo operation as an explicit multiply and shift operation. The 1991 Granlund- Montgomery paper explains this technique. Avoid performing regular expression operations on secret data. Consider performing constant-time comparison of values instead. Component kube-apiserver Location Function in file Function in file Anything else we need to know? See umbrella issue for current status of all issues created from these findings. The vendor gave this issue an ID of NCC-E003660-TTV and it was finding 15 of the report under the kube-apiserver section.


Question: // randBytes returns a random string consisting of the characters in // validBootstrapTokenChars, with the length customized by the parameter func randBytes(length int) (string, error) { <del> // len("0123456789abcdefghijklmnopqrstuvwxyz") = 36 which doesn't evenly divide // the possible values of a byte: 256 mod 36 = 4. Discard any random bytes we // read that are >= 252 so the bytes we evenly divide the character set. const maxByteValue = 252 </del> var ( <del> b     byte err   error </del> token = make([]byte, length) <ins> max   = new(big.Int).SetUint64(uint64(len(validBootstrapTokenChars))) </ins> ) <del> reader := bufio.NewReaderSize(rand.Reader, length*2) </del> for i := range token { <del> for { if b, err = reader.ReadByte(); err != nil { return "", err } if b < maxByteValue { break } </del> <ins> val, err := rand.Int(rand.Reader, max) if err != nil { return "", fmt.Errorf("could not generate random integer: %w", err) </ins> } <del> token[i] = validBootstrapTokenChars[int(b)%len(validBootstrapTokenChars)] </del> <ins> // Use simple operations in constant-time to obtain a byte in the a-z,0-9 // character range x := val.Uint64() res := x + 48 + (39 & ((9 - x) >> 8)) token[i] = byte(res) </ins> } return string(token), nil

The vendor considers this issue Informational Risk, High Impact, Undetermined Exploitability To view the original finding, begin on page 42 of the Test Environment Kubernetes 1.24.3
/sig security
/sig cluster-lifecycle
/assign
PR that attempts to fix this:
/triage accepted poked #sig-security at slack for review ^


Question: }, { in: &api.Pod{ <ins> Spec: api.PodSpec{ServiceAccountName: "serviceAccount1"}, }, fieldSelector: fields.ParseSelectorOrDie("spec.serviceAccountName=serviceAccount1"), expectMatch:   true, }, { in: &api.Pod{ Spec: api.PodSpec{SchedulerName: "serviceAccount1"}, }, fieldSelector: fields.ParseSelectorOrDie("spec.serviceAccountName=serviceAccount2"), expectMatch:   false, }, { in: &api.Pod{ </ins> Status: api.PodStatus{Phase: api.PodRunning}, }, fieldSelector: fields.ParseSelectorOrDie("status.phase=Running"),

Is this a BUG REPORT or FEATURE REQUEST?: /kind feature What happened: When trying to list object using field spec.ServiceAccountName, we are getting the below error message. field label not supported: spec.serviceAccountName What you expected to happen: To be able to fetch pods using the list and watch based on the field 'spec.serviceAccountName' How to reproduce it (as minimally and precisely as possible): in minikube run Anything else we need to know?: Description: Unable to fetch pods based on the field 'spec.serviceAccountName'. When trying to get an overview of RBAC privileges in a cluster it is hard to find the objects attached to service accounts Environment: Kubernetes version (use ): Client Version: version.Info{Major:"1", Minor:"11", GitVersion:"v1.11.0", GitCommit:"", GitTreeState:"clean", BuildDate:"2018-06-27T20:17:28Z", GoVersion:"go1.10.2", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"11", GitVersion:"v1.11.0", GitCommit:"", GitTreeState:"clean", BuildDate:"2018-06-27T20:08:34Z", GoVersion:"go1.10.2", Compiler:"gc", Platform:"linux/amd64"} Cloud provider or hardware configuration: minikube version: v0.28.0 OS (e.g. from /etc/os-release): NAME=Buildroot VERSION=2018.05 ID=buildroot VERSIONID=2018.05 PRETTYNAME="Buildroot 2018.05" Kernel (e.g. ): Linux minikube 4.16.14 SMP Mon Jun 11 16:20:15 UTC 2018 x86_64 GNU/Linux Install tools: kubeadm Others:
/sig architecture
You might do it by jsonpath filter
I intend to utilize it in the client-go list as an option
Not all fields are supported for field selector. There is a doc about field selector which is still under review: As far as I can see, supported field selectors for pods are listed here:
Didn't find that you have a pull request for it. :) Good luck!


Question: "fmt" "io/ioutil" "os" <ins> "sort" </ins> "strings" "testing"

/kind bug What happened: When building kubernetes-1.7.0 packages for openSUSE, results differ for each build in and other man pages. coming from that uses and probably sets do not have any specific order, so it would need to be sorted on output. What you expected to happen: Every build should give identical results. See https://reproducible- for why this matters. How to reproduce it (as minimally and precisely as possible): See Anything else we need to know?: Environment: Kubernetes version: 1.7.0 OS (e.g. from /etc/os-release): openSUSE Tumbleweed
/sig release
somehow this bug seems to have been forgotten - can you help getting the right people to have a look?
- can you please confirm that PR works for you?
I tested it and it looks good.
/assign I enjoy reading my highlights to find the issue has already been addressed. :)
Sorry this wasn't addressed sooner. It does seem like it fell into a black hole, and pinging was the right way to resolve that. :)
/area build-release
related to /reopen
Reopening this issue. <detailsIn response to : related to Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: } func (rc *resourceMetricsCollector) collectContainerStartTime(ch chan<- metrics.Metric, pod summary.PodStats, s summary.ContainerStats) { <del> if s.StartTime.Unix() == 0 { </del> <ins> if s.StartTime.Unix() <= 0 { </ins> return }

As described in , When the pod start or stop, negative metrics generated. generate the correct metrics pod restart No response <details</details<detailsnone </details<details</details<details</details<details</details<details</details
/sig node /sig instrumentation
can you please give a little bit more information on steps to repro. Is it happens all the time? Is the root cause known (I see a linked PR). /triage needs-information
This is a feedback from contributors in kubernetes-sigs/metrics-server. It is riggered on specific conditions. From the description of several issues, the way to reproduce should be pod start or stop. such as:
Do you know if this is specific to ?
I'm not sure. However, obtains data from the kubelet endpoint. If there is a negative value, it will cause statistical problems.
/triage accepted /assign
FYI I encounter this with . seems to be an uninitialized time type. The in is create from the response of cri RPC, this maybe cause either by cri-o or the metrics collect part of Kubernetes. I have not dig into it yet.


Question: # TODO: split this out into args when we no longer need to pipe stdout to a file #6428 - sh - -c <del> - '/glbc --verbose=true --apiserver-host=http://localhost:8080 --default-backend-service=kube-system/default-http-backend --sync-period=600s --running-in-cluster=false --use-real-cloud=true --config-file-path=/etc/gce.conf --healthz-port=8086 1>>/var/log/glbc.log 2>&1' </del> <ins> - 'exec /glbc --verbose=true --apiserver-host=http://localhost:8080 --default-backend-service=kube-system/default-http-backend --sync-period=600s --running-in-cluster=false --use-real-cloud=true --config-file-path=/etc/gce.conf --healthz-port=8086 1>>/var/log/glbc.log 2>&1' </ins> volumes: - hostPath: path: /etc/gce.conf

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: When kube-apiserver's livenessProbe suggests that pod is unhealthy, kubelet tries to kill its container with 30 seconds grace period. The container is killed ungracefully with SIGKILL when grace period passes. This happens each time. What you expected to happen: I expected that SIGTERM will be passed to kube-apiserver which will try to gracefully stop before 30 seconds pass (at least in some cases). How to reproduce it (as minimally and precisely as possible): Inject kube-apiserver failure (e.g. ) Watch kubelet logs for container's death You will see that container is killed after 30 seconds each time Anything else we need to know?: I believe the problem is that in kube-apiserver.manifest uses '/bin/sh -c CMD' as a PID1 process which doesn't pass SIGTERM signals to child process. See Environment: Kubernetes version (use ): v1.7.11
/sig node
Looks like without the pipe command will have to be removed which will make the logs hard to track. There is indeed some which makes kube-apiserver run in background and trapping SIGTERM then send a SIGTERM to apiserver's PID.
I think this is the case for all our components, not just kube-apiserver. - FYI
How about using 'exec'-based solution suggested here: In our case it means prepending 'exec': I did changed this in my cluster and the process seems to meet requirements: fd/1 and fd/2 are redirected to /var/log/kube- container is killed instantaneously If everyone is fine with such a solution, I can prepare PR adding 'exec ' in front of all kubernetes/cluster/saltbase/salt/kube-apiserver/*.manifest using '/bin/sh -c CMD' pattern.
Not sure if understand what you mean. Before there was no logs, so /bin/sh -c wasn't needed. -based solution seems to have all required properties. What is your concern about using it?
Umm. just ignore my previous comment. I just tried prepending and it works fine for me.


Question: # TODO: split this out into args when we no longer need to pipe stdout to a file #6428 - sh - -c <del> - '/glbc --verbose=true --apiserver-host=http://localhost:8080 --default-backend-service=kube-system/default-http-backend --sync-period=600s --running-in-cluster=false --use-real-cloud=true --config-file-path=/etc/gce.conf --healthz-port=8086 1>>/var/log/glbc.log 2>&1' </del> <ins> - 'exec /glbc --verbose=true --apiserver-host=http://localhost:8080 --default-backend-service=kube-system/default-http-backend --sync-period=600s --running-in-cluster=false --use-real-cloud=true --config-file-path=/etc/gce.conf --healthz-port=8086 1>>/var/log/glbc.log 2>&1' </ins> volumes: - hostPath: path: /etc/gce.conf

<!-- Thanks for filing an issue! Before hitting the button, please answer these questions.--Is this a request for help? (If yes, you should use our troubleshooting guide and community support channels, see ): No What keywords did you search in Kubernetes issues before filing this one? (If you have found any duplicates, you should instead reply there.): Is this a BUG REPORT or FEATURE REQUEST? (choose one): BUG <!-- If this is a BUG REPORT, please: Fill in as much of the template below as you can. If you leave out information, we can't help you as well. If this is a FEATURE REQUEST, please: Describe in detail the feature/behavior/change you'd like to see. In both cases, be ready for followup questions, and please respond in a timely manner. If we can't reproduce a bug or think a feature already exists, we might close your issue. If we're wrong, PLEASE feel free to reopen it and explain why. --Kubernetes version (use ): 3 Environment: Cloud provider or hardware configuration: OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others: What happened: All of the salt scripts bundled in kubernetes (for example, ) start up the component in a broken way. They use which means the shell will intercept all signals. So these daemons are not gracefully shutdown when the pod is stopped. This causes serious problems since when apiserver is killed abruptly it doesn't seem to close watches correctly. This was observed in for example. What you expected to happen: Running is incorrect. It should run the binary directly. Otherwise, it needs to perform an so the binary correctly replaces the shell script process. How to reproduce it (as minimally and precisely as possible): Run kube-apiserver inside a container with and try to stop it. You can't - eventually docker will force kill it. Anything else we need to know:
It looks like we changed from running the binaries directly to using in (from a few years ago to enable us to capture logs. or may recall why we had to use a shell since they reviewed that PR.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close.


Question: # TODO: split this out into args when we no longer need to pipe stdout to a file #6428 - sh - -c <del> - '/glbc --verbose=true --apiserver-host=http://localhost:8080 --default-backend-service=kube-system/default-http-backend --sync-period=600s --running-in-cluster=false --use-real-cloud=true --config-file-path=/etc/gce.conf --healthz-port=8086 1>>/var/log/glbc.log 2>&1' </del> <ins> - 'exec /glbc --verbose=true --apiserver-host=http://localhost:8080 --default-backend-service=kube-system/default-http-backend --sync-period=600s --running-in-cluster=false --use-real-cloud=true --config-file-path=/etc/gce.conf --healthz-port=8086 1>>/var/log/glbc.log 2>&1' </ins> volumes: - hostPath: path: /etc/gce.conf

Prevent issues from auto-closing with an comment. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale


Question: } } <del> func (s *StatefulSetTester) waitForStatus(ss *apps.StatefulSet, expectedReplicas int32) { </del> <ins> // WaitForStatus waits for the ss.Status.Replicas to be equal to expectedReplicas func (s *StatefulSetTester) WaitForStatus(ss *apps.StatefulSet, expectedReplicas int32) { </ins> Logf("Waiting for statefulset status.replicas updated to %d", expectedReplicas) ns, name := ss.Namespace, ss.Name

https://k8s- do you know the statefulset owners?
cc
I did not see this happening until recently.
The controller misses that the first pod transitioned to unready and proceeds to create the second pod. The test uses a client to hit the api server and will scale up the statefulset as soon as the broken pod is observed but the controller uses caches so it may observe the scaled up statefulset quicker than the updated pod.
I will update the test to ensure that we wait until the unreadiness is observed on StatefulSet.Status.Replicas field prior to the scaling operation
SGTM


Question: // cloudPublicFeatureGates consists of cloud-specific feature keys. // To add a new feature, define a key for it at k8s.io/api/pkg/features and add it here. var cloudPublicFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ <del> LegacyNodeRoleBehavior:           {Default: false, PreRelease: featuregate.GA, LockToDefault: true}, ServiceNodeExclusion:             {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, </del> IPv6DualStack:                    {Default: true, PreRelease: featuregate.Beta}, ControllerManagerLeaderMigration: {Default: false, PreRelease: featuregate.Alpha}, }

What is missing? Based on SIG call it's probably "tests" but I am not sure. Could use some investigation and followup - how do we get it to GA? /sig network /kind cleanup
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
/assign
This one seems not a good first one, I'll take this to myself and start bothering people in Slack to check what's missing, mostly as I've seen this is also related to cloud providers :)
I think this one originally. According to the KEP in This should have went GA in v.19 and be removed in v1.20. Given it's still marked beta, at the very least we should mark it GA in v1.20 and remove it in v1.21. anything I'm missing here?
yeap. Honest question, changing a Feature Gate to GA process is only changing the related line in and also putting a comment that this goes to GA? If so, probably this one can also be a good first issue and we can open a future issue for the removal in v1.21 once v1.20 ships. I think this will also need to be changed in the in v1.21 right? If is ok with that, I would like then to mark this as a good first issue to turn the feature GA in v1.20 :)
Well, taking a look into how secure is to move to GA: In the KEP there are some considerations about removing the IsMasterNode function, which seems to be already dealt in commit E2E tests needs to be also checked about usage of IsMasterNode(), which I could not find anymore and also seems to be removed from the same commit above. About the same Kep, it's directly related to other Feature Gate: ` So what I think that needs to be done: Promote LegacyNodeRoleBehavior to GA and Default it to false as proposed in the Kep -I'll open another issue just for this According to the LegacyNodeRoleBehavior should be removed in v1.19 but according to the Kep in v1.21, so let's stick with v1.21? ServiceNodeExclusion - Should be moved to GA, already defaulted to true. This issue is going to deal ONLY with this.


Question: // cloudPublicFeatureGates consists of cloud-specific feature keys. // To add a new feature, define a key for it at k8s.io/api/pkg/features and add it here. var cloudPublicFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ <del> LegacyNodeRoleBehavior:           {Default: false, PreRelease: featuregate.GA, LockToDefault: true}, ServiceNodeExclusion:             {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, </del> IPv6DualStack:                    {Default: true, PreRelease: featuregate.Beta}, ControllerManagerLeaderMigration: {Default: false, PreRelease: featuregate.Alpha}, }

NodeDisruptionExclusion - Should be moved to GA, already defaulted to true. I'll open another issue for this I'm afraid of this having some sort of direct impact in cloud providers, do you want me to take this to the sig-cloudproviders meet? Otherwise can we move the above Feature Gates to GA, and schedule their removal for v1.21, wdyt?
Per deprecation lifecycle it seems that a feature needs 6 months or 2 releases (the longer one) in Beta before going to GA. So this actually seems to be the missing part before moving to GA. As the 3 FeatureGates are directed related, probably this issue should be: Move all of them to GA in v1.21 Default LegacyNodeRoleBehavior to false Sounds a good approach? I can keep this opened until we reach v1.21
If this issue has been triaged, please comment . If you aren't able to handle this issue, consider unassigning yourself and/or adding the label. I am a bot run by vllry.
/remove-triage unresolved
/retitle Move Feature gate ServiceNodeExclusion to GA and LockToDefault
/triage accepted as this FG is scheduled to be moved to GA in v1.21, do you want to tackle on this?
fixed in /close
Closing this issue. <detailsIn response to : fixed in /close </details


Question: fmt.Println("[reset] docker doesn't seem to be running, skipping the removal of running kubernetes containers") } <del> dirsToClean := []string{"/var/lib/kubelet", "/etc/cni/net.d", "/var/lib/dockershim"} </del> <ins> dirsToClean := []string{"/var/lib/kubelet", "/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes"} </ins> // Only clear etcd data when the etcd manifest is found. In case it is not found, we must assume that the user // provided external etcd endpoints. In that case, it is his own responsibility to reset etcd

BUG REPORT: (I think?) What happened: I ran the following steps on Ubuntu 16.04: Upon doing this, I receive: I've tried uninstalling , and a couple of times (even with ) and no matter what I do, it (kubeadm 1.7) doesn't generate a working . However, I run the following: and get: What you expected to happen: After initializing the master via , I expected to be able to use to install a network plugin; since it 's, I cannot do that. Environment: Kubernetes version (use ): 1.7 OS (e.g. from /etc/os-release): Ubuntu 16.04.2 LTS Kernel (e.g. ): Linux radium-control 4.4.0-83-generic -Ubuntu SMP Mon Jun 26 17:54:43 UTC 2017 x8664 x8664 x86_64 GNU/Linux
There are no sig labels on this issue. Please by:<br(1) mentioning a sig: <bre.g., for API Machinery<br(2) specifying the label manually: <bre.g., for sig/scalability<br<brNote: method (1) will trigger a notification to the team. You can find the team list and label list
/sig cluster-lifecycle
Unsure if this helps, but I had the same and realised I was using the old setup guide, copying into and setting . I cleared the environment variable and defaults back to using .
I'm also seeing this using kubeadm v1.7 - it's preventing nodes from joining the cluster
Same error for my installation. Try with v1.6.5 and 1.6.7 it works fine.
Same problem here. . ( seems okay) ( seems okay, too) (but fails)
do you have pointing to ?
After I set the to , now it gives me a timeout error. And now I want where the came from because there is no such statement in the I'm referencing.
From the output of the node join command:
Encountered same problem while playing with kubeadm. After and for a few times, kubelet will fail communicating with apiserver because (in kubelet logs). And also blocks for ever. After removing manually, all things come back. Maybe there are problems of cleaning certificates when running ?


Question: fmt.Println("[reset] docker doesn't seem to be running, skipping the removal of running kubernetes containers") } <del> dirsToClean := []string{"/var/lib/kubelet", "/etc/cni/net.d", "/var/lib/dockershim"} </del> <ins> dirsToClean := []string{"/var/lib/kubelet", "/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes"} </ins> // Only clear etcd data when the etcd manifest is found. In case it is not found, we must assume that the user // provided external etcd endpoints. In that case, it is his own responsibility to reset etcd

/area kubeadm
I am on kubeadm 1.8 and this problem still occurs. I manually checked . It was cleaned when I ran . Not sure what the actual problem is.
ATTENTION: "To start using your cluster, you need to run (as a regular user)" [root ~]# kubectl get nodes Unable to connect to the server: x509: certificate signed by unknown authority (possibly because of "crypto/rsa: verification error" while trying to verify candidate authority certificate "kubernetes") [root ~]# su - regularuser [regularuser ~] [regularuser ~] regularuser ~: [regular_user ~]$ kubectl get nodes NAME STATUS ROLES AGE VERSION NotReady master 6m v1.8.1 NotReady <none4m v1.8.1
Thanks. It works.
Please update docs with this workaround
If you kubeadm reset and then kubeadm init again, and if you ever ran the following as root, you need to run it again (as root) to get the new config: mkdir -p $ sudo cp -i $ sudo chown (id -g) $ Then you can still run as root.
if found that if you were ever going to run "sudo kubeadm reset" you'll need to remove your .kube dir to clear the cached dirs. After that you can follow oh, be sure to set KUBECONFIG to whatever you (re)name your config file to e.g. $
tanks freind ints true.
Config as the following lines, $kubectl get nodes works:
this is likely because you have a multi master setup and have generated the on each of the masters. Rather than copying them from the first master to the rest.
I found the solution in the kubernetes documentation while following the documentation dont forgot to create .kube directory using this command Because as you need this command it will move the .kube directory
For others that might have this issue, might want to try and move the folder to backup location if it exists and retry. Very possible to have a cached root version being used that is no longer valid, as you will run kubeadm as sudo.
My problem was I had custom certificates I had created during the KubeEdge Getting Started guide. Not messing around with ssl and kubeedge made it work.


Question: fmt.Println("[reset] docker doesn't seem to be running, skipping the removal of running kubernetes containers") } <del> dirsToClean := []string{"/var/lib/kubelet", "/etc/cni/net.d", "/var/lib/dockershim"} </del> <ins> dirsToClean := []string{"/var/lib/kubelet", "/etc/cni/net.d", "/var/lib/dockershim", "/var/run/kubernetes"} </ins> // Only clear etcd data when the etcd manifest is found. In case it is not found, we must assume that the user // provided external etcd endpoints. In that case, it is his own responsibility to reset etcd

export KUBECONFIG=$
that works for me , thanks a lot.
export kubectl get nodes is work from me
This worked!
After you have to remove $ folder and create new: mkdir -p $ sudo cp -i $ sudo chown (id -g) $
it worked! thx


Question: } func (udp *udpProxySocket) ProxyLoop(service proxy.ServicePortName, myInfo *serviceInfo, proxier *Proxier) { <del> activeClients := newClientCache() </del> var buffer [4096]byte // 4KiB should be enough for most whole-packets for { if info, exists := proxier.getServiceInfo(service); !exists || info != myInfo {

Depends on UDP clients have a timeout (so we don't leak connections forever), but the test does not cover this. To do this, we probably need to track active connections, and plumb that all the way down to the goroutines spawned by the ProxyLoop routines. This is probably something we want to do anyway to be able to monitor.


Question: Resources: resources, }, }, <ins> InitContainers: []v1.Container{ { Name:      "init-" + cname, Resources: resources, }, }, </ins> }, } }

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: When referencing init container's env value from its fields, kubelet complains What you expected to happen: Init container should be supported to get env values from its own fields, as to How to reproduce it (as minimally and precisely as possible): a pod with init container refering its fields value as env value kubectl create -f test-pod-1.yaml cannot be created pod events Anything else we need to know?: Environment: Kubernetes version (use ):Cloud provider or hardware configuration: OS (e.g: ):Kernel (e.g. ):Install tools: Others:
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
The err happens at this line , it seems that we should search for init containers as well