Question: <ins> /* Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cmd import ( "fmt" "io" "sort" "strings" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/kubernetes/pkg/kubectl/cmd/templates" cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" "k8s.io/kubernetes/pkg/printers" ) var ( apiresources_example = templates.Examples(` # Print the supported API Resources kubectl api-resources # Print the supported API Resources with more information kubectl api-resources -o wide # Print the supported namespaced resources kubectl api-resources --namespaced=true # Print the supported non-namespaced resources kubectl api-resources --namespaced=false # Print the supported API Resources with specific APIGroup kubectl api-resources --api-group=extensions`) ) // ApiResourcesOptions is the start of the data required to perform the operation.  As new fields are added, add them here instead of // referencing the cmd.Flags() type ApiResourcesOptions struct { out io.Writer Output     string APIGroup   string Namespaced bool NoHeaders  bool } // groupResource contains the APIGroup and APIResource type groupResource struct { APIGroup    string APIResource metav1.APIResource } func NewCmdApiResources(f cmdutil.Factory, out io.Writer) *cobra.Command { options := &ApiResourcesOptions{ out: out, } cmd := &cobra.Command{ Use:     "api-resources", Short:   "Print the supported API resources on the server", Long:    "Print the supported API resources on the server", Example: apiresources_example, Run: func(cmd *cobra.Command, args []string) { cmdutil.CheckErr(options.Complete(cmd)) cmdutil.CheckErr(options.Validate(cmd)) cmdutil.CheckErr(options.RunApiResources(cmd, f)) }, } cmdutil.AddOutputFlags(cmd) cmdutil.AddNoHeadersFlags(cmd) cmd.Flags().StringVar(&options.APIGroup, "api-group", "", "The API group to use when talking to the server.") cmd.Flags().BoolVar(&options.Namespaced, "namespaced", true, "Namespaced indicates if a resource is namespaced or not.") return cmd } func (o *ApiResourcesOptions) Complete(cmd *cobra.Command) error { o.Output = cmdutil.GetFlagString(cmd, "output") o.NoHeaders = cmdutil.GetFlagBool(cmd, "no-headers") return nil } func (o *ApiResourcesOptions) Validate(cmd *cobra.Command) error { validOutputTypes := sets.NewString("", "json", "yaml", "wide", "name", "custom-columns", "custom-columns-file", "go-template", "go-template-file", "jsonpath", "jsonpath-file") supportedOutputTypes := sets.NewString("", "wide") outputFormat := cmdutil.GetFlagString(cmd, "output") if !validOutputTypes.Has(outputFormat) { return fmt.Errorf("output must be one of '' or 'wide': %v", outputFormat) } if !supportedOutputTypes.Has(outputFormat) { return fmt.Errorf("--output %v is not available in kubectl api-resources", outputFormat) } return nil } func (o *ApiResourcesOptions) RunApiResources(cmd *cobra.Command, f cmdutil.Factory) error { w := printers.GetNewTabWriter(o.out) defer w.Flush() discoveryclient, err := f.DiscoveryClient() if err != nil { return err } // Always request fresh data from the server discoveryclient.Invalidate() lists, err := discoveryclient.ServerPreferredResources() if err != nil { return err } resources := []groupResource{} groupChanged := cmd.Flags().Changed("api-group") nsChanged := cmd.Flags().Changed("namespaced") for _, list := range lists { if len(list.APIResources) == 0 { continue } gv, err := schema.ParseGroupVersion(list.GroupVersion) if err != nil { continue } for _, resource := range list.APIResources { if len(resource.Verbs) == 0 { continue } // filter apiGroup if groupChanged && o.APIGroup != gv.Group { continue } // filter namespaced if nsChanged && o.Namespaced != resource.Namespaced { continue } resources = append(resources, groupResource{ APIGroup:    gv.Group, APIResource: resource, }) } } if o.NoHeaders == false { if err = printContextHeaders(w, o.Output); err != nil { return err } } sort.Stable(sortableGroupResource(resources)) for _, r := range resources { if o.Output == "wide" { if _, err := fmt.Fprintf(w, "%st%st%st%vt%st%vn", r.APIResource.Name, strings.Join(r.APIResource.ShortNames, ","), r.APIGroup, r.APIResource.Namespaced, r.APIResource.Kind, r.APIResource.Verbs); err != nil { return err } } else { if _, err := fmt.Fprintf(w, "%st%st%st%vt%sn", r.APIResource.Name, strings.Join(r.APIResource.ShortNames, ","), r.APIGroup, r.APIResource.Namespaced, r.APIResource.Kind); err != nil { return err } } } return nil } func printContextHeaders(out io.Writer, output string) error { columnNames := []string{"NAME", "SHORTNAMES", "APIGROUP", "NAMESPACED", "KIND"} if output == "wide" { columnNames = append(columnNames, "VERBS") } _, err := fmt.Fprintf(out, "%sn", strings.Join(columnNames, "t")) return err } type sortableGroupResource []groupResource func (s sortableGroupResource) Len() int      { return len(s) } func (s sortableGroupResource) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s sortableGroupResource) Less(i, j int) bool { ret := strings.Compare(s[i].APIGroup, s[j].APIGroup) if ret > 0 { return false } else if ret == 0 { return strings.Compare(s[i].APIResource.Name, s[j].APIResource.Name) < 0 } return true } </ins>

tl;dr: It's too difficult right now to go from "I want a user to be able to run " to "here is the RBAC rule to use for that" for any case where there isn't a specific example shown in the docs.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. Prevent issues from auto-closing with an comment. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
It seem that relative PR stop updated. If this issue still valuable, i will help fix it
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten /remove-lifecycle stale
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close


Question: // have a place they can schedule. testRootDir = makeTempDirOrDie("kubelet_integ_2.", "") glog.Infof("Using %s as root dir for kubelet #2", testRootDir) <del> kcfg = kubeletapp.SimpleKubelet(cl, &fakeDocker2, machineList[1], testRootDir, "", "127.0.0.1", 10251, api.NamespaceDefault, empty_dir.ProbeVolumePlugins(), nil, cadvisorInterface, "") </del> <ins> kcfg = kubeletapp.SimpleKubelet(cl, &fakeDocker2, machineList[1], testRootDir, secondManifestURL, "127.0.0.1", 10251, api.NamespaceDefault, empty_dir.ProbeVolumePlugins(), nil, cadvisorInterface, "") </ins> kcfg.PodStatusUpdateFrequency = 1 * time.Second kubeletapp.RunKubelet(kcfg) return apiServer.URL, configFilePath

Currently it assumes it can read ContainerManifest - it should try both
Why not hold on this until we have Kubelet-read-Pod in?
Also the HTTP source has the same problem.
It should read Pod. Changed the title.
Ideally, I'd like all Pods created directly via Kubelet to be to etcd and surfaced via the API. If updates are made via the API, I'd like those to propagate back to the node. We can discuss whether the config file should be updated or ignored. At some point, I'd like Kubelet to persistently cache pods received from the apiserver, perhaps by just writing them all out as config files.
is the issue to sync local pods back to apiserver.
this is somewhat related to other issues we talked about.
Detail: We want locally specified pods to be able to use YAML config files, but we also want to rip YAML out of the API server . The config-file behavior should be as similar to the kubectl experience. Kubelet could use parts of the kubectl library to translate YAML to JSON. Vision: Eventually, I'd like Kubelet to use the apiserver library and support exactly the same Pod API as the master, at which time I'd like kubectl be able to create pods directly on Kubelets, as well as reading from config files. I also expect to have a per-node controller . I'd like to be able to use the per-node controller to update pods started via config files (or direct kubectl-kubelet pods), overriding those original files. For availability and self-hosting, we'll eventually need local checkpointing , ideally just using serialized JSON for each object. If Kubelet used apiserver, this could be done using a custom registry implementation.
I think that is a prerequisite for it.
If noone picks it up, I'm going to start working on it tomorrow.
thanks, wojtek-t
fixed it for http channel


Question: kind: Pod metadata: name: b <ins> namespace: nsb </ins> labels: prune-group: "true" spec:

<!-- Please only use this template for submitting enhancement requests --What would you like to be : Actually, the parameter depends to all namespaces in the cluster. Filtering is only possible with labels. When running the should only depend on the objects defined in . Why is this needed: A typical use-case is having one repository and namespace per project. When working with declarative objects it should be possible to remove objects and get them pruned like having them described in the respository. This would also help when renaming objects within the repository - which actually needs a manual on the host for the old objects. This even could be a better solution for the problem described in
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig api-machinery
cc
/assign
GitHub didn't allow me to assign the following users: Liujingfang1. Note that only and repo collaborators can be assigned and that issues/PRs can only have 10 assignees at the same time. For more information please see <detailsIn response to : /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
We have been aware of the issues in prune and we want to improve it. Recently we opened a for improving pruning in . With the proposed solution, the problem from current will be fixed.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or .


Question: kind: Pod metadata: name: b <ins> namespace: nsb </ins> labels: prune-group: "true" spec:

/lifecycle stale
/remove-lifecycle stale
I just deleted a bunch of things in another team's namespace (:blush:). I'd would love to have this feature. I had been using dry-run in another context, but then forgot to check it again after switching contexts.
Is there any workaround for this? I want to prune resources within a single namespace only.
We're still working on improving prune, the current mechanism is broken enough that we don't recommend people us it.
Also look like it's been fixed recently.
Can someone confirm if this is implemented? I tried using this feature with kubectl 1.18.2, which has , yet as with previous versions kubectl it returned an error about pruning a nonNamespaced object of kind Namespace.
I assume the problem is that "Namespace" is a non-namespaced resource, and therefore is not filtered by namespace, so gets deleted by "prune". As a workaround it's possible to manually pass a (whitelist of resources allowed to be deleted), and not include "Namespace" in it: The list of resources is taken from .
Would wrapping this loop with achieve the same result?


Question: return nil, cleanupAction, fmt.Errorf("volume subpaths are disabled") } <del> if !utilfeature.DefaultFeatureGate.Enabled(features.VolumeSubpathEnvExpansion) { return nil, cleanupAction, fmt.Errorf("volume subpath expansion is disabled") } </del> subPath, err = kubecontainer.ExpandContainerVolumeMounts(mount, expandEnvs) if err != nil {

/kind cleanup /sig storage /sig node Description: Feature gate VolumeSubpathEnvExpansion was GA in 1.17 and is marked for removal in 1.19 Related Issues: GA Release:
/assign
/cc


Question: func CheckAffinity(execPod *v1.Pod, serviceIP string, servicePort int, shouldHold bool) bool { serviceIPPort := net.JoinHostPort(serviceIP, strconv.Itoa(servicePort)) cmd := fmt.Sprintf(`curl -q -s --connect-timeout 2 http://%s/`, serviceIPPort) <del> timeout := TestTimeout </del> <ins> timeout := AffinityTimeout </ins> if execPod == nil { timeout = LoadBalancerPollTimeout }

Which jobs are failing: pull-kubernetes-e2e-gce Which test(s) are failing: [sig-network] Services should have session affinity work for NodePort service [LinuxOnly] [sig-network] Services should be able to switch session affinity for service with type clusterIP [LinuxOnly] [sig-network] Services should be able to switch session affinity for NodePort service [LinuxOnly] [sig-network] Services should have session affinity work for service with type clusterIP [LinuxOnly] Since when has it been failing: At least since 9/30 (). But that's almost at the end of what's currently visible on the testgrid, so maybe longer Testgrid link: Not sure what I'm supposed to put here. ? Reason for failure: The test case connects to a service every 2 seconds for 1 minute and checks if it hit the same endpoint at least 15 times in a row. But the individual checks are taking unexpectedly long and so sometimes the test doesn't even manage to run 15 times total in 1 minute.


Question: func CheckAffinity(execPod *v1.Pod, serviceIP string, servicePort int, shouldHold bool) bool { serviceIPPort := net.JoinHostPort(serviceIP, strconv.Itoa(servicePort)) cmd := fmt.Sprintf(`curl -q -s --connect-timeout 2 http://%s/`, serviceIPPort) <del> timeout := TestTimeout </del> <ins> timeout := AffinityTimeout </ins> if execPod == nil { timeout = LoadBalancerPollTimeout }

eg: Oct 8 13:32:01.402: INFO: Running ' --server=https://34.82.143.14 -- exec --namespace=services-1060 execpod-affinityklpfn -- /bin/sh -x -c curl -q -s --connect-timeout 2 http://10.40.0.5:/' Oct 8 13:32:07.627: INFO: stderr: "+ curl -q -s --connect-timeout 2 http://10.40.0.5:/n" Oct 8 13:32:07.627: INFO: stdout: "affinity-nodeport-2z9hv" Oct 8 13:32:07.627: INFO: Received response from host: affinity-nodeport-2z9hv Oct 8 13:32:09.402: INFO: Running ' --server=https://34.82.143.14 -- exec --namespace=services-1060 execpod-affinityklpfn -- /bin/sh -x -c curl -q -s --connect-timeout 2 http://10.40.0.5:/' Oct 8 13:32:16.027: INFO: stderr: "+ curl -q -s --connect-timeout 2 http://10.40.0.5:/n" Oct 8 13:32:16.027: INFO: stdout: "affinity-nodeport-2z9hv" Oct 8 13:32:16.027: INFO: Received response from host: affinity-nodeport-2z9hv Note the gap between the "Running" line and the "stderr" line. It's not clear if the slowness is in connecting to the execpod or in connecting from the execpod to the affinity service. It is also not particularly consistent between attempts. (Here it was 6s, 1s, 2s, 3s, 5s, 7s, 7s, 3s, 4s, 6s, 7s, 8s.) Anything else we need to know: It looks the checks are also slow in successful runs, it's just that they're not quite as slow, so they pass.


Question: func CheckAffinity(execPod *v1.Pod, serviceIP string, servicePort int, shouldHold bool) bool { serviceIPPort := net.JoinHostPort(serviceIP, strconv.Itoa(servicePort)) cmd := fmt.Sprintf(`curl -q -s --connect-timeout 2 http://%s/`, serviceIPPort) <del> timeout := TestTimeout </del> <ins> timeout := AffinityTimeout </ins> if execPod == nil { timeout = LoadBalancerPollTimeout }

I'm not sure if the problem is "something is making the cluster slow and so this totally valid test is taking too long and failing" or "e2e-gce has grown over time and now cluster load has reached the point where this formerly-passing test now has unreasonable performance expectations". /sig network


Question: return json.Marshal(doc) } <del> // CreateMergePatch creates a merge patch as specified in http://tools.ietf.org/html/draft-ietf-appsawg-json-merge-patch-07 // // 'a' is original, 'b' is the modified document. Both are to be given as json encoded content. // The function will return a mergeable json document with differences from a to b. // // An error will be returned if any of the two documents are invalid. func CreateMergePatch(a, b []byte) ([]byte, error) { aI := map[string]interface{}{} bI := map[string]interface{}{} err := json.Unmarshal(a, &aI) </del> <ins> // resemblesJSONArray indicates whether the byte-slice "appears" to be // a JSON array or not. // False-positives are possible, as this function does not check the internal // structure of the array. It only checks that the outer syntax is present and // correct. func resemblesJSONArray(input []byte) bool { input = bytes.TrimSpace(input) hasPrefix := bytes.HasPrefix(input, []byte("[")) hasSuffix := bytes.HasSuffix(input, []byte("]")) return hasPrefix && hasSuffix } // CreateMergePatch will return a merge patch document capable of converting // the original document(s) to the modified document(s). // The parameters can be bytes of either two JSON Documents, or two arrays of // JSON documents. // The merge patch returned follows the specification defined at http://tools.ietf.org/html/draft-ietf-appsawg-json-merge-patch-07 func CreateMergePatch(originalJSON, modifiedJSON []byte) ([]byte, error) { originalResemblesArray := resemblesJSONArray(originalJSON) modifiedResemblesArray := resemblesJSONArray(modifiedJSON) // Do both byte-slices seem like JSON arrays? if originalResemblesArray && modifiedResemblesArray { return createArrayMergePatch(originalJSON, modifiedJSON) } // Are both byte-slices are not arrays? Then they are likely JSON objects... if !originalResemblesArray && !modifiedResemblesArray { return createObjectMergePatch(originalJSON, modifiedJSON) } // None of the above? Then return an error because of mismatched types. return nil, errBadMergeTypes } // createObjectMergePatch will return a merge-patch document capable of // converting the original document to the modified document. func createObjectMergePatch(originalJSON, modifiedJSON []byte) ([]byte, error) { originalDoc := map[string]interface{}{} modifiedDoc := map[string]interface{}{} err := json.Unmarshal(originalJSON, &originalDoc) </ins> if err != nil { return nil, errBadJSONDoc } <del> err = json.Unmarshal(b, &bI) </del> <ins> err = json.Unmarshal(modifiedJSON, &modifiedDoc) </ins> if err != nil { return nil, errBadJSONDoc } <del> dest, err := getDiff(aI, bI) </del> <ins> dest, err := getDiff(originalDoc, modifiedDoc) </ins> if err != nil { return nil, err } return json.Marshal(dest) } <ins> // createArrayMergePatch will return an array of merge-patch documents capable // of converting the original document to the modified document for each // pair of JSON documents provided in the arrays. // Arrays of mismatched sizes will result in an error. func createArrayMergePatch(originalJSON, modifiedJSON []byte) ([]byte, error) { originalDocs := []json.RawMessage{} modifiedDocs := []json.RawMessage{} err := json.Unmarshal(originalJSON, &originalDocs) if err != nil { return nil, errBadJSONDoc } err = json.Unmarshal(modifiedJSON, &modifiedDocs) if err != nil { return nil, errBadJSONDoc } total := len(originalDocs) if len(modifiedDocs) != total { return nil, errBadJSONDoc } result := []json.RawMessage{} for i := 0; i < len(originalDocs); i++ { original := originalDocs[i] modified := modifiedDocs[i] patch, err := createObjectMergePatch(original, modified) if err != nil { return nil, err } result = append(result, json.RawMessage(patch)) } return json.Marshal(result) } </ins> // Returns true if the array matches (must be json types). // As is idiomatic for go, an empty array is not the same as a nil array. func matchesArray(a, b []interface{}) bool {

/kind bug What happened: Wrote a that returned a faulty json patch which specified an our of range index (e.G. when there is only one container). This resulted in the apiserver crashing. Stacktrace: What you expected to happen: No crash /sig-api-machinery
/sig api-machinery
This maybe a bug in . I will follow this question.
This bug had been fixed in PR: you can checkout it.
Thanks for your comment but I don't think the linked PR fixed the issue, as it got merged almost a year ago and I am running Kube 1.10.1 which does contain that patch but still panics. Just from quickly looking at the code I the linked PR I believe there are two things wrong: The check if the index is valid only happens when a negative index was given should be
Once the PR of is merged I will update kubernetes.
/kind bug


Question: } type Zones interface { <del> // List returns the managed Zones, or an error if the list operation failed. </del> <ins> // List returns the (possibly empty) list of managed Zones, or an error if the list operation failed. </ins> List() ([]Zone, error) // Add creates and returns a new managed zone, or an error if the operation failed Add(Zone) (Zone, error)

/kind bug What happened: walteraa [6:55 AM] Hi, The controller manager of the federation using Google cloud DNS is crashing, the log showed is that: the hyperkube command is that: I have created this in the Google DNS Someone knows why the DNS zone query is getting the 404 error?? :disappointed: What you expected to happen: It should have created the missing domain. How to reproduce it (as minimally and precisely as possible): As per above. Anything else we need to know?: Environment: Kubernetes version (use ): 1.7.4 Cloud provider or hardware configuration**: OpenStack (irrelevant) OS (e.g. from /etc/os-release): irrelevant Kernel (e.g. ): irrelevant Install tools: irrelevant Others: irrelevant
The bug is related to this: The above code assumes that the provider code returns no error but an empty list if the domain does not exist, but instead the provider returns a 404. Need to fix the provider and/or the above code.
/sig federation
I am unlikely to have time to work on this soon, so if anyone else wants to, go right ahead.
/assign
An additional information: I have created a which is based on the credential JSON file created through instruction. After that, I have configured the Federation controller manager to mount this secret as a volume in the container and set the environment variable to the JSON file inside this volume. Before doing that, I was getting an authentication error so the google DNS provider couldn't be instantiated, and after doing the instructions that I mentioned before, this authentication error was solved but now I'm getting 404 error when Google DNS provider tries access the specified managed zone. The file that I have generated follow this structure:


Question: "net/http/httptest" "os" "reflect" <ins> "strconv" </ins> "strings" "sync" "testing"

Which jobs are failing: Which test(s) are failing: Since when has it been failing: Testgrid link: Reason for failure: Anything else we need to know: /sig api-machinery /assign
/cc
want to take a look at the audit event issue first? I can dig in after that if issues remain.
yes, I'm looking at the audit event issue now
actually the failure message suggests that the missing audit log is caused by unexpected admission invocation: I think it's unrelated to auditing Note that there are two tests named "no reinvocation of webhook B when in-tree or prior webhook mutations" :( and from the audit log (), it also suggests that the test is hitting a wrong endpoint
Thanks I'll dig in.


Question: <del> // +build cgo,linux </del> <ins> // +build linux </ins> /* Copyright 2015 The Kubernetes Authors.

In cluster/saltbase/salt/kube-proxy/kube-proxy.manifest, we have this: But kube-proxy also has a flag. Is there a reason to use the approach instead of using the flag? cc as you wrote the code, in
/cc who's working on moving kube-proxy to a DaemonSet
In general, oomscoreadj is not expected to be directly updated by pods and kubelet is expected to manage it. If kube-proxy runs in the Guaranteed QoS class, then it's OOM score adj will be set to a low number. Until that happens, kube-proxy process may want to be shielded from OOM kills to an extent (this matters only when kube-proxy's memory usage memory request) Meanwhile, it does not make sense to have two means to configure the same thing. I'll let decide which option he prefers and sunset the other option.
Thanks for clarifying. At this point, I prefer to use the built-in flag instead of relying on composing bash commands. Sent for this.


Question: }, { "ImportPath": "github.com/evanphx/json-patch", <del> "Rev": "ed7cfbae1fffc071f71e068df27bf4f0521402d8" </del> <ins> "Rev": "94e38aa1586e8a6c8a75770bddf5ff84c48a106b" </ins> }, { "ImportPath": "github.com/ghodss/yaml",


Question: if cleanupInfo != nil { // we don't perform the clean up just yet at that could destroy information // needed for the container to start (e.g. Windows credentials stored in <del> // registry keys); instead, we'll clean up after the container successfully // starts or gets removed </del> <ins> // registry keys); instead, we'll clean up when the container gets removed </ins> ds.containerCleanupInfos[containerID] = cleanupInfo } return &runtimeapi.CreateContainerResponse{ContainerId: containerID}, nil } <del> // the creation failed, let's clean up right away </del> <ins> // the creation failed, let's clean up right away - we ignore any errors though, // this is best effort </ins> ds.performPlatformSpecificContainerCleanupAndLogErrors(containerName, cleanupInfo) return nil, createErr

What would you like to be : Periodically remove cleanupInfos for old containers from dockershim's map. One way of doing that would be to just forget about s older than say an hour, as they should always be short-lived. Why is this needed: introduced a new in-memory map for the dockershim, that could potentially (in pathological cases) cause memory leaks - for containers that use GMSA cred specs, get created successfully, but then never get started nor removed. This might be okay for the alpha release of the GMSA feature, but should be addressed in the beta. See
/assign wk8
/sig windows
/milestone v1.15
got moved out. let's move this issue with it as well /milestone v1.16
Hello! I'm bug triage lead for the 1.16 release cycle and considering this issue is tagged for 1.16, but not updated for a long time, I'd like to check its status. The code freeze is starting on August 29th (about 1.5 weeks from now), which means that there should be a PR ready (and merged) until then. Do we still target this issue to be fixed for 1.16?
: yup we'd love to have this in 1.16, still waiting for kind reviewers on :) ...


Question: // 1. Add a volume to DSW and wait until it's mounted volumeSpec := &volume.Spec{Volume: &pod.Spec.Volumes[0]} <ins> // copy before reconciler runs to avoid data race. volumeSpecCopy := &volume.Spec{Volume: &pod.Spec.Volumes[0]} </ins> podName := util.GetUniquePodName(pod) generatedVolumeName, err := dsw.AddPodToVolume( podName, pod, volumeSpec, volumeSpec.Name(), "" /* volumeGidValue */)

<!-- Please only use this template for submitting reports about flaky tests or jobs (pass or fail with no underlying change in code) in Kubernetes CI --pull-kubernetes-unit : (PR with no go code changes, should not have failed any unit tests) links to appreciated links to specific failures in spyglass appreciated <!-- Please see the deflaking doc () for more guidance! --/sig storage
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assigne
/assign
/assign tried to fix it in


Question: func (f *Framework) BeforeEach() { f.beforeEachStarted = true <del> // The fact that we need this feels like a bug in ginkgo. // https://github.com/onsi/ginkgo/v2/issues/222 f.cleanupHandle = AddCleanupAction(f.AfterEach) </del> if f.ClientSet == nil { ginkgo.By("Creating a kubernetes client") config, err := LoadConfig()

Here is a list of TODOs that could be fixed for Ginkgo migration from v1 to v2. [x] Lost the validation on the SIGDescribe: [x] Ginkgo V2 currently cannot get the number of specs that will run in , this might be fixed in the Ginkgo's minor release of V2 in the future. [x] <delGinkgo dry-run mode has dropped the test spec in V2 (), consider to switch back to use to generate test spec when the feature is ready in . </del--- This has been fixed in Ginkgo 2.1.4 [x] <delMigrate all the custom reporter in , should be lightweight and not sure where those custom reporters are used. </delFixed here: [x] Replace all the camel case flags to kebab case flags, e.g. to . [x] <delRemove the legacy e2e test binary's parameter and in 1.28, those parameters were used to manage report, this is no longer needed for Ginkgo v2. </delno longer needed. [x] upgrade vendor when is migrated to ginkgo v2, this will help to remove the dependency of - [x] <delrevisit to explore the possibility to cover the test in Ginkgo V2. </delfixed by [x] remove the workaround introduced in Ginkgo V1 ( is not called after hitting ^c): partially addressed by: , some cleanup needed for each sig before removing the hook entirely. I think has the pr to fix this entirely, but I cannot get the link for that, mark it as resolved. : [x] New work item (?): capture only a single output text output stream () in JUnit (this issue was resolved along with another one in ) [ ] Due to the junit format change, some tool which use to grab "value" directly cannot get the failure reason as what it does in v1. [x] Test failures are missing information since GinkgoV2 migration [ ] is clustering stack traces, not failure text () Address those leftovers if there is any update from Ginkgo in the future. see the relevant link on the each of the issues.


Question: func (f *Framework) BeforeEach() { f.beforeEachStarted = true <del> // The fact that we need this feels like a bug in ginkgo. // https://github.com/onsi/ginkgo/v2/issues/222 f.cleanupHandle = AddCleanupAction(f.AfterEach) </del> if f.ClientSet == nil { ginkgo.By("Creating a kubernetes client") config, err := LoadConfig()

No response_ <details</details<detailsall </details<details</details<details</details<details</details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig testing
/cc
/cc
hello! I would like to work on this issues
I think we cannot do anything now except the last two, and those two should be based on top of the:
I would like to work on task 5 of this issue
no problem, pls wait for an while until that PR got merged, then you can create a pr to fix those deprecated parameters.
Regarding : searching for points out the places where the workaround is used and needs to be replaced once is solved.
I just noticed this comment in
seems like is not used in k/k, but in other sigs.
hmm... I just grepped the test code, will push a pr to fix this.
/open
/re-open
okay, this is not resolved actually, but I have no way to re-open it.
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
~ it seems that some more jobs needs to be adapted to not use parallel~ [repo][0]=kubernetes/test-infra~ ~ACTION REQUIRED: Ginkgo V2 doesn't accept go test's flags to parallelize Ginkgo specs, please switch to use or instead.~ Sorry for the noise, the flag is passed to kubetest, and it correctly translate it to , it is not the issue here
np, thanks for the update!
New work item (?): capture only a single output text output stream () in JUnit.
updated the issue, haven't got a chance to take a close look at the item though.
This should be resolved by "open stdout" now should only appear if a test intentionally records SpecReports.
mark it as resolved.
is this complete now?


Question: func (f *Framework) BeforeEach() { f.beforeEachStarted = true <del> // The fact that we need this feels like a bug in ginkgo. // https://github.com/onsi/ginkgo/v2/issues/222 f.cleanupHandle = AddCleanupAction(f.AfterEach) </del> if f.ClientSet == nil { ginkgo.By("Creating a kubernetes client") config, err := LoadConfig()

, Only two of them left, I need to follow this one, here is the issue for this one opened in the Ginkgo: I think this should be fixed in the third party tools instead of Kubernetes, if we agree with that, we can mark it as completed as well. For the first one, I will continue to follow until there is a update from Ginkgo's side, for the later one, there is no consensus yet IIUC, we can open a separate issue for that, I am gonna to close this one since there is no much thing we can do now. /close
We agreed to fix the tool, but had no issue open for it. I just created one:
Another issue for


Question: for n, pods := range nodeToDaemonPods { if !isNodeRunning[n] { for _, pod := range pods { <del> results = append(results, pod.Name) </del> <ins> if len(pod.Spec.NodeName) == 0 { results = append(results, pod.Name) } </ins> } } }

What happened: Occasionally we get a Daemonset stuck with less Current replicas than Desired, causing a Daemonset sync tends to fix it, e.g (add a new node). This happens to all our Daemonsets. I'm using fluentd as an example. There are no Daemonsets pods in pending nor are there any errors when describing the Daemonset What you expected to happen: The Daemonset Pod to be scheduled How to reproduce it (as minimally and precisely as possible): Reproducing this state is very tricky, I've had the most success by terminating nodes in our ASG causing a new one to come up. However I think I've identified why this is happening. The issue appears to be related to daemonset Pod deletion, a Pod gets deleted causing to trigger, this lowers the expectations by as expected. The delete pod event then triggers a sync of the daemonset, in which it finds which involves fetching pods from the informer cache This is somehow returning the deleted pod. You can see in the logs here thinks that it needs to delete the pod that was deleted. It is also setting the expectation back to after the delete pod event previously lowered it by 1. At this point you're stuck as the function can't get past this line As the expectation can not be satisfied as syncNodes set it back to and there's nothing it to get it back to . So I believe this is what's causing us to get less replicas than expected, the function is stuck due to the expectation so it can't perform a full sync to schedule the new pods. There's also not much to trigger a new sync once the expectation has expired until a new node comes up or one goes down. Environment: Kubernetes version (use ):Cloud provider or hardware configuration: AWS OS (e.g: ):Kernel (e.g. ):
/sig apps
I investigated this as much as I could without getting some external input, it feels like it's some kind of race condition in the informer cache. If anyone has ideas of if places to dig deeper that would be handy.
Reiterating the mentions to trigger a notification: <detailsIn response to : I investigated this as much as I could without getting some external input, it feels like it's some kind of race condition in the informer cache. If anyone has ideas of if places to dig deeper that would be handy.


Question: for n, pods := range nodeToDaemonPods { if !isNodeRunning[n] { for _, pod := range pods { <del> results = append(results, pod.Name) </del> <ins> if len(pod.Spec.NodeName) == 0 { results = append(results, pod.Name) } </ins> } } }

Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
This problem is definitely related to node termination, it's trying to delete the already deleted pod from the node that's been terminated. The problem seems to go away if is set, I think although this kind of race condition may have always been possible, it's being frequently caused via Which is finding the non-existent pod in the pod cache and seeing that it is assigned to a non-running node, resulting in a request to delete the non-existent pod. cc
Reiterating the mentions to trigger a notification: <detailsIn response to : This problem is definitely related to node termination, it's trying to delete the already deleted pod from the node that's been terminated. Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign could you please take a look at this? It seems that it is related to the code you in .
Thanks for reporting the bug. Bug was introduced in which deleting pods without nodes. There is race condition between removing pod from cache and listing pods for DaemonSet. Bug occurs during following scenario: is deleted. for node is deleted by PodGCController. handler is executed by informer () set is to queue by delete handler () is processed and pods are taken from cache before deleted pod is removed. for deleted pod is not found so we add deleted pod to podsToDelete list (). are set to del:1() succeeds as pod is not found(). are never met so we need to wait till expectations expire. As fix I will send a change to so only pods that can't be deleted by PodGCController are deleted by DaemonSet controller.


Question: <ins> // +build linux /* Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package e2enode import ( "time" "golang.org/x/sys/unix" "github.com/onsi/ginkgo" "github.com/onsi/gomega" "k8s.io/kubernetes/test/e2e/framework" ) const contentionLockFile = "/var/run/kubelet.lock" var _ = SIGDescribe("Lock contention [Slow] [Disruptive] [NodeFeature:LockContention]", func() { ginkgo.It("Kubelet should stop when the test acquires the lock on lock file and restart once the lock is released", func() { ginkgo.By("perform kubelet health check to check if kubelet is healthy and running.") // Precautionary check that kubelet is healthy before running the test. gomega.Expect(kubeletHealthCheck(kubeletHealthCheckURL)).To(gomega.BeTrue()) ginkgo.By("acquiring the lock on lock file i.e /var/run/kubelet.lock") // Open the file with the intention to acquire the lock, this would imitate the behaviour // of the another kubelet(self-hosted) trying to start. When this lock contention happens // it is expected that the running kubelet must terminate and wait until the lock on the // lock file is released. // Kubelet uses the same approach to acquire the lock on lock file as shown here: // https://github.com/kubernetes/kubernetes/blob/master/cmd/kubelet/app/server.go#L530-#L546 // and the function definition of Acquire is here: // https://github.com/kubernetes/kubernetes/blob/master/pkg/util/flock/flock_unix.go#L25 fd, err := unix.Open(contentionLockFile, unix.O_CREAT|unix.O_RDWR|unix.O_CLOEXEC, 0600) framework.ExpectNoError(err) // Defer the lock release in case test fails and we don't reach the step of the release // lock. This ensures that we release the lock for sure. defer func() { err = unix.Flock(fd, unix.LOCK_UN) framework.ExpectNoError(err) }() // Acquire lock. err = unix.Flock(fd, unix.LOCK_EX) framework.ExpectNoError(err) ginkgo.By("verifying the kubelet is not healthy as there was a lock contention.") // Once the lock is acquired, check if the kubelet is in healthy state or not. // It should not be. gomega.Eventually(func() bool { return kubeletHealthCheck(kubeletHealthCheckURL) }, 10*time.Second, time.Second).Should(gomega.BeFalse()) ginkgo.By("releasing the lock on lock file i.e /var/run/kubelet.lock") // Release the lock. err = unix.Flock(fd, unix.LOCK_UN) framework.ExpectNoError(err) ginkgo.By("verifying the kubelet is healthy again after the lock was released.") // Releasing the lock triggers kubelet to re-acquire the lock and restart. // Hence the kubelet should report healthy state. gomega.Eventually(func() bool { return kubeletHealthCheck(kubeletHealthCheckURL) }, 10*time.Second, time.Second).Should(gomega.BeTrue()) }) }) </ins>

<!-- Please only use this template for submitting enhancement requests --What would you like to be : Node E2E tests for Kubelet lock contention flags: --exit-on-lock-contention --lock-file Why is this needed: These flags are being graduated to KubeletConfiguration on , we should add E2E tests for these flags before proceeding.
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig node /wg component-standard
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle rotten
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-contributor-experience at . /close
Closing this issue. <detailsIn response to : Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", <ins> "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", </ins> "//staging/src/k8s.io/client-go/dynamic:go_default_library", "//test/integration/framework:go_default_library", "//vendor/github.com/coreos/etcd/clientv3:go_default_library",

What happened: Set up a HA cluster with 1.9 and 1.10 API servers (occurs on any apiserver skew boundary where a cohabitating resource's storage version changed groups) Observed errors when modifying workload objects: when modified via the 1.10 server, saw this error in the 1.9 server: when modified via the 1.9 server, saw this error in the 1.10 server: When this error was encountered, all watchers for the resource in question would be dropped and have to re-establish, and the internal watchers feeding the watch cache would drop and re-establish What you expected to happen: Watch events from all supported storage versions are processed correctly How to reproduce it (as minimally and precisely as possible): Start a server from master (for inspectability, store in json). With Create a deployment (e.g. kube-dns in kube-system) Establish a watch via extensions (kubectl get deployments.extensions --all-namespaces -w) Establish a watch via apps (kubectl get --all-namespaces -w) Tail the kube-apiserver log Change the apps/v1 deployment in etcd to extensions/v1beta1: Observer client watches drop and server log errors: Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others: /sig api-machinery /cc /priority critical-urgent
error is triggered by this returning false: constructed here: the info about expected cohabitating groups is not included in the list of acceptable inputs


Question: if err != nil { t.Fatal(err) } ss1Rev1.Namespace = ss1.Namespace ss1Rev2, err := NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 2, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev2.Namespace = ss1.Namespace ss1Rev3, err := NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 3, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } <del> ss1Rev3.Namespace = ss1.Namespace </del> <ins> ss1Rev3Time2, err := NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 3, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev3Time2.Namespace = ss1.Namespace ss1Rev3Time2.CreationTimestamp = metav1.Time{Time: ss1Rev3.CreationTimestamp.Add(time.Second)} ss1Rev3Time2Name2, err := NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 3, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev3Time2Name2.Namespace = ss1.Namespace ss1Rev3Time2Name2.CreationTimestamp = metav1.Time{Time: ss1Rev3.CreationTimestamp.Add(time.Second)} </ins> tests := []testcase{ {

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: Statefulsets are sometimes continually rolling pods and I don't know why statefulset created: storageclass referenced: kubectl describe statefulset statefulset-test: As you can see from the above describe, the pod in the statefulset has been restarted a lot of times. Observing the created pod, 1) The pod comes up ready 2) immediately transitions to terminating 3) Is terminated 4) A new pod is created 5) Go back to (1) (repeats forever) Despite no updates to the statefulset spec ocurring. Making a change to the statefulset template (for example a trivial change adding a nonsense annotation) stops the rolling and fixes the issue When this statefulset was created, there was a mutating webhook enabled. After the observed pod rolling. I disabled this by making its configuration invalid and making it non required, and the rolling persisted. The status of the statefulset: : You can see that there is a discrepency between currentRevision and updateRevision. currentRevision: statefulset-test- updateRevision: statefulset-test- A short while later the status is changed to contain currentRevision: statefulset-test- updateRevision: statefulset-test- This change happened at Sun Apr 1 19:48:48 2018 Presumably after the rollout of statefulset-test- is complete.


Question: if err != nil { t.Fatal(err) } ss1Rev1.Namespace = ss1.Namespace ss1Rev2, err := NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 2, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev2.Namespace = ss1.Namespace ss1Rev3, err := NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 3, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } <del> ss1Rev3.Namespace = ss1.Namespace </del> <ins> ss1Rev3Time2, err := NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 3, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev3Time2.Namespace = ss1.Namespace ss1Rev3Time2.CreationTimestamp = metav1.Time{Time: ss1Rev3.CreationTimestamp.Add(time.Second)} ss1Rev3Time2Name2, err := NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 3, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev3Time2Name2.Namespace = ss1.Namespace ss1Rev3Time2Name2.CreationTimestamp = metav1.Time{Time: ss1Rev3.CreationTimestamp.Add(time.Second)} </ins> tests := []testcase{ {

Here are all the controllerrevisions from above: kubectl get controllerrevision statefulset-test- -o yaml kubectl get controllerrevision statefulset-test- -o yaml: kubectl get controllerrevision statefulset-test- -o yaml: The differences between the above controllerrevisions are insignificant (the hash, the creation timestamp, the name, the resource version, the uid, the self link) Interestingly all these controllerrevisions were created very close together in time shortly after I created the statefulset The current controllerrevisions are After watching the state change for a while, it actually seems like the statefulset is just cycling through the updateRevision being one of the above 10 controllerrevisions Leaders during this time From kube-controller-manager logs during this time, here is one excerpt: The error sync in there is suspicous, and as you can see the controllerrevisions are changing in the logs I have the full logs from a 5 minute window on either side of the above if necessary. The reason I'm inclined to think this is a kubernetes bug is that any update to the statefulset template fixes the problem. How to reproduce it (as minimally and precisely as possible): Not really sure. Worth just trying to create the above statefulset (requires creating the I tried deleting and recreating the above statefulset and it didn't happen, so something flaky is going on Anything else we need to know?: kubernetes 1.9.3 (client and server) cloud provider: aws
/sig apps
I realize this will probably need some further debugging, happy to get some direction
Consistent repro: Create a statefulset that creates a controllerrevision whose hash is between max int32 and max uint32 (this shouldn't be that hard to do, possibly a 50/50 chance of this). Manually create a controllerrevision identical to the first (same revision), incrementing both the name and hash by one. See that the statefulset controller will continually switch which controller revision should be the target of the statefulset.
Due to the issue fixed here: , under some circumstances controllerrevisions whose revision is the same were being considered equal.


Question: if err != nil { t.Fatal(err) } ss1Rev1.Namespace = ss1.Namespace ss1Rev2, err := NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 2, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev2.Namespace = ss1.Namespace ss1Rev3, err := NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 3, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } <del> ss1Rev3.Namespace = ss1.Namespace </del> <ins> ss1Rev3Time2, err := NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 3, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev3Time2.Namespace = ss1.Namespace ss1Rev3Time2.CreationTimestamp = metav1.Time{Time: ss1Rev3.CreationTimestamp.Add(time.Second)} ss1Rev3Time2Name2, err := NewControllerRevision(ss1, parentKind, ss1.Spec.Template.Labels, rawTemplate(&ss1.Spec.Template), 3, ss1.Status.CollisionCount) if err != nil { t.Fatal(err) } ss1Rev3Time2Name2.Namespace = ss1.Namespace ss1Rev3Time2Name2.CreationTimestamp = metav1.Time{Time: ss1Rev3.CreationTimestamp.Add(time.Second)} </ins> tests := []testcase{ {

This ends up causing this bug to manifest as follows: ListRevisions finds many controllerrevisions with the same revision (unclear how this state happens, but it can happen) The sort here randomly sorts them Due to the linked bug , FindEqualRevisions incorrectly finds that all these controller revisions are equal to the proposed udpate. The updateRevision selects the last revision found here Therefore, updateRevision, is randomly selected and causes the bug. This fixes the immediate issue, but the sort should probably be made stable (i.e not just look at the revision number, perhaps just consider the hash for tie breakers)
As an update, fixing the equality check will not be sufficient because it is possible for multiple controller revisions to be created that are actually equal, including the same hash, that have the same revision number. I believe the fix will need to be equal controllerrevisions with the same revision number from being created sort the controllerrevisions so that the controllerrevision that is the target of the statefulset is stable
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
ping I use the K8S which version is 1.11.4 and encounter the same problem. I want to reproduce the problem and find your way: Manually create a controllerrevision identical to the first (same revision), incrementing both the name and hash by one. Could you tell me how to mannually create a same controllerrevision identical to the first? Thanks in anvance.


Question: } framework.ExpectEqual(successes, largeCompletions, "expected %d successful job pods, but got  %d", largeCompletions, successes) }) <ins> ginkgo.It("should apply changes to a job status", func() { ns := f.Namespace.Name jClient := f.ClientSet.BatchV1().Jobs(ns) ginkgo.By("Creating a job") job := e2ejob.NewTestJob("notTerminate", "suspend-false-to-true", v1.RestartPolicyNever, parallelism, completions, nil, backoffLimit) job, err := e2ejob.CreateJob(f.ClientSet, f.Namespace.Name, job) framework.ExpectNoError(err, "failed to create job in namespace: %s", f.Namespace.Name) ginkgo.By("Ensure pods equal to paralellism count is attached to the job") err = e2ejob.WaitForAllJobPodsRunning(f.ClientSet, f.Namespace.Name, job.Name, parallelism) framework.ExpectNoError(err, "failed to ensure number of pods associated with job %s is equal to parallelism count in namespace: %s", job.Name, f.Namespace.Name) // /status subresource operations ginkgo.By("patching /status") // we need to use RFC3339 version since conversion over the wire cuts nanoseconds now1 := metav1.Now().Rfc3339Copy() jStatus := batchv1.JobStatus{ StartTime: &now1, } jStatusJSON, err := json.Marshal(jStatus) framework.ExpectNoError(err) patchedStatus, err := jClient.Patch(context.TODO(), job.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"patchedstatus":"true"}},"status":`+string(jStatusJSON)+`}`), metav1.PatchOptions{}, "status") framework.ExpectNoError(err) framework.ExpectEqual(patchedStatus.Status.StartTime.Equal(&now1), true, "patched object should have the applied StartTime status") framework.ExpectEqual(patchedStatus.Annotations["patchedstatus"], "true", "patched object should have the applied annotation") ginkgo.By("updating /status") // we need to use RFC3339 version since conversion over the wire cuts nanoseconds now2 := metav1.Now().Rfc3339Copy() var statusToUpdate, updatedStatus *batchv1.Job err = retry.RetryOnConflict(retry.DefaultRetry, func() error { statusToUpdate, err = jClient.Get(context.TODO(), job.Name, metav1.GetOptions{}) if err != nil { return err } statusToUpdate.Status.StartTime = &now2 updatedStatus, err = jClient.UpdateStatus(context.TODO(), statusToUpdate, metav1.UpdateOptions{}) return err }) framework.ExpectNoError(err) framework.ExpectEqual(updatedStatus.Status.StartTime.Equal(&now2), true, fmt.Sprintf("updated object status expected to have updated StartTime %#v, got %#v", statusToUpdate.Status.StartTime, updatedStatus.Status.StartTime)) ginkgo.By("get /status") jResource := schema.GroupVersionResource{Group: "batch", Version: "v1", Resource: "jobs"} gottenStatus, err := f.DynamicClient.Resource(jResource).Namespace(ns).Get(context.TODO(), job.Name, metav1.GetOptions{}, "status") framework.ExpectNoError(err) statusUID, _, err := unstructured.NestedFieldCopy(gottenStatus.Object, "metadata", "uid") framework.ExpectNoError(err) framework.ExpectEqual(string(job.UID), statusUID, fmt.Sprintf("job.UID: %v expected to match statusUID: %v ", job.UID, statusUID)) }) </ins> }) // waitForJobFailure uses c to wait for up to timeout for the Job named jobName in namespace ns to fail.

[x] Test PR : [x] Two weeks soak end date : 6 April 2022 [X] Test promotion PR : According to this APIsnoop query, there are still some remaining Job status endpoints which are untested. There are conformance tests for CronJob status endpoints already the existing as a template, a new for job test has been created. The e2e logs for this test are listed below. This query shows the endpoints hit within a short period of running the e2e test If a test with these calls gets merged, test coverage will go up by 3 points This test is also created with the goal of conformance promotion. /sig testing /sig architecture /area conformance
/triage accepted
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: publicIP := "" if ipConfig.PublicIPAddress != nil && ipConfig.PublicIPAddress.ID != nil { pipID := *ipConfig.PublicIPAddress.ID <del> pipName, err := getLastSegment(pipID) if err != nil { return "", "", fmt.Errorf("failed to get publicIP name for node %q with pipID %q", nodeName, pipID) </del> <ins> matches := vmssPIPConfigurationRE.FindStringSubmatch(pipID) if len(matches) == 7 { resourceGroupName := matches[1] virtualMachineScaleSetName := matches[2] virtualmachineIndex := matches[3] networkInterfaceName := matches[4] IPConfigurationName := matches[5] publicIPAddressName := matches[6] pip, existsPip, err := ss.getVMSSPublicIPAddress(resourceGroupName, virtualMachineScaleSetName, virtualmachineIndex, networkInterfaceName, IPConfigurationName, publicIPAddressName) if err != nil { klog.Errorf("ss.getVMSSPublicIPAddress() failed with error: %v", err) return "", "", err } if existsPip && pip.IPAddress != nil { publicIP = *pip.IPAddress } } else { klog.Warningf("Failed to get VMSS Public IP with ID %s", pipID) </ins> } <ins> } </ins> <del> resourceGroup, err := ss.GetNodeResourceGroup(nodeName) if err != nil { return "", "", err } </del> <ins> return internalIP, publicIP, nil } </ins> <del> pip, existsPip, err := ss.getPublicIPAddress(resourceGroup, pipName) if err != nil { return "", "", err } if existsPip { publicIP = *pip.IPAddress } </del> <ins> func (ss *scaleSet) getVMSSPublicIPAddress(resourceGroupName string, virtualMachineScaleSetName string, virtualmachineIndex string, networkInterfaceName string, IPConfigurationName string, publicIPAddressName string) (pip network.PublicIPAddress, exists bool, err error) { var realErr error var message string ctx, cancel := getContextWithCancel() defer cancel() pip, err = ss.PublicIPAddressesClient.GetVirtualMachineScaleSetPublicIPAddress(ctx, resourceGroupName, virtualMachineScaleSetName, virtualmachineIndex, networkInterfaceName, IPConfigurationName, publicIPAddressName, "") exists, message, realErr = checkResourceExistsFromError(err) if realErr != nil { return pip, false, realErr </ins> } <del> return internalIP, publicIP, nil </del> <ins> if !exists { klog.V(2).Infof("Public IP %q not found with message: %q", publicIPAddressName, message) return pip, false, nil } return pip, exists, err </ins> } // returns a list of private ips assigned to node

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Node external IP of azure VMSS cannot be obtained when cloud-controller-manager is enabled. Corresponding logs are shown as follows:
/sig azure
/milestone v1.16


Question: return g.service } <ins> type rateLimitedRoundTripper struct { rt      http.RoundTripper limiter flowcontrol.RateLimiter } func (rl *rateLimitedRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { rl.limiter.Accept() return rl.rt.RoundTrip(req) } </ins> func getProjectAndZone() (string, string, error) { result, err := metadata.Get("instance/zone") if err != nil {

On a large cluster, the routecontroller basically makes O(n) requests all at once, and most of them error out with "Rate Limit Exceeded". We then wait for operations to complete on the ones that did stick, and damn the rest, they'll be caught next reconcile (which is a while). In fact, in some cases it appears we're spamming hard enough to get rate limited by the upper-level API DOS protections and seeing something akin to , e.g.: We can do better than this!
c.f. , which is the other leg of this problem
I took a look into code and it seems that we are doing full reconciliation every 10s. If we need to create 2000 routes at the beginning it seems to frequently. I think that we need to: first compute how many routes we have to create based on that try to spread them over some time to reduce possibility of causing 'rate limit exceeded' I will try to put some small PR together as a starting point.
I'm reopening this one, since this doesn't seem to be solved. So basically something strange is happening here. With PR we have 10/s limit on API calls send to GCE. And we are still rate-limitted. However, according to documentation we are allowed for 20/s: So something either doesn't work or is weird.
What is super interesting from the logs of recent run: It starts to creating routes: After 2 minutes from that, we are hitting "Rate limit Exceeded" for the first time: And only after 9 minutes we create the first route ever. The second one is created after 15 minutes, and from that point we create a number of them So it seems that nothing useful is happening for the first 2 minutes, and then between 3m30 and 9m30
OK - so I think that what is happening is that all controllers are sharing GCE interface (and thuse they have common throttling). So if one controller is generating a lot of API calls, then other controllers may be throttled. has a hypothesis that it may be caused by nodecontroller.
One thing my PR didn't address (because it was late): Now we have some obnoxious differences in logging when you go to make a request, because the versus could actually be absorbing a fair amount of ratelimiter.


Question: return g.service } <ins> type rateLimitedRoundTripper struct { rt      http.RoundTripper limiter flowcontrol.RateLimiter } func (rl *rateLimitedRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { rl.limiter.Accept() return rl.rt.RoundTrip(req) } </ins> func getProjectAndZone() (string, string, error) { result, err := metadata.Get("instance/zone") if err != nil {

It could before because of the operation rate limiter and operation polling anyways, but now could be worse.
- yes I'm aware of that. And probably this is exactly the case here.
I got fooled by that - I guess we should fix it somehow.
One approach to fixing it is a similar approach to the operation polling code, but in the : warn if we ratelimited for more than N seconds (and dump the request or something? not clear how to clarify what we were sending).
I found something else - Kubelet is also building GCE interface: We have 2000 kubelets, so if all of them send multiple requests to GCE, we may hit limits.
Any idea in what circumstances Kubelet will contact cloud provider?
Hmm - it seems it contacts GCE exactly once at the beginning. So it doesn't explain much.
I think it's only here: I've rarely seen GCE ratelimit readonly calls, in practice, and kubelets come up pretty well staggered.
OK - so with logs that I , it is pretty clear that we are very heavily throttled on GCE api calls. In particular, when I was running 1000-node cluster, I see lines where we were throttle for 1m30. I'm pretty sure it's significantly higher in 2000-node cluster. I'm going to investigate a bit more where all those reuqests come from.
Actually - it seems that we have a pretty big problem here. Basically, Kubelet seems to send a lot of requests to GCE. So to clarify: every time Kubelet is updating NodeStatus (so every 10s by default) it is sending an API call to GCE to get it's not addresses: that means, that if we have 2000 nodes, kubelets generate 200 QPS to GCE (which is significantly more than we can afford) We should think how to solve this issue, but I think the options are: increasing QPS quota or calling GCE only once per X statusUpdates.
The second issue is that we are throttled at the controller level too, and this one I'm tryin to understand right now.


Question: return g.service } <ins> type rateLimitedRoundTripper struct { rt      http.RoundTripper limiter flowcontrol.RateLimiter } func (rl *rateLimitedRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { rl.limiter.Accept() return rl.rt.RoundTrip(req) } </ins> func getProjectAndZone() (string, string, error) { result, err := metadata.Get("instance/zone") if err != nil {

- that said, any rate-limitting on our side, will not solve the kubelet-related problem
Hmm - actually looking into implementation it seems that NodeAddresses are only contacting metadata server, so I'm not longer that convinced...
Also, I think that important thing is that a single call to GCE API can translate to multiple api calls to GCE. In particular - CreateRoute translates to: get instance insert route a bunch of getOperation (until this is finished)
OK - so I think that what is happening here is that since CreateRoute translates to: get instance insert route a bunch of getOperations That means that if we call say 2000 CreateRoute() at the same time, all of them will try to issue: getInstance at the beginning. So we will quickly accumulate 2000 GET requests in the queue. Once they are processed, we generate the "POST route" then for any processed. So It's kind of expected what is happening here. So getting back to the Zach`s PR - I think that throttling POST and GET and OPERATION calls separate is kind of good idea in general.
But I still don't really understand why do we get "RateLimitExceeded". Is that because of Kubelets?
I think that one problem we have at the RouteController level is that if CreateRoute fails, we simply don't react on it. And we will wait at least 5 minutes before even retrying it. We should add some retries at the route controller level too. I will send out some short PR for it.
But I still don't understand why do we get those "RateLimitExceeded" error
OK - so as an update. In GCE, we have a separate rate-limit for number of in-flight CreateRoute call per project. is supposed to address this issue.
OK - so this one is actually fixed. I mean that it is still long, but currently it's purely blocked on GCE.
As a comment - we have an internal bug for it already.


Question: <ins> /* Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package service import "context" /* Copied from: k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2 */ // Service allows encrypting and decrypting data using an external Key Management Service. type Service interface { // Decrypt a given bytearray to obtain the original data as bytes. Decrypt(ctx context.Context, uid string, req *DecryptRequest) ([]byte, error) // Encrypt bytes to a ciphertext. Encrypt(ctx context.Context, uid string, data []byte) (*EncryptResponse, error) // Status returns the status of the KMS. Status(ctx context.Context) (*StatusResponse, error) } // EncryptResponse is the response from the Envelope service when encrypting data. type EncryptResponse struct { Ciphertext  []byte KeyID       string Annotations map[string][]byte } // DecryptRequest is the request to the Envelope service when decrypting data. type DecryptRequest struct { Ciphertext  []byte KeyID       string Annotations map[string][]byte } // StatusResponse is the response from the Envelope service when getting the status of the service. type StatusResponse struct { Version string Healthz string KeyID   string } </ins>

generation for local KEK based on counter; also rotate once a week even if the counter does not hit 2 million? using AES-GCM encryption of local KEK ref: The implementation would be to the directory.
/sig auth /triage accepted
/assign
GitHub didn't allow me to assign the following users: ibihim. Note that only , repo collaborators and people who have commented on this issue/PR can be assigned. Additionally, issues/PRs can only have 10 assignees at the same time. For more information please see <detailsIn response to : /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign
thank you !
/milestone v1.26
Hi bug-triage shadow team member for 1.26 here. Looks like this issue is in the 1.26 milestone, but has been moved to 1.27. Is there something else or should this issue go to 1.27 as well? Thank you!
I just forgot to move this issue /milestone v1.27
/reopen is the partial implementation.
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
My suggestion on the reference implementation, after reading 's comments is like so... The GRPC server, which implements the expects an implementation that satisfies . The local key logic can be a middleware. So it expects a it can fallback to, if it doesn't know a key or needs to encrypt its own local KEK. So the local implementation would be need to: have a storage of s have a reference to the current in use (if we put it into the cache, it might vanish) have a reference to another , which should be the remote KMS. So the sub-stories could look like: write Middleware write Transformers that can get exhausted (by time, by usage) write 's storage write some dummy for remote KMS? I have some sketches of what I am talking about .


Question: <ins> /* Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package service import "context" /* Copied from: k8s.io/apiserver/pkg/storage/value/encrypt/envelope/kmsv2 */ // Service allows encrypting and decrypting data using an external Key Management Service. type Service interface { // Decrypt a given bytearray to obtain the original data as bytes. Decrypt(ctx context.Context, uid string, req *DecryptRequest) ([]byte, error) // Encrypt bytes to a ciphertext. Encrypt(ctx context.Context, uid string, data []byte) (*EncryptResponse, error) // Status returns the status of the KMS. Status(ctx context.Context) (*StatusResponse, error) } // EncryptResponse is the response from the Envelope service when encrypting data. type EncryptResponse struct { Ciphertext  []byte KeyID       string Annotations map[string][]byte } // DecryptRequest is the request to the Envelope service when decrypting data. type DecryptRequest struct { Ciphertext  []byte KeyID       string Annotations map[string][]byte } // StatusResponse is the response from the Envelope service when getting the status of the service. type StatusResponse struct { Version string Healthz string KeyID   string } </ins>

I would implement that in terms of the interface so that it can be used both in testing (via a local key) and in real code (by delegating to a HSM).
Hi bug-triage team member for 1.27 here. This issue is still open but has had no recent activity, and no active PRs. Is everything on track for the Code Freeze in March 14th? Thank you!
/milestone clear
Could be closed, right? :)
We still need metrics and tracing support for the reference implementation (though it is all a lot less important now).
/retitle [KMSv2] Add metrics and tracing in for plugins
This is duplicate of /close
Closing this issue. <detailsIn response to : This is duplicate of Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: apps "k8s.io/api/apps/v1" "k8s.io/api/core/v1" <ins> apiequality "k8s.io/apimachinery/pkg/api/equality" </ins> "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels"

Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: a cluster, And node1 with resource 1 DaemonSet, request resource 4 run , There is no pod created. resource to the node: run , There is still no pod created. What you expected to happen: In step 5, There is one pod created. How to reproduce it (as minimally and precisely as possible): see above description Anything else we need to know?: Environment: Kubernetes version (use ):1.10.5 Cloud provider or hardware configuration: OS (e.g. from /etc/os-release):CentOS7 Kernel (e.g. ): Install tools: Others: /sig app
xref
Thanks! is not work here! Because: DaemonSetsController didn't watch the changes.(we use to updateNode.) we restart kubelet, The DaemonSetsController watch the changes, but it seems that the node info is not updated, and still failed to create pod:
I found it is a bug here, ignore the changes. I made a fix , PTAL! Thanks!
Any workarounds for old cluster versions? Up: looks like in my case helped but after some delay.
, which version of k8s are you using? If it's sunset, you have to merge the pr yourself :). I think restart kubelet trigger delete/add to make sure it sync .
Sorry, I didn't say
I think we still support 1.12 for now :) let's cherry pick it if not and wait for next 1.12 release.


Question: <ins> # Releasing Kubernetes This document explains how to create a Kubernetes release (as in version) and how the version information gets embedded into the built binaries. ## Origin of the Sources Kubernetes may be built from either a git tree (using `hack/build-go.sh`) or from a tarball (using either `hack/build-go.sh` or `go install`) or directly by the Go native build system (using `go get`). When building from git, we want to be able to insert specific information about the build tree at build time. In particular, we want to use the output of `git describe` to generate the version of Kubernetes and the status of the build tree (add a `-dirty` prefix if the tree was modified.) When building from a tarball or using the Go build system, we will not have access to the information about the git tree, but we still want to be able to tell whether this build corresponds to an exact release (e.g. v0.3) or is between releases (e.g. at some point in development between v0.3 and v0.4). ## Version Number Format In order to account for these use cases, there are some specific formats that may end up representing the Kubernetes version. Here are a few examples: - **v0.5**: This is official version 0.5 and this version will only be used when building from a clean git tree at the v0.5 git tag, or from a tree extracted from the tarball corresponding to that specific release. - **v0.5-15-g0123abcd4567**: This is the `git describe` output and it indicates that we are 15 commits past the v0.5 release and that the SHA1 of the commit where the binaries were built was `0123abcd4567`. It is only possible to have this level of detail in the version information when building from git, not when building from a tarball. - **v0.5-15-g0123abcd4567-dirty** or **v0.5-dirty**: The extra `-dirty` prefix means that the tree had local modifications or untracked files at the time of the build, so there's no guarantee that the source code matches exactly the state of the tree at the `0123abcd4567` commit or at the `v0.5` git tag (resp.) - **v0.5-dev**: This means we are building from a tarball or using `go get` or, if we have a git tree, we are using `go install` directly, so it is not possible to inject the git version into the build information. Additionally, this is not an official release, so the `-dev` prefix indicates that the version we are building is after `v0.5` but before `v0.6`. (There is actually an exception where a commit with `v0.5-dev` is not present on `v0.6`, see later for details.) ## Injecting Version into Binaries In order to cover the different build cases, we start by providing information that can be used when using only Go build tools or when we do not have the git version information available. To be able to provide a meaningful version in those cases, we set the contents of variables in a Go source file that will be used when no overrides are present. We are using `pkg/version/base.go` as the source of versioning in absence of information from git. Here is a sample of that file's contents: ``` var ( gitVersion   string = "v0.4-dev"  // version from git, output of $(git describe) gitCommit    string = ""          // sha1 from git, output of $(git rev-parse HEAD) ) ``` This means a build with `go install` or `go get` or a build from a tarball will yield binaries that will identify themselves as `v0.4-dev` and will not be able to provide you with a SHA1. To add the extra versioning information when building from git, the `hack/build-go.sh` script will gather that information (using `git describe` and `git rev-parse`) and then create a `-ldflags` string to pass to `go install` and tell the Go linker to override the contents of those variables at build time. It can, for instance, tell it to override `gitVersion` and set it to `v0.4-13-g4567bcdef6789-dirty` and set `gitCommit` to `4567bcdef6789...` which is the complete SHA1 of the (dirty) tree used at build time. ## Handling Official Versions Handling official versions from git is easy, as long as there is an annotated git tag pointing to a specific version then `git describe` will return that tag exactly which will match the idea of an official version (e.g. `v0.5`). Handling it on tarballs is a bit harder since the exact version string must be present in `pkg/version/base.go` for it to get embedded into the binaries. But simply creating a commit with `v0.5` on its own would mean that the commits coming after it would also get the `v0.5` version when built from tarball or `go get` while in fact they do not match `v0.5` (the one that was tagged) exactly. To handle that case, creating a new release should involve creating two adjacent commits where the first of them will set the version to `v0.5` and the second will set it to `v0.5-dev`. In that case, even in the presence of merges, there will be a single comit where the exact `v0.5` version will be used and all others around it will either have `v0.4-dev` or `v0.5-dev`. The diagram below illustrates it. ![Diagram of git commits involved in the release](./releasing.png) After working on `v0.4-dev` and merging PR 99 we decide it is time to release `v0.5`. So we start a new branch, create one commit to update `pkg/version/base.go` to include `gitVersion = "v0.5"` and `git commit` it. We test it and make sure everything is working as expected. Before sending a PR for it, we create a second commit on that same branch, updating `pkg/version/base.go` to include `gitVersion = "v0.5-dev"`. That will ensure that further builds (from tarball or `go install`) on that tree will always include the `-dev` prefix and will not have a `v0.5` version (since they do not match the official `v0.5` exactly.) We then send PR 100 with both commits in it. Once the PR is accepted, we can use `git tag -a` to create an annotated tag *pointing to the one commit* that has `v0.5` in `pkg/version/base.go` and push it to GitHub. (Unfortunately GitHub tags/releases are not annotated tags, so this needs to be done from a git client and pushed to GitHub using SSH.) ## Parallel Commits While we are working on releasing `v0.5`, other development takes place and other PRs get merged. For instance, in the example above, PRs 101 and 102 get merged to the master branch before the versioning PR gets merged. This is not a problem, it is only slightly inaccurate that checking out the tree at commit `012abc` or commit `345cde` or at the commit of the merges of PR 101 or 102 will yield a version of `v0.4-dev` *but* those commits are not present in `v0.5`. In that sense, there is a small window in which commits will get a `v0.4-dev` or `v0.4-N-gXXX` label and while they're indeed later than `v0.4` but they are not really before `v0.5` in that `v0.5` does not contain those commits. Unfortunately, there is not much we can do about it. On the other hand, other projects seem to live with that and it does not really become a large problem. As an example, Docker commit a327d9b91edf has a `v1.1.1-N-gXXX` label but it is not present in Docker `v1.2.0`: ``` $ git describe a327d9b91edf v1.1.1-822-ga327d9b91edf $ git log --oneline v1.2.0..a327d9b91edf a327d9b91edf Fix data space reporting from Kb/Mb to KB/MB (Non-empty output here means the commit is not present on v1.2.0.) ``` </ins>

Right now, we have a mixture of user documentation and developer documentation in the top-level directory. We should make it easier to find a just user documentation. Step 1 would be to move developer documentation into a subdir like docs/development or something like that. User docs would stay in the top-level docs dir.
move to


Question: cmds.AddCommand(NewCmdPlugin(f, in, out, err)) cmds.AddCommand(NewCmdVersion(f, out)) cmds.AddCommand(NewCmdApiVersions(f, out)) <ins> cmds.AddCommand(NewCmdApiResources(f, out)) </ins> cmds.AddCommand(NewCmdOptions(out)) return cmds

As the RBAC role need to be related to resources. I think we could use a command to get the supported resources. ref to
This would be cool.
I thought we a command that listed resources in the discover doc a long time ago.
is just the versions, but principle is the same. We want to list all the types, their resources, and tie that back to "explain".
What's the best way to determine this information currently? I'm having difficulty figuring out the exact set of permissions (and how they would be written in an RBAC role) for any given kubectl command. The documentation for this right now is split across multiple places and certain things are not explained at all. Specifically, it's difficult to determine which "API verb" is used for which operations. Take as an example. It's unclear if the way to allow this is to use the "proxy" subresource of a pod or service with "get"/"create"/etc. verbs, or to use the "proxy" verb with the resource (pod, service) directly. is a similar case. The almost works for this purpose, but it's still difficult. Operations for a given resource are grouped into categories like write, read, status, and proxy, but these things don't obviously correlate with API verbs. Again, proxy is especially confusing as there is no mention of a proxy "verb". The API operations are only shown using their HTTP verbs. One thing that could help on this document would be a section on the right column under "kubectl command" that shows exactly the rule needed in an RBAC role to allow this command via kubectl. And even then, that assumes an exact 1-to-1 mapping between API endpoints and kubectl commands, which I doubt is always the case. (For example, port fowarding to a pod seems to require both a GET on the pod resource and a CREATE on a port forward subresource.) There also needs to be better documentation about how this information could have been determined in the first place. I've resorted to manually looking through API requests made with but even then it's difficult to figure out. Amusingly, I was the one who wrote which is referenced in the original issue description (as well as the table explaining verbs in ).


Question: cmds.AddCommand(NewCmdPlugin(f, in, out, err)) cmds.AddCommand(NewCmdVersion(f, out)) cmds.AddCommand(NewCmdApiVersions(f, out)) <ins> cmds.AddCommand(NewCmdApiResources(f, out)) </ins> cmds.AddCommand(NewCmdOptions(out)) return cmds

tl;dr: It's too difficult right now to go from "I want a user to be able to run " to "here is the RBAC rule to use for that" for any case where there isn't a specific example shown in the docs.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. Prevent issues from auto-closing with an comment. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
It seem that relative PR stop updated. If this issue still valuable, i will help fix it
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten /remove-lifecycle stale
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close


Question: name: "TestPlugin1", inj:  injectedResult{FilterStatus: int(framework.Error)}, }, { name: "TestPlugin2", inj:  injectedResult{FilterStatus: int(framework.Error)},

changed the default behavior of scheduler framework's - from stopping at the first failed filter to always running all filters. This change breaks Cluster Autoscaler at scale. Cluster Autoscaler imports the scheduler framework to simulate filters on artificial nodes. In a (pretty common) one-pod-per-node scenario, the number of calls is ~quadratic wrt. the number of nodes needed for unschedulable pods - it can be a lot of calls. Before , in such scenario the vast majority of calls would stop at failing Fit or NodeAffinity filters, and not run the rest of them (e.g. PV-related filters). After the change, every call to runs all filters, which completely breaks Cluster Autoscaler at scale (an example scenario of 300 1-pod-per-node pods and 200 node groups takes 10min in 1.24, and ~50s in <1.22). Can we get the old behavior back? Or re-introduce the config option so that Cluster Autoscaler can set it accordingly? Ideally, we'd also like to get this backported as far as possible (the change was introduced in 1.24).
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig scheduling
This seems to be a regression. I would be interested in adding a toggle flag that allows one to toggle on that behavior if this was done on purpose.
That PR wasn't supposed to change the behavior actually. Scheduler is always defaulted to stop upon the first Filter failure, and it's just we offered an option to continue go through other Filter plugins upon failures, and that option was removed recently. Let me check the code to see if there is a regression.
When removed the following changes: It should have kept . I will submit a fix along with test, and backport to 1.24, 1.25, 1.26. Thanks for reporting!


Question: name: "TestPlugin1", inj:  injectedResult{FilterStatus: int(framework.Error)}, }, { name: "TestPlugin2", inj:  injectedResult{FilterStatus: int(framework.Error)},

cc /assign /kind bug /kind regression /priority critical-urgent /triage/accepted
Thanks Kuba, this is indeed a regression on the scheduler end.
/open for patches
/re-open /reopen
Reopened this issue. <detailsIn response to : /re-open /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Thanks for the reporting, and thanks for the fix.
Really sorry about this regression, and hope not to happen again. I think there's no e2e tests between kubernetes scheduler and cluster-autoscaler, so can we also have testcases of pods unschedulable in scheduling perf-test, maybe this can also help to prevent this happening again. UT benchmark test also makes sense, but it's not graphical.
Thanks for the quick fix! Unfortunately I'm afraid this kind of issue would only be caught in a large-scale test scenario, which is not cheap to run continuously.
For this particular issue, UT ( in the PR) is able to catch it. And hence doesn't need to be that complicated to test with cluster-autoscaler. However, standing from a broader angle, why we didn't catch the regression in perf test is b/c the triggering actor is unschedulable Pods, instead of schedulable Pods (which are the happy path most perf tests covered). So going forward, I'd like to see more perf tests to mixin unschedulable Pods, and hence more likely to surface this kind of issue. (we have preemptio preemption test suite but doesn't seem adequate)
This is not for the particular issue, but keeping an eye on whether there's any performance degradation between two kubernetes components, like we do for scheduling performance. but it's actually out of the scope of scheduler. I'll add it as one of the goals of
Actually, the schedulerperf has a test case . And, the schedulerperf collects the metric (), which has the label in it (). But, the problem is currently we don't distinguish the metric values by label on each data; we just collect/aggregate it regardless the label.


Question: name: "TestPlugin1", inj:  injectedResult{FilterStatus: int(framework.Error)}, }, { name: "TestPlugin2", inj:  injectedResult{FilterStatus: int(framework.Error)},

(We can distinguish the metric values by any labels in the schedulerperf as we do in the metric ()) I'd say just distinguishing the happy path and unhappy path with label in metric should be enough to observe the particular performance degradation in the unhappy path.
could you help share the perf-dash link where this metric () is exposed?
Hm, cannot see any scheduler metrics result from the current perf-dash I see some configuration that seems to be to parse the schedulerperf result file though. https://perf- TBH, I haven't visited the perf-dash so much, so not sure whether it's not working for a long time, or got broken recently. Also, another thing, I noticed the perf-dash doesn't show all test cases. All test cases after haven't been executed.
So... probably, something wrong happens in  job (probably in the test case) and the schedulerperf is failing. The schedulerperf doesn't make the result file when fails, so I'm guessing that's why we cannot see any scheduler metrics results from the perf-dash. Okay, I found it. Let me open another issue for this.
cc is this actually a regression, and is it resolved yet?
It's resolved by , and also cherry-picked to 1.24.x ~ 1.26.x. /close
Closing this issue. <detailsIn response to : It's resolved by , and also cherry-picked to 1.24.x ~ 1.26.x. Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: Subsets:    []corev1.EndpointSubset{subset}, }, epSlice } <ins> func TestEndpointsAdapterEnsureEndpointSliceFromEndpoints(t *testing.T) { endpoints1, epSlice1 := generateEndpointsAndSlice("foo", "testing", []int{80, 443}, []string{"10.1.2.3", "10.1.2.4"}) endpoints2, epSlice2 := generateEndpointsAndSlice("foo", "testing", []int{80, 443}, []string{"10.1.2.3", "10.1.2.4", "10.1.2.5"}) testCases := map[string]struct { endpointSlicesEnabled bool expectedError         error expectedEndpointSlice *discovery.EndpointSlice endpointSlices        []*discovery.EndpointSlice namespaceParam        string endpointsParam        *corev1.Endpoints }{ "existing-endpointslice-no-change": { endpointSlicesEnabled: true, expectedError:         nil, expectedEndpointSlice: epSlice1, endpointSlices:        []*discovery.EndpointSlice{epSlice1}, namespaceParam:        "testing", endpointsParam:        endpoints1, }, "existing-endpointslice-change": { endpointSlicesEnabled: true, expectedError:         nil, expectedEndpointSlice: epSlice2, endpointSlices:        []*discovery.EndpointSlice{epSlice1}, namespaceParam:        "testing", endpointsParam:        endpoints2, }, "missing-endpointslice": { endpointSlicesEnabled: true, expectedError:         nil, expectedEndpointSlice: epSlice1, endpointSlices:        []*discovery.EndpointSlice{}, namespaceParam:        "testing", endpointsParam:        endpoints1, }, "endpointslices-disabled": { endpointSlicesEnabled: false, expectedError:         nil, expectedEndpointSlice: nil, endpointSlices:        []*discovery.EndpointSlice{}, namespaceParam:        "testing", endpointsParam:        endpoints1, }, } for name, testCase := range testCases { t.Run(name, func(t *testing.T) { client := fake.NewSimpleClientset() epAdapter := EndpointsAdapter{endpointClient: client.CoreV1()} if testCase.endpointSlicesEnabled { epAdapter.endpointSliceClient = client.DiscoveryV1alpha1() } for _, endpointSlice := range testCase.endpointSlices { _, err := client.DiscoveryV1alpha1().EndpointSlices(endpointSlice.Namespace).Create(endpointSlice) if err != nil { t.Fatalf("Error creating EndpointSlice: %v", err) } } err := epAdapter.EnsureEndpointSliceFromEndpoints(testCase.namespaceParam, testCase.endpointsParam) if !apiequality.Semantic.DeepEqual(testCase.expectedError, err) { t.Errorf("Expected error: %v, got: %v", testCase.expectedError, err) } endpointSlice, err := client.DiscoveryV1alpha1().EndpointSlices(testCase.namespaceParam).Get(testCase.endpointsParam.Name, metav1.GetOptions{}) if err != nil && !errors.IsNotFound(err) { t.Fatalf("Error getting Endpoint Slice: %v", err) } if !apiequality.Semantic.DeepEqual(endpointSlice, testCase.expectedEndpointSlice) { t.Errorf("Expected Endpoint Slice: %v, got: %v", testCase.expectedEndpointSlice, endpointSlice) } }) } } </ins>

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: EndpointSlice for masters was not created after enabling EndpointSlice on a pre-existing cluster. Only those EndpointSlices managed by kube-controller-manager were created. What you expected to happen: EndpointSlice for masters should be created after enabling EndpointSlice on a pre-existing cluster, the same as the EndpointSlices managed by kube-controller-manager. How to reproduce it (as minimally and precisely as possible): kubeadm to create a cluster: the EndpointSlice feature by editing the kube-apiserver, kube-controller-manager, kube-proxy manifest kube-apiserver and kube-controller-manager is restarted, the EndpointSlice in namespace will be missing. Anything else we need to know?: Environment: Kubernetes version (use ):Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools:Network plugin and version (if this is a network-related bug): Others:
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry.
/remove-triage unresolved /assign