Question: <ins> /* Copyright 2020 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package storage import ( "context" "fmt" "path" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/kubernetes/pkg/kubelet/events" "k8s.io/kubernetes/test/e2e/framework" e2eevents "k8s.io/kubernetes/test/e2e/framework/events" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" "k8s.io/kubernetes/test/e2e/storage/utils" imageutils "k8s.io/kubernetes/test/utils/image" "github.com/onsi/ginkgo" ) var _ = utils.SIGDescribe("HostPathType Directory [Slow]", func() { f := framework.NewDefaultFramework("host-path-type-directory") var ( ns           string hostBaseDir  string mountBaseDir string basePod      *v1.Pod targetDir    string hostPathUnset             = v1.HostPathUnset hostPathDirectoryOrCreate = v1.HostPathDirectoryOrCreate hostPathDirectory         = v1.HostPathDirectory hostPathFile              = v1.HostPathFile hostPathSocket            = v1.HostPathSocket hostPathCharDev           = v1.HostPathCharDev hostPathBlockDev          = v1.HostPathBlockDev ) ginkgo.BeforeEach(func() { ns = f.Namespace.Name ginkgo.By("Create a pod for further testing") hostBaseDir = path.Join("/tmp", ns) mountBaseDir = "/mnt/test" basePod = f.PodClient().CreateSync(newHostPathTypeTestPod(map[string]string{}, hostBaseDir, mountBaseDir, &hostPathDirectoryOrCreate)) ginkgo.By(fmt.Sprintf("running on node %s", basePod.Spec.NodeName)) targetDir = path.Join(hostBaseDir, "adir") ginkgo.By("Should automatically create a new directory 'adir' when HostPathType is HostPathDirectoryOrCreate") verifyPodHostPathType(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetDir, &hostPathDirectoryOrCreate) }) ginkgo.It("Should fail on mounting non-existent directory 'does-not-exist-dir' when HostPathType is HostPathDirectory", func() { dirPath := path.Join(hostBaseDir, "does-not-exist-dir") verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, dirPath, fmt.Sprintf("%s is not a directory", dirPath), &hostPathDirectory) }) ginkgo.It("Should be able to mount directory 'adir' successfully when HostPathType is HostPathDirectory", func() { verifyPodHostPathType(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetDir, &hostPathDirectory) }) ginkgo.It("Should be able to mount directory 'adir' successfully when HostPathType is HostPathUnset", func() { verifyPodHostPathType(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetDir, &hostPathUnset) }) ginkgo.It("Should fail on mounting directory 'adir' when HostPathType is HostPathFile", func() { verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetDir, fmt.Sprintf("%s is not a file", targetDir), &hostPathFile) }) ginkgo.It("Should fail on mounting directory 'adir' when HostPathType is HostPathSocket", func() { verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetDir, fmt.Sprintf("%s is not a socket", targetDir), &hostPathSocket) }) ginkgo.It("Should fail on mounting directory 'adir' when HostPathType is HostPathCharDev", func() { verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetDir, fmt.Sprintf("%s is not a character device", targetDir), &hostPathCharDev) }) ginkgo.It("Should fail on mounting directory 'adir' when HostPathType is HostPathBlockDev", func() { verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetDir, fmt.Sprintf("%s is not a block device", targetDir), &hostPathBlockDev) }) }) var _ = utils.SIGDescribe("HostPathType File [Slow]", func() { f := framework.NewDefaultFramework("host-path-type-file") var ( ns           string hostBaseDir  string mountBaseDir string basePod      *v1.Pod targetFile   string hostPathUnset             = v1.HostPathUnset hostPathDirectoryOrCreate = v1.HostPathDirectoryOrCreate hostPathDirectory         = v1.HostPathDirectory hostPathFileOrCreate      = v1.HostPathFileOrCreate hostPathFile              = v1.HostPathFile hostPathSocket            = v1.HostPathSocket hostPathCharDev           = v1.HostPathCharDev hostPathBlockDev          = v1.HostPathBlockDev ) ginkgo.BeforeEach(func() { ns = f.Namespace.Name ginkgo.By("Create a pod for further testing") hostBaseDir = path.Join("/tmp", ns) mountBaseDir = "/mnt/test" basePod = f.PodClient().CreateSync(newHostPathTypeTestPod(map[string]string{}, hostBaseDir, mountBaseDir, &hostPathDirectoryOrCreate)) ginkgo.By(fmt.Sprintf("running on node %s", basePod.Spec.NodeName)) targetFile = path.Join(hostBaseDir, "afile") ginkgo.By("Should automatically create a new file 'afile' when HostPathType is HostPathFileOrCreate") verifyPodHostPathType(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetFile, &hostPathFileOrCreate) }) ginkgo.It("Should fail on mounting non-existent file 'does-not-exist-file' when HostPathType is HostPathFile", func() { filePath := path.Join(hostBaseDir, "does-not-exist-file") verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, filePath, fmt.Sprintf("%s is not a file", filePath), &hostPathFile) }) ginkgo.It("Should be able to mount file 'afile' successfully when HostPathType is HostPathFile", func() { verifyPodHostPathType(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetFile, &hostPathFile) }) ginkgo.It("Should be able to mount file 'afile' successfully when HostPathType is HostPathUnset", func() { verifyPodHostPathType(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetFile, &hostPathUnset) }) ginkgo.It("Should fail on mounting file 'afile' when HostPathType is HostPathDirectory", func() { verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetFile, fmt.Sprintf("%s is not a directory", targetFile), &hostPathDirectory) }) ginkgo.It("Should fail on mounting file 'afile' when HostPathType is HostPathSocket", func() { verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetFile, fmt.Sprintf("%s is not a socket", targetFile), &hostPathSocket) }) ginkgo.It("Should fail on mounting file 'afile' when HostPathType is HostPathCharDev", func() { verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetFile, fmt.Sprintf("%s is not a character device", targetFile), &hostPathCharDev) }) ginkgo.It("Should fail on mounting file 'afile' when HostPathType is HostPathBlockDev", func() { verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetFile, fmt.Sprintf("%s is not a block device", targetFile), &hostPathBlockDev) }) }) var _ = utils.SIGDescribe("HostPathType Socket [Slow]", func() { f := framework.NewDefaultFramework("host-path-type-socket") var ( ns           string hostBaseDir  string mountBaseDir string basePod      *v1.Pod targetSocket string hostPathUnset             = v1.HostPathUnset hostPathDirectoryOrCreate = v1.HostPathDirectoryOrCreate hostPathDirectory         = v1.HostPathDirectory hostPathFile              = v1.HostPathFile hostPathSocket            = v1.HostPathSocket hostPathCharDev           = v1.HostPathCharDev hostPathBlockDev          = v1.HostPathBlockDev ) ginkgo.BeforeEach(func() { ns = f.Namespace.Name ginkgo.By("Create a pod for further testing") hostBaseDir = path.Join("/tmp", ns) mountBaseDir = "/mnt/test" basePod = f.PodClient().CreateSync(newHostPathTypeTestPod(map[string]string{}, hostBaseDir, mountBaseDir, &hostPathDirectoryOrCreate)) ginkgo.By(fmt.Sprintf("running on node %s", basePod.Spec.NodeName)) targetSocket = path.Join(hostBaseDir, "asocket") ginkgo.By("Create a socket for further testing") _, err := utils.PodExec(f, basePod, fmt.Sprintf("nc -lU %s &", path.Join(mountBaseDir, "asocket"))) framework.ExpectNoError(err) }) ginkgo.It("Should fail on mounting non-existent socket 'does-not-exist-socket' when HostPathType is HostPathSocket", func() { socketPath := path.Join(hostBaseDir, "does-not-exist-socket") verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, socketPath, fmt.Sprintf("%s is not a socket", socketPath), &hostPathSocket) }) ginkgo.It("Should be able to mount socket 'asocket' successfully when HostPathType is HostPathSocket", func() { verifyPodHostPathType(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetSocket, &hostPathSocket) }) ginkgo.It("Should be able to mount socket 'asocket' successfully when HostPathType is HostPathUnset", func() { verifyPodHostPathType(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetSocket, &hostPathUnset) }) ginkgo.It("Should fail on mounting socket 'asocket' when HostPathType is HostPathDirectory", func() { verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetSocket, fmt.Sprintf("%s is not a directory", targetSocket), &hostPathDirectory) }) ginkgo.It("Should fail on mounting socket 'asocket' when HostPathType is HostPathFile", func() { verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetSocket, fmt.Sprintf("%s is not a file", targetSocket), &hostPathFile) }) ginkgo.It("Should fail on mounting socket 'asocket' when HostPathType is HostPathCharDev", func() { verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetSocket, fmt.Sprintf("%s is not a character device", targetSocket), &hostPathCharDev) }) ginkgo.It("Should fail on mounting socket 'asocket' when HostPathType is HostPathBlockDev", func() { verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetSocket, fmt.Sprintf("%s is not a block device", targetSocket), &hostPathBlockDev) }) }) var _ = utils.SIGDescribe("HostPathType Character Device [Slow]", func() { f := framework.NewDefaultFramework("host-path-type-char-dev") var ( ns            string hostBaseDir   string mountBaseDir  string basePod       *v1.Pod targetCharDev string hostPathUnset             = v1.HostPathUnset hostPathDirectoryOrCreate = v1.HostPathDirectoryOrCreate hostPathDirectory         = v1.HostPathDirectory hostPathFile              = v1.HostPathFile hostPathSocket            = v1.HostPathSocket hostPathCharDev           = v1.HostPathCharDev hostPathBlockDev          = v1.HostPathBlockDev ) ginkgo.BeforeEach(func() { ns = f.Namespace.Name ginkgo.By("Create a pod for further testing") hostBaseDir = path.Join("/tmp", ns) mountBaseDir = "/mnt/test" basePod = f.PodClient().CreateSync(newHostPathTypeTestPod(map[string]string{}, hostBaseDir, mountBaseDir, &hostPathDirectoryOrCreate)) ginkgo.By(fmt.Sprintf("running on node %s", basePod.Spec.NodeName)) targetCharDev = path.Join(hostBaseDir, "achardev") ginkgo.By("Create a character device for further testing") _, err := utils.PodExec(f, basePod, fmt.Sprintf("mknod %s c 89 1", path.Join(mountBaseDir, "achardev"))) framework.ExpectNoError(err) }) ginkgo.It("Should fail on mounting non-existent character device 'does-not-exist-char-dev' when HostPathType is HostPathCharDev", func() { charDevPath := path.Join(hostBaseDir, "does-not-exist-char-dev") verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, charDevPath, fmt.Sprintf("%s is not a character device", charDevPath), &hostPathCharDev) }) ginkgo.It("Should be able to mount character device 'achardev' successfully when HostPathType is HostPathCharDev", func() { verifyPodHostPathType(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetCharDev, &hostPathCharDev) }) ginkgo.It("Should be able to mount character device 'achardev' successfully when HostPathType is HostPathUnset", func() { verifyPodHostPathType(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetCharDev, &hostPathUnset) }) ginkgo.It("Should fail on mounting character device 'achardev' when HostPathType is HostPathDirectory", func() { verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetCharDev, fmt.Sprintf("%s is not a directory", targetCharDev), &hostPathDirectory) }) ginkgo.It("Should fail on mounting character device 'achardev' when HostPathType is HostPathFile", func() { verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetCharDev, fmt.Sprintf("%s is not a file", targetCharDev), &hostPathFile) }) ginkgo.It("Should fail on mounting character device 'achardev' when HostPathType is HostPathSocket", func() { verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetCharDev, fmt.Sprintf("%s is not a socket", targetCharDev), &hostPathSocket) }) ginkgo.It("Should fail on mounting character device 'achardev' when HostPathType is HostPathBlockDev", func() { verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetCharDev, fmt.Sprintf("%s is not a block device", targetCharDev), &hostPathBlockDev) }) }) var _ = utils.SIGDescribe("HostPathType Block Device [Slow]", func() { f := framework.NewDefaultFramework("host-path-type-block-dev") var ( ns             string hostBaseDir    string mountBaseDir   string basePod        *v1.Pod targetBlockDev string hostPathUnset             = v1.HostPathUnset hostPathDirectoryOrCreate = v1.HostPathDirectoryOrCreate hostPathDirectory         = v1.HostPathDirectory hostPathFile              = v1.HostPathFile hostPathSocket            = v1.HostPathSocket hostPathCharDev           = v1.HostPathCharDev hostPathBlockDev          = v1.HostPathBlockDev ) ginkgo.BeforeEach(func() { ns = f.Namespace.Name ginkgo.By("Create a pod for further testing") hostBaseDir = path.Join("/tmp", ns) mountBaseDir = "/mnt/test" basePod = f.PodClient().CreateSync(newHostPathTypeTestPod(map[string]string{}, hostBaseDir, mountBaseDir, &hostPathDirectoryOrCreate)) ginkgo.By(fmt.Sprintf("running on node %s", basePod.Spec.NodeName)) targetBlockDev = path.Join(hostBaseDir, "ablkdev") ginkgo.By("Create a block device for further testing") _, err := utils.PodExec(f, basePod, fmt.Sprintf("mknod %s b 89 1", path.Join(mountBaseDir, "ablkdev"))) framework.ExpectNoError(err) }) ginkgo.It("Should fail on mounting non-existent block device 'does-not-exist-blk-dev' when HostPathType is HostPathBlockDev", func() { blkDevPath := path.Join(hostBaseDir, "does-not-exist-blk-dev") verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, blkDevPath, fmt.Sprintf("%s is not a block device", blkDevPath), &hostPathBlockDev) }) ginkgo.It("Should be able to mount block device 'ablkdev' successfully when HostPathType is HostPathBlockDev", func() { verifyPodHostPathType(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetBlockDev, &hostPathBlockDev) }) ginkgo.It("Should be able to mount block device 'ablkdev' successfully when HostPathType is HostPathUnset", func() { verifyPodHostPathType(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetBlockDev, &hostPathUnset) }) ginkgo.It("Should fail on mounting block device 'ablkdev' when HostPathType is HostPathDirectory", func() { verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetBlockDev, fmt.Sprintf("%s is not a directory", targetBlockDev), &hostPathDirectory) }) ginkgo.It("Should fail on mounting block device 'ablkdev' when HostPathType is HostPathFile", func() { verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetBlockDev, fmt.Sprintf("%s is not a file", targetBlockDev), &hostPathFile) }) ginkgo.It("Should fail on mounting block device 'ablkdev' when HostPathType is HostPathSocket", func() { verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetBlockDev, fmt.Sprintf("%s is not a socket", targetBlockDev), &hostPathSocket) }) ginkgo.It("Should fail on mounting block device 'ablkdev' when HostPathType is HostPathCharDev", func() { verifyPodHostPathTypeFailure(f, map[string]string{"kubernetes.io/hostname": basePod.Spec.NodeName}, targetBlockDev, fmt.Sprintf("%s is not a character device", targetBlockDev), &hostPathCharDev) }) }) func newHostPathTypeTestPod(nodeSelector map[string]string, hostDir, mountDir string, hostPathType *v1.HostPathType) *v1.Pod { pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "test-hostpath-type-", }, Spec: v1.PodSpec{ NodeSelector:  nodeSelector, RestartPolicy: v1.RestartPolicyNever, Containers: []v1.Container{ { Name:  "host-path-testing", Image: imageutils.GetE2EImage(imageutils.Agnhost), VolumeMounts: []v1.VolumeMount{ { Name:      "host", MountPath: mountDir, ReadOnly:  false, }, }, }, }, Volumes: []v1.Volume{ { Name: "host", VolumeSource: v1.VolumeSource{ HostPath: &v1.HostPathVolumeSource{ Path: hostDir, Type: hostPathType, }, }, }, }, }, } return pod } func verifyPodHostPathTypeFailure(f *framework.Framework, nodeSelector map[string]string, hostDir, pattern string, hostPathType *v1.HostPathType) { pod := newHostPathTypeTestPod(nodeSelector, hostDir, "/mnt/test", hostPathType) ginkgo.By(fmt.Sprintf("Creating pod %s", pod.Name)) pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), pod, metav1.CreateOptions{}) framework.ExpectNoError(err) ginkgo.By("Checking for HostPathType error event") eventSelector := fields.Set{ "involvedObject.kind":      "Pod", "involvedObject.name":      pod.Name, "involvedObject.namespace": f.Namespace.Name, "reason":                   events.FailedMountVolume, }.AsSelector().String() msg := "hostPath type check failed" err = e2eevents.WaitTimeoutForEvent(f.ClientSet, f.Namespace.Name, eventSelector, msg, framework.PodStartTimeout) // Events are unreliable, don't depend on the event. It's used only to speed up the test. if err != nil { framework.Logf("Warning: did not get event about FailedMountVolume") } // Check the pod is still not running p, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), pod.Name, metav1.GetOptions{}) framework.ExpectNoError(err, "could not re-read the pod after event (or timeout)") framework.ExpectEqual(p.Status.Phase, v1.PodPending, "Pod phase isn't pending") f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(context.TODO(), pod.Name, *metav1.NewDeleteOptions(0)) } func verifyPodHostPathType(f *framework.Framework, nodeSelector map[string]string, hostDir string, hostPathType *v1.HostPathType) { newPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), newHostPathTypeTestPod(nodeSelector, hostDir, "/mnt/test", hostPathType), metav1.CreateOptions{}) framework.ExpectNoError(err) framework.ExpectNoError(e2epod.WaitTimeoutForPodRunningInNamespace(f.ClientSet, newPod.Name, newPod.Namespace, framework.PodStartShortTimeout)) f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(context.TODO(), newPod.Name, *metav1.NewDeleteOptions(0)) } </ins>

Send feedback to sig-testing, kubernetes/test-infra and/or . /close
Closing this issue. <detailsIn response to : Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: <!-- END MUNGE: GENERATED_TOC --> When specifying a [pod](pods.md), you can optionally specify how much CPU and memory (RAM) each <del> container needs.  When containers have resource limits, the scheduler is able to make better decisions about which nodes to place pods on, and contention for resources can be handled in a consistent manner. </del> <ins> container needs.  When containers have their resource requests specified, the scheduler is able to make better decisions about which nodes to place pods on; and when containers have their limits specified, contention for resources on a node can be handled in a specified manner. For more details about the difference between requests and limits, please refer to [Resource QoS](../proposals/resource-qos.md). </ins> *CPU* and *memory* are each a *resource type*.  A resource type has a base unit.  CPU is specified in units of cores.  Memory is specified in units of bytes.

did which addresses some of them, but turns up a lot more.
Alright, I will do that.


Question: go_library( name = "go_default_library", <del> srcs = select({ "@io_bazel_rules_go//go/platform:linux_amd64": [ "fake.go", ], "//conditions:default": [], }), </del> <ins> srcs = ["fake.go"], </ins> importpath = "k8s.io/kubernetes/pkg/proxy/ipvs/testing", tags = ["automanaged"], )

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: IPVS proxy UT breaks in macOS platform. What you expected to happen: Should run IPVS proxy UTs everywhere. How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: That's because there is a flag in , see Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others: /sig network
/assign cc


Question: return false, fmt.Errorf("error checking rule: %v", err) } <del> // Sadly, iptables has inconsistent quoting rules for comments. // Just unquote any arg that is wrapped in quotes. argsCopy := make([]string, len(args)) copy(argsCopy, args) for i := range argsCopy { unquote(&argsCopy[i]) </del> <ins> // Sadly, iptables has inconsistent quoting rules for comments. Just remove all quotes. // Also, quoted multi-word comments (which are counted as a single arg) // will be unpacked into multiple args, // in order to compare against iptables-save output (which will be split at whitespace boundary) // e.g. a single arg('"this must be before the NodePort rules"') will be unquoted and unpacked into 7 args. var argsCopy []string for i := range args { tmpField := strings.Trim(args[i], """) argsCopy = append(argsCopy, strings.Fields(tmpField)...) </ins> } argset := util.NewStringSet(argsCopy...)

I am running kubernetes 0.5.2 + and I've noticed that the number of entries in iptables increases every few seconds. I started with a clean install, no pods or replicationControllers, there is now two services by default (kubernetes-ro and kubernetes). I ran and got . Next, I turned up 7 services and 3 replicationControllers which resulted in 3 pods also being turned up. I started a watch... I saw "38", "43, "50, "64", "66", "68", "70", ..., "115" and on until I stopped . It appears the proxy continues to add REDIRECT's for the services, for example the lines that keep being look like: A similar line for each service gets every second or so, it appears to all be in the table. I'm thinking the newly function might be the culprit, since calls and in the log I see every few seconds (looks like does a sync every 5 seconds).
I noticed that the 0.5.3 and 0.5.4 releases don't include the , though it's in master. Perhaps the issue I've filed is already known and I missed it when searching the issues.
I have not heard this issue. Can you paste a listing of iptables-save -t nat when you are seeing this problem? Alternatively, I can cut you a branch with some extra debug instrumentation if you're willing to test it out today...
I'm not able to test it out today. As far as the iptables output, I'd have to type it in (can't copy/paste -- different physical network). However, the output has lines, like the one given in the issue, which are every 5 seconds to the "nat" table. It seems like the check to see if the rule already exists is perhaps not functioning properly. I am working on a fix for (because it uses the iptables -C flag which isn't supported in older iptables), in that fix I created a function to check if an existing rule already exists: Example call: Using may not be ideal, perhaps breaking up the output of the iptables command into a slice and checking each line would be better(?) sure yet.
Fix is pending - would be great if someone could verify it against an actually old iptables (I just faked the version string).


Question: return false, fmt.Errorf("error checking rule: %v", err) } <del> // Sadly, iptables has inconsistent quoting rules for comments. // Just unquote any arg that is wrapped in quotes. argsCopy := make([]string, len(args)) copy(argsCopy, args) for i := range argsCopy { unquote(&argsCopy[i]) </del> <ins> // Sadly, iptables has inconsistent quoting rules for comments. Just remove all quotes. // Also, quoted multi-word comments (which are counted as a single arg) // will be unpacked into multiple args, // in order to compare against iptables-save output (which will be split at whitespace boundary) // e.g. a single arg('"this must be before the NodePort rules"') will be unquoted and unpacked into 7 args. var argsCopy []string for i := range args { tmpField := strings.Trim(args[i], """) argsCopy = append(argsCopy, strings.Fields(tmpField)...) </ins> } argset := util.NewStringSet(argsCopy...)

should be able to try it out friday, maybe monday
Feel free to rip off our checks with out echeck function :) On Nov 26, 2014 7:36 PM, "Craig Wickesser" wrote:
that it actually works. On Nov 26, 2014 7:40 PM, "Tim Hockin" wrote:
I fired up kuberenetes 0.5.2 (plus + your changes) and things looked good initially (when starting up kubernetes, no pods/services, etc). The number of entries iptables-save remained constant. I then turned up some services and watched the length of iptables-save increase. I turned down the services and the number of entries in iptables-save continues to increase. At the moment, every 5 seconds it's back to adding two entries (one for the kubernetes service and one for the kubernetes-ro service). With log level set to 0, I see "Opened iptables portal for service.." messages in the proxy log every 5 seconds ().
Are you sure you have the right proxy version running? I could repro the initial problem, but not this - it works just fine. To be clear when you say it increases after creating a service - do you mean it increases by 1 rule or that it increases continually? , Craig Wickesser wrote:
On Monday/Tuesday I'll try the latest from master, perhaps I messed something up. To clarify, the increase was continuous. When I initially setup kubernetes (with a fresh/clean etcd), things looked normal with regards to iptables. I then deployed some pods and services and kept watching iptables-save. The number of entries increased every 5 seconds. I deleted pods, replication controllers and services and the number of entries continued to increase, but only by two (for the kubernetes and kubernetes-ro services) every 5 seconds.
Did you paste the output of iptables-save? Its possible that it is slightly different from my version. This is why parsing text is no fun :( On Nov 28, 2014 3:42 PM, "Craig Wickesser" wrote:
I did not include the output of iptables-save. I'll see if I can get additional output besides the line included in the initial issue.
Just a couple lines so I can see the format.


Question: return false, fmt.Errorf("error checking rule: %v", err) } <del> // Sadly, iptables has inconsistent quoting rules for comments. // Just unquote any arg that is wrapped in quotes. argsCopy := make([]string, len(args)) copy(argsCopy, args) for i := range argsCopy { unquote(&argsCopy[i]) </del> <ins> // Sadly, iptables has inconsistent quoting rules for comments. Just remove all quotes. // Also, quoted multi-word comments (which are counted as a single arg) // will be unpacked into multiple args, // in order to compare against iptables-save output (which will be split at whitespace boundary) // e.g. a single arg('"this must be before the NodePort rules"') will be unquoted and unpacked into 7 args. var argsCopy []string for i := range args { tmpField := strings.Trim(args[i], """) argsCopy = append(argsCopy, strings.Fields(tmpField)...) </ins> } argset := util.NewStringSet(argsCopy...)

On Nov 28, 2014 7:32 PM, "Craig Wickesser" wrote:
here's a few lines from : All of the lines follow the exact same format. Looks like the proxier (to fit with the output I provided) would need to look like this:
FWIW - I'm using CentOS 6.5 with iptables version 1.4.7.
I recompiled with changes to proxier as mentioned earlier (changed the order of args) and did not resolve the issue. I continue seeing (in the proxy log):
I've some additional logging to log the and the . I am logging them using these two lines: When I reduce the output between the and the (when it should return "true", that fields is a superset), the result looks like: Notice has quotes around it in the set, and the does not? I think perhaps that's the issue and that's why the check is false.
I just tried changing to: and that's working. Basically, strip out quotes from the line. There might be a better alternative.
Uggh, seriously? Your iptables add quotes around the comment, mine omits the quotes for single tokens. I guess I can force it to not be a single word, something like "service: " service name. What a hack. Any better ideas? , Craig Wickesser wrote:
Hmm, it seems worse than that. If I feed in a comment with quotes, I get ""foo"", so I can't do that to be compatible with your implementation. Can you try something? iptables -t nat -I PREROUTING -d 1.2.3.4 -m comment --comment "foo1" iptables -t nat -I PREROUTING -d 1.2.3.4 -m comment --comment '"foo2"' iptables-save | grep foo -A PREROUTING -d 1.2.3.4/32 -m comment --comment ""foo2"" -A PREROUTING -d 1.2.3.4/32 -m comment --comment foo1 , Tim Hockin wrote:
i'll try that in about 8 hours :)
here's what I got:
ugh, that is SO BROKEN. I'm going to have to do something really unpleasant.


Question: return false, fmt.Errorf("error checking rule: %v", err) } <del> // Sadly, iptables has inconsistent quoting rules for comments. // Just unquote any arg that is wrapped in quotes. argsCopy := make([]string, len(args)) copy(argsCopy, args) for i := range argsCopy { unquote(&argsCopy[i]) </del> <ins> // Sadly, iptables has inconsistent quoting rules for comments. Just remove all quotes. // Also, quoted multi-word comments (which are counted as a single arg) // will be unpacked into multiple args, // in order to compare against iptables-save output (which will be split at whitespace boundary) // e.g. a single arg('"this must be before the NodePort rules"') will be unquoted and unpacked into 7 args. var argsCopy []string for i := range args { tmpField := strings.Trim(args[i], """) argsCopy = append(argsCopy, strings.Fields(tmpField)...) </ins> } argset := util.NewStringSet(argsCopy...)

, Craig Wickesser wrote:
I have a potential fix, but airport wifi prevents me from pushing git <shrug. will publish it later.
sounds good, I appreciate the effort.
PR posted - please test before we commit.
I've tried the PR and a comment to it.
got the same problem here using Kubernetes v0.6.0-12-gd927031aecdeb6 on redhat 6.5
It looks like 0.6 was cut before - - can we patch that in, or should we wait of 0.7? , polariss0i wrote:
got the same problem on kubernetes v1.0.3 and iptables-v1.4.7. Should I file a new issue?


Question: return a.Add(int64Amount{value: -b.value, scale: b.scale}) } <ins> // Mul multiplies the provided b to the current amount, or // returns false if overflow or underflow would result. func (a *int64Amount) Mul(b int64) bool { switch { case a.value == 0: return true case b == 0: a.value = 0 a.scale = 0 return true case a.scale == 0: c, ok := int64Multiply(a.value, b) if !ok { return false } a.value = c case a.scale > 0: c, ok := int64Multiply(a.value, b) if !ok { return false } if _, ok = positiveScaleInt64(c, a.scale); !ok { return false } a.value = c default: c, ok := int64Multiply(a.value, b) if !ok { return false } if _, ok = negativeScaleInt64(c, -a.scale); !ok { return false } a.value = c } return true } </ins> // AsScale adjusts this amount to set a minimum scale, rounding up, and returns true iff no precision // was lost. (1.1e5).AsScale(5) would return 1.1e5, but (1.1e5).AsScale(6) would return 1e6. func (a int64Amount) AsScale(scale Scale) (int64Amount, bool) {

I'd like to add a multiplication method for the resource.Quantity. Currently, the resource package provides helper methods to calculate Quantity, such as and : Batch workloads often replicate the same PodTemplate, and controllers calculate the resources needed for all containers. In that case, It is inconvenient to call the function as many times as podTemplate, so I would like to create a helper function for multiplication.
/cc /sig api-machinary /wg batch
The label(s) cannot be applied, because the repository doesn't have them. <detailsIn response to : /cc /sig api-machinary /wg batch </details
/sig api-machinery /wg batch
Do you have an example of usage for these utilities? Just curious?
/triage accepted
Sorry for the late response. Taking k-sigs/kueue as an example, I assumed the below: It helps the controllers for batch workloads such as k-sigs/kueue, k-sigs/jobset, kubeflow/mpi-operator, and kubeflow/training-operator. I'm thinking of implementing this function once the code freeze term is finished.
/assign
Hi, may i take this up?
I took this one, and working on this now.


Question: ImagePullPolicy:          "IfNotPresent", TerminationMessagePolicy: "File", }, <ins> { Name:                     "env-from-source", Image:                    "image", ImagePullPolicy:          "IfNotPresent", TerminationMessagePolicy: "File", EnvFrom: []api.EnvFromSource{ { ConfigMapRef: &api.ConfigMapEnvSource{ LocalObjectReference: api.LocalObjectReference{ Name: "test", }, }, }, }, }, </ins> {Name: "abc-1234", Image: "image", ImagePullPolicy: "IfNotPresent", TerminationMessagePolicy: "File", SecurityContext: fakeValidSecurityContext(true)}, } if errs := validateContainers(successCase, volumes, field.NewPath("field")); len(errs) != 0 {

Scenario: Expected results: pod would not be created as [0] is invalid Actual results: pod is created. Turns out we are not doing any validation of envFrom in container spec today.
There are no sig labels on this issue. Please by:<br(1) mentioning a sig: <br(2) specifying the label manually: <br<brNote: method (1) will trigger a notification to the team. You can find the team list .
I don't know that we can tighten without failing pods.
The validation code exists, but there was no wiring done from validateContainers.
Pods fail on kubelet already. So those pods were not doing much already. We had a BZ on this.
Fail how - fail to even start, or fail to get the input?


Question: rotateCertificates: true runtimeRequestTimeout: "${RUNTIME_REQUEST_TIMEOUT}" staticPodPath: "${POD_MANIFEST_PATH}" <ins> resolvConf: "${KUBELET_RESOLV_CONF}" </ins> EOF { # authentication

changed from "kube-dns" to "core-dns". "core-dns" is failing for me on Debian Buster: works when I use . "core-dns" should work. Perhaps related to this? Kubernetes version (use ): current master () OS (e.g: ): Debian Buster Kernel (e.g. ): 5.5.0-0.bpo.2-amd64 /cc
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
provides a description of the error, causes, and workarounds. Typically this is due to a local dns resolver installed on the host. Solution is to configure kubelet to use a different (more detail in link).
That mirrors what I found through the issue. Needs to be fixed in then.
/sig testing /cc
/cc
I think we have to parameterize this
I think the safer way is to configure kublet to send the correct to all “default” dns policy pods, not just fix in coredns.
Doh, right :)


Question: logOptions.SinceSeconds = &sec } <del> if len(o.Selector) > 0 && o.Tail != -1 { </del> <ins> if len(o.Selector) > 0 && o.Tail == -1 { </ins> logOptions.TailLines = &selectorTail } else if o.Tail != -1 { logOptions.TailLines = &o.Tail

What happened: I do . I see the last 10 lines of output of each pod matching . What you expected to happen: I was expecting to see only the last line of output of each pod. How to reproduce it (as minimally and precisely as possible): The image prints the date every second. Normally we should just see two lines of output; instead we see 20. Anything else we need to know?: This seems to be a regression in Kubernetes 1.12, as this worked correctly from 1.8 to 1.11. (I observed this problem starting with k8s 1.12.0 and it's still present in 1.12.2) Environment: Kubernetes version (use ): 1.12.2 Cloud provider or hardware configuration: AWS OS (e.g. from /etc/os-release): Ubuntu Kernel (e.g. ): 4.4.0 Install tools: kubeadm Others: <!-- DO NOT EDIT BELOW THIS LINE --/kind bug
It looks like the issue was introduced by commit , specifically . If I understand that code correctly, it forces the number of lines to (10) even if is set. Since that seems to be in : /sig cli
/assign
didn't that commit just move the code from ?
Not quite; the old code was: The new code is: I don't know why it was changed, but the new logic reads as "if the user specified a selector and a number of lines different from -1, then set the number of lines to 10".


Question: if isHTEnabled() { cpuList = cpuset.MustParse(getCPUSiblingList(0)).ToSlice() cpu1 = cpuList[1] <ins> } else if isMultiNUMA() { cpuList = cpuset.MustParse(getCoreSiblingList(0)).ToSlice() cpu1 = cpuList[1] </ins> } expAllowedCPUsListRegex = fmt.Sprintf("^%dn$", cpu1) err = f.PodClient().MatchContainerOutput(pod.Name, pod.Spec.Containers[0].Name, expAllowedCPUsListRegex)

Which jobs are failing: N/A - AFAICT CI jobs works fine Which test(s) are failing: Since when has it been failing: no historical record on CI Testgrid link: N/A Reason for failure: I believe the tests make too many assumptions around the hardware, and the code is actually OK. to be confirmed. Anything else we need to know:
/sig node /sig testing
also affects the recently topology manager e2e tests. My PR uses a different approach to check resource alignment so is not affected. /cc
I'll work on a PR to fix this issue ASAP
Thanks for opening this issue. We knew about this, just had not opened an issue to track it, so thanks for doing that! Deepthi has been looing at CPU Manager tests, not sure if she has some PR in progress or not. I'll make sure to forward this to her (can find her user id here ....) in case she has something in progress


Question: for i := 0; i < AllowedInflightRequestsNo; i++ { // These should hang waiting on block... go func() { <del> expectHTTP(server.URL+"/foo/bar/watch", http.StatusOK, t) </del> <ins> if err := expectHTTP(server.URL+"/foo/bar/watch", http.StatusOK); err != nil { t.Error(err) } responses.Done() </ins> }() } // Check that sever is not saturated by not-accounted calls <del> expectHTTP(server.URL+"/dontwait", http.StatusOK, t) oneAccountedFinished := sync.WaitGroup{} oneAccountedFinished.Add(1) var once sync.Once </del> <ins> if err := expectHTTP(server.URL+"/dontwait", http.StatusOK); err != nil { t.Error(err) } </ins> // These should hang and be accounted, i.e. saturate the server for i := 0; i < AllowedInflightRequestsNo; i++ { // These should hang waiting on block... go func() { <del> expectHTTP(server.URL, http.StatusOK, t) once.Do(oneAccountedFinished.Done) </del> <ins> if err := expectHTTP(server.URL, http.StatusOK); err != nil { t.Error(err) } responses.Done() </ins> }() } // We wait for all calls to be received by the server

Have only seen this once, personally.
This flaked in Jenkins as well. [job/kubernetes-test-go/5351]
can you take a look at this? I think has a lot already :)


Question: if [[ -z "${is_push}" ]]; then echo "== kube-up node config starting ==" set-broken-motd <ins> ensure-basic-networking </ins> ensure-install-dir set-kube-env [[ "${KUBERNETES_MASTER}" == "true" ]] && mount-master-pd

Node 2's kubelet logs: controller-manager side: I have a cluster to hand off if someone wants to debug. It's not a transient state: node 2 just failed to ever post a valid status. cc cc (who looked at it briefly with me)
I will take a look soon.
Ok, why kubelet on node 2 talks to ip address of master instead of hostname? root ps -ef | grep kubelet root 1 3 17:35 ? 00:10:45 /usr/local/bin/kubelet --apiservers=https://10.240.49.47 --config=/etc/kubernetes/manifests --allowprivileged=False --v=2 --clusterdns=10.51.240.10 -- It looks like a cluster up issue to me, not a node problem.
It looks like failed to translate the master name into the fqdn (which is done ).
Fascinating. I think we're seeing instance-DNS flakiness, because that name is supposed to get converted ASAP. But what's weirder is ... that's not even the master:
The is definitely right:
Okay, this is messed up. Here's node 1, which is happy: Here's our unhappy node .... that decided to bring itself as a master:
haha... I checked kube-env and did suspect unhappy node treat itself as master node based on those variables. But I didn't check salt state :-)
I ... just can't get there. By the time we get here: , node-2 has set: My best guess is that flaked here: . I think we're seeing a systemic issue where we're racing with GCE network configuration.
I'm going to work up a PR that hopefully improves the reliability of this section to some extent.
<a href=""<img src="" title="made at "/</a
My working theory is that we raced with the configuration done normally by the GCE NetworkManager, leaving us in a bogus state when we made that check.
Why are we relying on inferring the master/node status from the hostname. It should be obvious from the variables set in the kube-env if you are the master or not. Masters don't get passed so why not just check that? Making sure DNS is working is probably also good for reliability...
On GKE, we actually pass the same kube-env on both sides for simplicity.


Question: if [[ -z "${is_push}" ]]; then echo "== kube-up node config starting ==" set-broken-motd <ins> ensure-basic-networking </ins> ensure-install-dir set-kube-env [[ "${KUBERNETES_MASTER}" == "true" ]] && mount-master-pd

We can obviously change that, but I was going for a change that didn't involve a deployment change on our side, too.


Question: // container.  The container's absolute path refers to its hierarchy in the // cgroup file system. e.g. The root container, which represents the whole // machine, has path "/"; all docker containers have path "/docker/<docker id>" <del> func (kl *Kubelet) statsFromContainerPath(containerPath string, req *info.ContainerInfoRequest) (*info.ContainerInfo, error) { cinfo, err := kl.cadvisorClient.ContainerInfo(containerPath, getCadvisorContainerInfoRequest(req)) </del> <ins> func (kl *Kubelet) statsFromContainerPath(cc CadvisorInterface, containerPath string, req *info.ContainerInfoRequest) (*info.ContainerInfo, error) { cinfo, err := cc.ContainerInfo(containerPath, getCadvisorContainerInfoRequest(req)) </ins> if err != nil { return nil, err }

So today, cadvisor became wedged and just stopped paying attention to its socket. Not kubelet's fault. But when I would launch all I would get would be: It took me a bit of trolling through netstat and lsof to figure out that the problem was cadvisor being completely locked up. kill -9 cadvisor and everything started working well. I wonder if we should emit more messages during startup about what operation we are about to attempt, not just the results of operations, as I was blaming docker for a while....
kubelet shouldn't block on this. I'll take a look.
Any ideas what was locking cAdvisor? and was it just not responding to requests?
I don't have any idea why it was locked. If I find another one locked I'll hang onto it to debug. Journal had: obviously the 'kill' came from me. It would not respond to systemctl cadvisor stop. netstat looking at port 4194 showed me At which point I realized cadvisor wasn't handling its sockets right since so many long dead kubelet sockets hadn't been cleaned up, so I kill -9'd it.
I cut some interesting info from the cadvisor journal log (trying to cut hostnames) You can see the last message about failed to update stats for container ... happened Oct 08 01:06:01. It was then silent until I killed it Oct 08 19:20:13, even though I'm pretty sure dnf was still being launched every hour on the hour. So it was wedged for about 18 hours.
What kernel is this on? We've seen there is a kernel bug fixed in 3.12+ that sometimes hangs us forever on exit. We're not quite sure how it gets triggered...
16.3


Question: exit 1 fi <ins> # mark all ./test/integration/* targets as integration # see comment above re: buildozer exit codes buildozer -quiet 'add tags integration' '//test/integration/...:%go_test' && ret=$? || ret=$? if [[ $ret != 0 && $ret != 3 ]]; then exit 1 fi </ins> # we need to set this because gazelle doesn't support pkg-config, which would set this link option # see comment above re: buildozer exit codes buildozer -quiet 'set clinkopts select({"@io_bazel_rules_go//go/platform:linux":["-lseccomp",],"//conditions:default":[],})' //vendor/github.com/seccomp/libseccomp-golang:go_default_library && ret=$? || ret=$?

What happened: when running integration tags were missing on files What you expected to happen: add the needed tags on test/integration files How to reproduce it (as minimally and precisely as possible): add an integration test package under cc
/sig testing /priority important-soon /area build-release
/cc
if we want to assume that all tests in a certain directory are integration tests this is very easy to automate, here's an example that we do now to mark go tests and binaries as
Is the assumption that all tests in should be tagged as integration? I'd be happy to add this to the script, but would prefer to get in without conflicts first
/assign /lifecycle active


Question: wantStatus: framework.NewStatus(framework.Unschedulable, getErrReason(v1.ResourceEphemeralStorage)), }, { <ins> pod: newResourceInitPod(newResourcePod(framework.Resource{EphemeralStorage: 25}), framework.Resource{EphemeralStorage: 25}), nodeInfo: framework.NewNodeInfo( newResourcePod(framework.Resource{MilliCPU: 2, Memory: 2})), name: "ephemeral local storage request is ignored due to disabled feature gate", features: map[featuregate.Feature]bool{ features.LocalStorageCapacityIsolation: false, }, }, { </ins> pod: newResourcePod(framework.Resource{EphemeralStorage: 10}), nodeInfo: framework.NewNodeInfo( newResourcePod(framework.Resource{MilliCPU: 2, Memory: 2})),

What happened: In a v1.18 cluster with the LocalStorageCapacityIsolation feature gate disabled setting an ephemeral-storage request or limit causes the pod to be unschedulable: If the ephemeral-storage request is removed the pod is scheduled. This behavior exists in v1.19 and a recent v1.20 build. What you expected to happen: In previous releases such pods are scheduled. I expect that any ephemeral-storage limit is not enforced. How to reproduce it (as minimally and precisely as possible): Disable the LocalStorageCapacityIsolation feature guide by adding to the kube-scheduler. Create a pod like this: Anything else we need to know?: The problem was originally seen after moving from IBM Cloud Red Hat OpenShift v4.4 to v4.5. Pods that worked in 4.4 (Kubernetes v1.17) remained in a Pending state in OpenShift v4.5 (Kubernetes 1.18). We discovered that the IBM Cloud OpenShift service has the LocalStorageCapacityIsolation feature-gate disabled while Red Hat has enabled the feature. We then replicated the behavior with a plain Kubernetes cluster. Environment: Kubernetes version (use ): v1.18.10, v1.19.3, v1.20.0 Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig sig-scheduling
The label(s) cannot be applied, because the repository doesn't have them <detailsIn response to : /sig sig-scheduling Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig scheduling
I can confirm it's a bug. Will work on it.


Question: wantStatus: framework.NewStatus(framework.Unschedulable, getErrReason(v1.ResourceEphemeralStorage)), }, { <ins> pod: newResourceInitPod(newResourcePod(framework.Resource{EphemeralStorage: 25}), framework.Resource{EphemeralStorage: 25}), nodeInfo: framework.NewNodeInfo( newResourcePod(framework.Resource{MilliCPU: 2, Memory: 2})), name: "ephemeral local storage request is ignored due to disabled feature gate", features: map[featuregate.Feature]bool{ features.LocalStorageCapacityIsolation: false, }, }, { </ins> pod: newResourcePod(framework.Resource{EphemeralStorage: 10}), nodeInfo: framework.NewNodeInfo( newResourcePod(framework.Resource{MilliCPU: 2, Memory: 2})),

/assign
Clarifying the recreate steps: I disabled the LocalStorageCapacityIsolation feature guide by adding to the kube-scheduler since this behavior seems to be within the scheduler.
Thanks for confirming!
This bug has been fixed in master branch, and will be backported to [1.18] and [1.19] soon. [1.18]: [1.19]:


Question: // hoping external CCM or admin can set it. Returning // default values from here will mean, no one can // override them. <del> glog.Errorf("failed to get azure cloud in GetVolumeLimits, plugin.host: %s", plugin.host.GetHostName()) return volumeLimits, nil </del> <ins> return nil, fmt.Errorf("failed to get azure cloud in GetVolumeLimits, plugin.host: %s", plugin.host.GetHostName()) </ins> } instances, ok := az.Instances()

Is this a BUG REPORT or FEATURE REQUEST?: What happened: After upgrade of Kubernetes 1.11.3 to 1.12.1 via kubeadm in bare-metal setup I see these errors in kubelet logs: What you expected to happen: Since this is a bare-metal setup I expect no messages concerning Azure cloud How to reproduce it (as minimally and precisely as possible): Do an upgrade from 1.11.3 to 1.12.1 Anything else we need to know?: kubelet-1.12.1- was downloaded from Environment: Kubernetes version (use ): Client Version: version.Info{Major:"1", Minor:"12", GitVersion:"v1.12.1", GitCommit:"", GitTreeState:"clean", BuildDate:"2018-10-05T16:46:06Z", GoVersion:"go1.10.4", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"12", GitVersion:"v1.12.1", GitCommit:"", GitTreeState:"clean", BuildDate:"2018-10-05T16:36:14Z", GoVersion:"go1.10.4", Compiler:"gc", Platform:"linux/amd64"} Cloud provider or hardware configuration: bare-metal OS (e.g. from /etc/os-release): Oracle Linux Server 7.4 Kernel (e.g. ): 62 Install tools: Others:
Lines from kubelet log which say that no cloud provider was specified (that is a bare-metal setup): ..
/sig azure
have the same issue related to the azure.. also a bare-metal cluster.. these messages are flooding my centralized logging elasticsearch (though they aren't really problematic more than being an annoyance). i also upgraded from 1.11.3 to 1.12.1 on a 3-master HA set up.


Question: // hoping external CCM or admin can set it. Returning // default values from here will mean, no one can // override them. <del> glog.Errorf("failed to get azure cloud in GetVolumeLimits, plugin.host: %s", plugin.host.GetHostName()) return volumeLimits, nil </del> <ins> return nil, fmt.Errorf("failed to get azure cloud in GetVolumeLimits, plugin.host: %s", plugin.host.GetHostName()) </ins> } instances, ok := az.Instances()

i also noted (perhaps worthy of a separate issue) that the two of my three control-planes that are showing the azure error also previously had been having cert authority errors from the api-server container, when having used kubeadm upgrade.. (on the second two control plane systems, with the issue) (and on the first control without the issue) (and i needed to kubeadm reset and then kubeadm init all over again with 1.12.1 on the 2nd two control-plane nodes).. the first control-plane node that didn't need to do a reset/init didn't have the azure errors (nor the cert errors). so i kind of suspect it might not be an kubeadm upgrade issue but a general issue going from 1.11.3 to 1.12.1 ?
Same problems +1
Same problem on brand new bm 1.12.1 cluster built with kubeadm and latest 1alpha3 config based off of customized kubeadm config print-default example
/assign
Same problems +1
Oct 9 11:11:32 localhost kubelet: E1009 11:11:32. 3342 ] failed to get azure cloud in GetVolumeLimits, kubernetes-1 Oct 9 11:11:42 localhost kubelet: E1009 11:11:42. 3342 ] failed to get azure cloud in GetVolumeLimits, kubernetes-1 Oct 9 11:11:52 localhost kubelet: E1009 11:11:52. 3342 ] failed to get azure cloud in GetVolumeLimits, kubernetes-1 Oct 9 11:12:02 localhost kubelet: E1009 11:12:02. 3342 ] failed to get azure cloud in GetVolumeLimits, kubernetes-1 Oct 9 11:12:12 localhost kubelet: E1009 11:12:12. 3342 ] failed to get azure cloud in GetVolumeLimits, kubernetes-1 Oct 9 11:12:23 localhost kubelet: E1009 11:12:23. 3342 ] failed to get azure cloud in GetVolumeLimits, kubernetes-1 Oct 9 11:12:33 localhost kubelet: E1009 11:12:33.


Question: // hoping external CCM or admin can set it. Returning // default values from here will mean, no one can // override them. <del> glog.Errorf("failed to get azure cloud in GetVolumeLimits, plugin.host: %s", plugin.host.GetHostName()) return volumeLimits, nil </del> <ins> return nil, fmt.Errorf("failed to get azure cloud in GetVolumeLimits, plugin.host: %s", plugin.host.GetHostName()) </ins> } instances, ok := az.Instances()

3342 ] failed to get azure cloud in GetVolumeLimits, kubernetes-1 Oct 9 11:12:43 localhost kubelet: E1009 11:12:43. 3342 ] failed to get azure cloud in GetVolumeLimits, kubernetes-1 Oct 9 11:12:53 localhost kubelet: E1009 11:12:53. 3342 ] failed to get azure cloud in GetVolumeLimits, kubernetes-1 Oct 9 11:13:03 localhost kubelet: E1009 11:13:03. 3342 ] failed to get azure cloud in GetVolumeLimits, kubernetes-1
how often did you hit this error log: I am planning to make this log as , is that ok for you? Usually V(2) is the default info(see ) log for k8s on azure I am not sure what the default info log by kubeadmin installation.
It is logged every 10 seconds. I disagree. At least previous incarnation of GetVolumeLimits in version 1.11.3 : would have logged an error "No cloudprovider present". Which I didn't see in kubelet log. In fact, I didn't see any message in kubelet logs with 'azure' substring at all. I think it means that AzureDataDisk plugin was not instantiated at all in 1.11.3. So the question is: why azuredd is instantiated now in 1.12.1 in bare-metal setup (that is without cloud provider)? Do you mind checking this idea? (Besides your method of lowering message priority won't work as I said this is a bare-metal setup where kubelet systemd unit file has no --v option at all )
That's because feature reaches beta in v1.12, in v1.11.3, even the code is there, it won't be invoked since feature is still alpha in v1.11
And in v1.12, aws, gce, and azure disk plugins are all loaded, take aws ebs for example, it will return error directly: And logging as V(4): I suppose V(4) logs won't display in logs. Let me make the same change as aws and gce.
also tries to address this error.


Question: // hoping external CCM or admin can set it. Returning // default values from here will mean, no one can // override them. <del> glog.Errorf("failed to get azure cloud in GetVolumeLimits, plugin.host: %s", plugin.host.GetHostName()) return volumeLimits, nil </del> <ins> return nil, fmt.Errorf("failed to get azure cloud in GetVolumeLimits, plugin.host: %s", plugin.host.GetHostName()) </ins> } instances, ok := az.Instances()

The same mistake, I also encountered NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME 172.16.16.13 Ready <none5m27s v1.12.1 172.16.16.13 <noneCentOS Linux 7 (Core) 4.18.12- docker://18.6.0 failed to get azure cloud in GetVolumeLimits, 172.16.16.13 failed to get azure cloud in GetVolumeLimits, 172.16.16.13 failed to get azure cloud in GetVolumeLimits, 172.16.16.13
I have cherry picked the fix to v1.12
I understand that the feature Node-specific Volume Limits has now reached beta in 1.12, but for those of us running Kubernetes on-premises, this log error is pretty useless. In fact, I would even want to disable this plugin alltogether, since it will never be used on-prem. Is there a way to stop the plugin from loading, or in the very least make sure the log does not appear? Right now, it is polluting the system log file quite a lot.
I second that. That was the key message of my bug report. That is we must not even instantiate cloud-specific plugins if no cloud provider is configured.
I'm having exactly the same problem, doing an installation from scratch on DigitalOcean
+1
+1
Hi I have the same issue. It would be really nice to know how to disable this error message that floods the logs for no reason.
For those using logstash it was sufficient to just temporarily add this to your syslog filter: if [message] =~ "" { drop { } }
After a fresh installation of Raspbian and kubernetes, same issue is occurring in my 4 node cluster.


Question: // hoping external CCM or admin can set it. Returning // default values from here will mean, no one can // override them. <del> glog.Errorf("failed to get azure cloud in GetVolumeLimits, plugin.host: %s", plugin.host.GetHostName()) return volumeLimits, nil </del> <ins> return nil, fmt.Errorf("failed to get azure cloud in GetVolumeLimits, plugin.host: %s", plugin.host.GetHostName()) </ins> } instances, ok := az.Instances()

Client Version: version.Info{Major:"1", Minor:"12", GitVersion:"v1.12.1", GitCommit:"", GitTreeState:"clean", BuildDate:"2018-10-05T16:46:06Z", GoVersion:"go1.10.4", Compiler:"gc", Platform:"linux/arm"} Server Version: version.Info{Major:"1", Minor:"12", GitVersion:"v1.12.1", GitCommit:"", GitTreeState:"clean", BuildDate:"2018-10-05T16:36:14Z", GoVersion:"go1.10.4", Compiler:"gc", Platform:"linux/arm"} flood of this: epsilon kubelet[2412]: E1024 12:25:58. 2412 ] failed to get azure cloud in GetVolumeLimits, epsilon
disable in controller plane should work: "--feature-gates=AttachVolumeLimit=false"
I have tried "--feature-gates=AttachVolumeLimit=false" and it's not working. I still see messages kubelet[]: E1024 22:51:02. ] failed to get azure cloud in GetVolumeLimits,
I have the same problem here.., ] failed to get azure cloud in GetVolumeLimits,
+1 same case here, also tried with --feature-gates=AttachVolumeLimit=false
Will this be disabled by default in the next release?
It's already fixed in v1.12.2 and v1.13.0. do you know how to disable in kubelet in v1.12? Would add in kubelet command arguments work?
I haven't updated to 1.12.2 yet (still on 1.12.1) but just to confirm, disabling that feature works. For those that are also running on CentOS and/or RHEL, just to the following: Edit file on your master server (control plane), and add the following: If you already have extra arguments in there, just add this one at the end of the line. Then restart kubelet on your master server:
Yes that will work.


Question: // hoping external CCM or admin can set it. Returning // default values from here will mean, no one can // override them. <del> glog.Errorf("failed to get azure cloud in GetVolumeLimits, plugin.host: %s", plugin.host.GetHostName()) return volumeLimits, nil </del> <ins> return nil, fmt.Errorf("failed to get azure cloud in GetVolumeLimits, plugin.host: %s", plugin.host.GetHostName()) </ins> } instances, ok := az.Instances()

--feature-gates=AttachVolumeLimit property doesn't exist in the documentation
Update tho 1.12.2 resolve for me Em qui, 1 de nov de 2018 9:22 PM, Abhilash Pallerlamudi < escreveu:
add follow line to "your kubelet.service" file. like this: [root ~]# systemctl daemon-reload [root ~]# sudo systemctl restart kubelet then it's ok
So to get rid of a bug we should disable an entire feature? Says it is supported by Azure-Disk. Why isn't node reporting the limit? It is getting instance metadata okay as all my nodes are tagged with SKU. Being that SKU determines attach limit as well, I do not understand why this cannot be made working? I am on 1.14 in Azure on Ubuntu.
how do you deploy k8s on Azure? To get volume limit, kubelet on agent node needs azure identity, e.g. service principal, usually it's under file
I deployed with kubespray. I have cloudconfig set as such: KUBELETCLOUDPROVIDER="--cloud-provider=azure --cloud-config=/etc/kubernetes/cloudconfig" Service principle is set in cloudconfig file.
I triple checked and location was misspelled on some of the nodes for the cloud provider. Thanks!
the get vm size list call depends on location, so now it works?
Yes it does. Thanks!


Question: Names: []string{"/k8s_POD_foo.new.test_.deadbeef_42"}, ID:    "3876", }, <ins> }, containerDetails: map[string]*docker.Container{ "1876": { State: docker.State{ Running: false, }, ID:      "1876", Created: time.Now(), }, }, expectedRemoved: []string{"1876"}, }, // Only remove non-running containers. { containers: []docker.APIContainers{ </ins> { // pod infra container Names: []string{"/k8s_POD_foo.new.test_.deadbeef_42"}, <del> ID:    "4876", </del> <ins> ID:    "1876", }, { // pod infra container Names: []string{"/k8s_POD_foo.new.test_.deadbeef_42"}, ID:    "2876", </ins> }, { // pod infra container Names: []string{"/k8s_POD_foo.new.test_.deadbeef_42"}, <del> ID:    "5876", </del> <ins> ID:    "3876", </ins> }, { // pod infra container Names: []string{"/k8s_POD_foo.new.test_.deadbeef_42"}, <del> ID:    "6876", </del> <ins> ID:    "4876", </ins> }, }, containerDetails: map[string]*docker.Container{ "1876": { State: docker.State{ <del> Running: false, </del> <ins> Running: true, </ins> }, ID:      "1876", Created: time.Now(), }, <ins> "2876": { State: docker.State{ Running: false, }, ID:      "2876", Created: time.Now(), }, </ins> }, <del> expectedRemoved: []string{"1876"}, </del> <ins> expectedRemoved: []string{"2876"}, </ins> }, <ins> // Less than maxContainerCount doesn't delete any. </ins> { containers: []docker.APIContainers{ {

Successfully ran containers with "onFailure" restartPolicy are restarting after purging oldest docker containers. Kubelet purges/removes oldest containers periodically from docker. After purging, we loose the exit code and history of previously ran containers and restart containers as if they were never run.
Thanks & I can help fix this. Let me know. Thanks
Thanks for reporting Just to make sure I understand the issue: You ran some containers with an "onFailure" restartPolicy. When the Kubelet purged old containers the container was restarted (unexpectedly).
Ah, after reading again I think the issue is that after the containers are purged, when the container exits successfully it get restarted although it shouldn't. Is that correct?
After reading through the code I found a bug in the implementation that is most likely causing this error. Should have a PR soon.
Yes. Correct. After initial container run, it should never be run again, since it already ran to completion. Will wait for your PR.
PR is out. This should fix the issue. Feel free to test if you can :)


Question: package dockershim import ( <ins> "bytes" "errors" </ins> "fmt" <ins> "sync" </ins> "testing" dockertypes "github.com/docker/docker/api/types"

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Conformance test failed: xref What you expected to happen: I expected this conformance test to pass. How to reproduce it (as minimally and precisely as possible): We have configured docker's logging driver to use journald, and then ran sonobouy tests against a k8s 1.16.3 cluster. Anything else we need to know?: Environment: Kubernetes version (use ):Cloud provider or hardware configuration: OS (e.g: ): Red Hat Enterprise Linux Server release 7.7 (Maipo) Kernel (e.g. ): 5.3.11- SMP Tue Nov 12 14:45:59 EST 2019 x8664 x8664 x8664 GNU/Linux Install tools: Network plugin and version (if this is a network-related bug): Others:
There are no sig labels on this issue. Please add a sig label by either: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. See the . The in method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign
/sig cli
/assign
I have a working patch that fixes this, will push a PR shortly.


Question: podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/apis/core/validation" "k8s.io/kubernetes/pkg/features" <del> utilpath "k8s.io/utils/path" </del> ) // Whether AppArmor should be disabled by default.

As mentioned the return value of is not being used any longer and the entire method can be removed. The usage of was removed in Now there is no need to have the function and store it's result at all. /sig node /kind cleanup /good-first-issue /help-wanted KEP xref:
This request has been marked as suitable for new contributors. Please ensure that the issue body includes answers to the following questions: Why are we solving this issue? To address this issue, are there any code changes? If there are code changes, what needs to be done in the code and what places can the assignee treat as reference points? Does this issue have zero to low barrier of entry? How can the assignee reach out to you for help? For more details on the requirements of such an issue, please see and ensure that they are met. If this request no longer meets these requirements, the label can be removed by commenting with the command. <detailsIn response to : As mentioned the return value of is not being used any longer and the entire method can be removed. /kind cleanup /good-first-issue /help-wanted Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/triage accepted
/assign
I am having issues submitting the pull request. I removed that portion of the code under however this is my first time contributing and would like some help submitting this new updated version..
/assign
/assign
/assign
/assign


Question: "k8s.io/kubernetes/pkg/api/legacyscheme" podutil "k8s.io/kubernetes/pkg/api/v1/pod" extensionsinternal "k8s.io/kubernetes/pkg/apis/extensions" <del> "k8s.io/kubernetes/pkg/controller" </del> "k8s.io/kubernetes/pkg/controller/daemon" "k8s.io/kubernetes/pkg/kubectl" "k8s.io/kubernetes/pkg/scheduler/schedulercache"

Please see : https://k8s- fails just after this line: Failure was introduced in: But this was not noticed until it started running in [Conformance] tests. You can see the same problem in other dashboards too: https://k8s- https://k8s-
cc /sig apps
/priority critical-urgent
/kind bug /priority failing-test
/milestone v1.10
Is this just an oversight of not removing the template generation when the API update was done?
/cc
triaged in progress
i have never touched this test before ... so i just threw in a WIP, feel free to use/discard it. there seems to be lot of methods in there that is no longer used, may be we can rip those out too. (see )
xref
[MILESTONENOTIFIER] Milestone Issue Needs Attention Action Required: This issue has not been updated since Mar 5. Please provide an update. Note: This issue is marked as , and must be updated every 1 day during code freeze. Example update: <details<summaryIssue Labels</summary- : Issue will be escalated to these SIGs if needed. : Never automatically move issue out of a release milestone; continually escalate to contributor and SIG through all available channels. : Fixes a bug discovered during the current release. </details<details<summaryHelp</summary<ul<li<a href=""Additional instructions</a</li<li<a href=""Commands for setting labels</a</li</ul</details


Question: permitPlugin.timeoutPermit = false permitPlugin.waitAndRejectPermit = false permitPlugin.waitAndAllowPermit = true <ins> permitPlugin.waitingPod = "waiting-pod" </ins> lowPriority, highPriority := int32(100), int32(300) resourceRequest := v1.ResourceRequirements{Requests: v1.ResourceList{ <ins> v1.ResourceCPU:    *resource.NewMilliQuantity(200, resource.DecimalSI), v1.ResourceMemory: *resource.NewQuantity(200, resource.DecimalSI)}, } preemptorResourceRequest := v1.ResourceRequirements{Requests: v1.ResourceList{ </ins> v1.ResourceCPU:    *resource.NewMilliQuantity(400, resource.DecimalSI), v1.ResourceMemory: *resource.NewQuantity(400, resource.DecimalSI)}, } <del> // First pod will go waiting. </del> <ins> // First pod will go running. runningPod := initPausePod(&pausePodConfig{Name: "running-pod", Namespace: testCtx.NS.Name, Priority: &lowPriority, Resources: &resourceRequest}) runningPod.Spec.TerminationGracePeriodSeconds = new(int64) runningPod, err = createPausePod(testCtx.ClientSet, runningPod) if err != nil { t.Errorf("Error while creating the waiting pod: %v", err) } // Wait until the pod scheduled, then create a preemptor pod to preempt it. wait.Poll(100*time.Millisecond, 30*time.Second, podScheduled(testCtx.ClientSet, runningPod.Name, runningPod.Namespace)) // Second pod will go waiting. </ins> waitingPod := initPausePod(&pausePodConfig{Name: "waiting-pod", Namespace: testCtx.NS.Name, Priority: &lowPriority, Resources: &resourceRequest}) waitingPod.Spec.TerminationGracePeriodSeconds = new(int64) waitingPod, err = createPausePod(testCtx.ClientSet, waitingPod)

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via -- We may delete podS though they may be waiting in Do not delete waiting pods if they are not scheduled. Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig scheduling
/cc WDYT ?
It sounds to me that you prefer to prioritize waiting Pod over other assigned Pods. In that case, you may want to exclude waiting Pods in the earlier Filtering/Sorting Victims step, instead of simply disregard the deletion in PrepareCandidate(). In general, when Permit plugins work along with PostFilter (preemption), the logic could be complicated. There may not be a general preemption implementation (probably in the future we can exercise pluginArgs to provide different options). So for now the best bet is to implement your own PostFilter plugin.
Got it! But for the default preemption plugin, we delete the victims to free resources. If pod has assigned, we should delete it. If pod has not assigned, we just need reject it in WaitingPod because the pod just exists in the cache.
Oh, your point was to verify if it's a waiting pod first, and then delete it? If so, then it makes sense to me. Feel free to raise a PR. /assign
Yes, I will open a PR to do this


Question: <ins> /* Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package create import ( "testing" batchv1 "k8s.io/api/batch/v1" batchv1beta1 "k8s.io/api/batch/v1beta1" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestCreateCronJob(t *testing.T) { cronjobName := "test-job" tests := map[string]struct { image    string command  []string schedule string restart  string expected *batchv1beta1.CronJob }{ "just image and OnFailure restart policy": { image:    "busybox", schedule: "0/5 * * * ?", restart:  "OnFailure", expected: &batchv1beta1.CronJob{ TypeMeta: metav1.TypeMeta{APIVersion: batchv1beta1.SchemeGroupVersion.String(), Kind: "CronJob"}, ObjectMeta: metav1.ObjectMeta{ Name: cronjobName, }, Spec: batchv1beta1.CronJobSpec{ Schedule: "0/5 * * * ?", JobTemplate: batchv1beta1.JobTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: cronjobName, }, Spec: batchv1.JobSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ { Name:  cronjobName, Image: "busybox", }, }, RestartPolicy: corev1.RestartPolicyOnFailure, }, }, }, }, }, }, }, "image, command , schedule and Never restart policy": { image:    "busybox", command:  []string{"date"}, schedule: "0/5 * * * ?", restart:  "Never", expected: &batchv1beta1.CronJob{ TypeMeta: metav1.TypeMeta{APIVersion: batchv1beta1.SchemeGroupVersion.String(), Kind: "CronJob"}, ObjectMeta: metav1.ObjectMeta{ Name: cronjobName, }, Spec: batchv1beta1.CronJobSpec{ Schedule: "0/5 * * * ?", JobTemplate: batchv1beta1.JobTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: cronjobName, }, Spec: batchv1.JobSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ Containers: []corev1.Container{ { Name:    cronjobName, Image:   "busybox", Command: []string{"date"}, }, }, RestartPolicy: corev1.RestartPolicyNever, }, }, }, }, }, }, }, } for name, tc := range tests { t.Run(name, func(t *testing.T) { o := &CreateCronJobOptions{ Name:     cronjobName, Image:    tc.image, Command:  tc.command, Schedule: tc.schedule, Restart:  tc.restart, } cronjob := o.createCronJob() if !apiequality.Semantic.DeepEqual(cronjob, tc.expected) { t.Errorf("expected:n%#vngot:n%#v", tc.expected, cronjob) } }) } } </ins>

<!-- Please only use this template for submitting enhancement requests --What would you like to be : Why is this needed: Because of is not recommended to create cronjob (with flags) anymore. But there is no imperative command available for to create cronjob. /sig CLI <!-- DO NOT EDIT BELOW THIS LINE --/kind feature
/assign
GitHub didn't allow me to assign the following users: Pingan2017. Note that only and repo collaborators can be assigned. For more information please see <detailsIn response to : /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
i wanna work on this. give me several days
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale


Question: } else { c.ContainerStatus.State = runtimeapi.ContainerState_CONTAINER_RUNNING } <del> c.ContainerStatus.Id = string(uuid.NewUUID()) </del> <ins> c.ContainerStatus.Id = strings.ReplaceAll(string(uuid.NewUUID()), "-", "") </ins> return c }

Since upgrading from Kubernetes to we no longer get memory metrics from the endpoint on . Example: In our previous version, we would get a response like the following that included utilisation metrics: Hit the endpoint of on a cluster running version . Kubernetes version (use ): Cloud provider or hardware configuration: Provisioned on AWS with OS (e.g: ): Flatcar Container Linux by Kinvolk 2765.2.6 (Oklo) Kernel (e.g. ): 5.10.43-flatcar
/sig node
Hi AlexLastwhich container runtimes are you using? containerd?
Hi AlexLast, I have test on v1.22 with containerd, I did not reproduce this issue. If memory limit is set in the container spec of the pod, /stats/summary will report availableBytes, if memory limit is not set, /stats/summary will not be reported. just like this: ! ! code is here:
/cc We have reports of missing memory metrics on 1.20.8 and 1.21.2 now... was there a cadvisor or similar change that happened?
also /cc as an FYI
If you are using cri-o, it looks like a similar issue to and (possibly an issue with cadvisor/the way cri-o integrates with cadvisor). This would also apply to the dockershim
I'm not able to repro on GKE v1.20.8-gke.700 on containerd
Can you please provide more info about your environment, container runtime etc and possible repro steps? Thanks!
/triage needs-information
Hi, First of all, I am sorry to cut in on you. I have the same issue with Kubelet, in my case with Kubeadm. Here you are the issue: Here you are my installation: Thanks in advance
^ quick summary of the above, appears to be single-node ubuntu (not sure what kernel?) containerd 1.4.6 k8s 1.21.2 kubeadm installer
It is one node. It is only for making easier where is the problem .


Question: } else { c.ContainerStatus.State = runtimeapi.ContainerState_CONTAINER_RUNNING } <del> c.ContainerStatus.Id = string(uuid.NewUUID()) </del> <ins> c.ContainerStatus.Id = strings.ReplaceAll(string(uuid.NewUUID()), "-", "") </ins> return c }

Kubernetes version: 1.21.2 Cloud provider or hardware configuration: Bare metal (VMware) OS: Ubuntu 18.04.5 LTS Kernel: 4.15.0-147-generic Install tools: Kubeadm Network plugin and version (if this is a network-related bug): Weave 2.8.1 CRI: Containerd 1.4.6
Sure, we're setting the following flags for : Version: We deploy our clusters using , these missing metrics are for all pods in the cluster, including pods with memory requests and limits set.
Hitting the same issue on such kind of environments Then I tested and found no issue on environment
Besides memory metrics, we noticed a loss of network in pods. I commented at the block of container and the block of pod The loss of pods occurs at the same time as loss of pods
I created cluster as follows on 1.21.2 with kubetest containerd 1.4.6 on ubuntu. This is on older version of ubuntu 18.04 LTS on kernel I don't see memory metrics being lost here.
Can we have more details on exact kernel/OS image being used here? I can't seem to repro this on 1.21.2 with same containerd/ ubuntu version as mentioned in
Hi, may I know your output of network metrics in pods?
Hi, I have tested with two kernel versions: P.S: I have just updated my in order to make easier the test. You can use it (it is necessary to install and ). Thanks in advance
We tested with two kernel versions too.
I was able to repro finally -- it looks like missing key here is that you must be using the systemd cgroup driver. With this patch and starting cluster as follows:
Hi do you mean that if using systemd cgroup driver, this issue would happen. In my environment, both containerd and kubelet are using cgroupfs driver, but the issue still exists containerd config dump
Hi, Here you are my Containerd & Kubelet config: ` As far as I know, systemd cgroup driver should be configured in order to have a proper perfomance in Kubernetes. Is it a bug with Kubelet?


Question: } else { c.ContainerStatus.State = runtimeapi.ContainerState_CONTAINER_RUNNING } <del> c.ContainerStatus.Id = string(uuid.NewUUID()) </del> <ins> c.ContainerStatus.Id = strings.ReplaceAll(string(uuid.NewUUID()), "-", "") </ins> return c }

Thanks in advance
Hi thanks for the response, I've found the following in the , so will have a go at setting that parameter
Thanks for all the info! I was only able to repro this on systemd cgroup driver as mentioned in It did not repro for me when using cgroupfs. This will need some additional investigation to figure out the root cause and why memory metrics are not being reported correctly and see if this is a kubelet or cAdvisor issue. /cc
Hi, Thanks . By the way, there is a new issue reporting the same behaviour:
After setting kubelet's log verbosity level to 10, I found logs like Is this related? And I checked that according the log, all the pod sandboxID are missing
Thanks I was able to confirm that switching to fixed the missing metrics. From a bit of research, it's my understanding that using the driver on a node with is not recommended - can you confirm?
By stepping into the code, I found that the container info returned by this func is nil, so that the will always be false. Thus will always be . That explains the last log has expected fields. But because of 1), this func cannot be called. There's really a label-related issue on cadvisor: (Thanks With the workaround, I can recover the missed metrics
/remove-triage needs-information /triage accepted
Thanks for digging deeper into this I'm still a bit confused on the relationship between this and systemd cgroup driver. Do the labels being missing have something to do with systemd cgroup driver being used? Also are you using a different socket path for containerd i.e. not the default as mentioned in ?
We didn't use the default but used and not specified
Interesting! In the test I made in the container-runtime-endpoint is set to default (i.e. ).
With debugging, I found that on kubernetes 1.21.2 when using different cgroup driver, the cgroupPath format of containers are different. : systemd + containerd:cgroupfs : systemd + containerd: systemd : cgroupfs + containerd: cgroupfs The would return a map using the path.Base of cgroupPath Later, we use containerID to check to add metrics. So only in the third case, the key of are . That's how the cgroup driver impacts the metrics.


Question: } else { c.ContainerStatus.State = runtimeapi.ContainerState_CONTAINER_RUNNING } <del> c.ContainerStatus.Id = string(uuid.NewUUID()) </del> <ins> c.ContainerStatus.Id = strings.ReplaceAll(string(uuid.NewUUID()), "-", "") </ins> return c }

To sum up, and are two different cases that caused this issue. For the case, I think it's a bug we need fix: we should make sure the key of always be BTW do you know why the cgroupPath format are different when using different cgroup driver?
I believe it's expected that the cgroup naming will differ when using systemd cgroup driver vs cgroupfs. Kubelet internally has some logic for parsing and converting between the two different formats, e.g. and I left some comments on your proposed PR: Thanks for helping look into this issue!
/priority important-soon


Question: <ins> #!/bin/bash # Copyright 2015 The Kubernetes Authors All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Source this script in the Jenkins "Execute shell" build action to have all # test artifacts and the console log uploaded at the end of the test run. # For example, you might use the following line as the first command: # mkdir -p _tmp/ && curl -fsS --retry 3 -o _tmp/upload-to-gcs.sh  #   "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh"  #   && source _tmp/upload-to-gcs.sh # Note that this script requires the Jenkins shell binary to be set to bash, not # the system default (which may be dash on Debian-based systems). # Also, for magicfile support to work correctly, the "file" utility must be # installed. set -o errexit set -o nounset set -o pipefail set -o xtrace if [[ ${JOB_NAME} =~ -pull- ]]; then : ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/pr-logs/${ghprbActualCommit:-unknown}"} else : ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/logs"} fi : ${JENKINS_UPLOAD_TO_GCS:="y"} # Upload the build log and all test artifacts (under _artifacts) to GCS when # JENKINS_UPLOAD_TO_GCS is set to y. # We intentionally ignore gsutil errors since we don't want failed uploads to # fail the entire test run (#13548). function upload_logs_to_gcs() { if [[ ! ${JENKINS_UPLOAD_TO_GCS:-} =~ ^[yY]$ ]]; then return fi local -r artifacts_path="${WORKSPACE}/_artifacts" local -r gcs_job_path="${JENKINS_GCS_LOGS_PATH}/${JOB_NAME}" local -r gcs_build_path="${gcs_job_path}/${BUILD_NUMBER}" local -r gcs_acl="public-read" for upload_attempt in $(seq 3); do echo "Uploading to ${gcs_build_path} (attempt ${upload_attempt})" if [[ -d "${artifacts_path}" ]]; then gsutil -m -q -o "GSUtil:use_magicfile=True" cp -a "${gcs_acl}" -r -c  -z log,txt,xml "${artifacts_path}" "${gcs_build_path}/artifacts" || continue fi # Remove ANSI escape sequences from the console log before uploading. local -r filtered_console_log="${WORKSPACE}/console-log.txt" sed -r "s/x1B[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g"  "${JENKINS_HOME}/jobs/${JOB_NAME}/builds/${BUILD_NUMBER}/log"  > "${filtered_console_log}" gsutil -q cp -a "${gcs_acl}" -z txt "${filtered_console_log}" "${gcs_build_path}/" || continue # Mark this build as the latest completed. { echo "BUILD_NUMBER=${BUILD_NUMBER}" echo "GIT_COMMIT=${GIT_COMMIT:-}" } | gsutil -q -h "Content-Type:text/plain" -h "Cache-Control:private, max-age=0, no-transform"  cp -a "${gcs_acl}" - "${gcs_job_path}/latest-build.txt" || continue break  # all uploads succeeded if we hit this point done } # Automatically upload logs to GCS on exit or timeout. trap upload_logs_to_gcs EXIT SIGTERM SIGINT </ins>

In our scalability tests it sometimes happens that all tests are passing, but the run is marked red as "failure". This is because uploading logs to GCS sometimes failed with error: I've seen 2 examples during last 24 hours: For now it's just misleading. However, since scalability suite is now pretty stable, I would like to add it to list of projects that our merge bot requires to be green to merge anything. Unfortunately this looks like a blocker for it. Any thoughts how to fix that? cc
cc
Another example of it:
Yes, I think this is a good idea. It should be relatively trivial to mask the uploading error from Jenkins (in the script). We do ideally need some sort of warning to alert us if/when the uploads are failing repeatedly (which could be done in a separate PR). Ideally it would email the oncall, or automatically (re-)open and assign a github ticket. do you have time to look at this next week? Anyone else? If not, I will. Tentatively assigning to myself, although hoping that someone else will pick it up.
We're using a Jenkins plugin, not the e2e scripts, to upload build results to GCS. There's no way that I can find to make it not fail the build if an upload fails. Side note: the actual error message is a 401 error from GCS which seems to suggest an actual issue in the underlying infrastructure somewhere, which probably shouldn't be overlooked, but I have no idea where to start to try to figure which team to notify.
That said, it should be pretty straightforward to write a script to upload the artifacts and build log, rather than relying on the plugin.
Yes - that would be great if you could do it...
Just bumping it up - it happened again few times over the weekend and also today:
I looked at the existing plugin that we're using, and it does indeed retry (5 times!) Also searched the logs and found a lot of GCS errors: $ grep GoogleJsonResponseException: uniq -c 50 401 Unauthorized 22 500 Internal Server Error 12 503 Service Unavailable So I don't really think that there is much more that we can do here other than help GCS to track down their errors, and increase the retry count (although 5 with exponential backoff already seems rather excessive).


Question: <ins> #!/bin/bash # Copyright 2015 The Kubernetes Authors All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Source this script in the Jenkins "Execute shell" build action to have all # test artifacts and the console log uploaded at the end of the test run. # For example, you might use the following line as the first command: # mkdir -p _tmp/ && curl -fsS --retry 3 -o _tmp/upload-to-gcs.sh  #   "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/upload-to-gcs.sh"  #   && source _tmp/upload-to-gcs.sh # Note that this script requires the Jenkins shell binary to be set to bash, not # the system default (which may be dash on Debian-based systems). # Also, for magicfile support to work correctly, the "file" utility must be # installed. set -o errexit set -o nounset set -o pipefail set -o xtrace if [[ ${JOB_NAME} =~ -pull- ]]; then : ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/pr-logs/${ghprbActualCommit:-unknown}"} else : ${JENKINS_GCS_LOGS_PATH:="gs://kubernetes-jenkins/logs"} fi : ${JENKINS_UPLOAD_TO_GCS:="y"} # Upload the build log and all test artifacts (under _artifacts) to GCS when # JENKINS_UPLOAD_TO_GCS is set to y. # We intentionally ignore gsutil errors since we don't want failed uploads to # fail the entire test run (#13548). function upload_logs_to_gcs() { if [[ ! ${JENKINS_UPLOAD_TO_GCS:-} =~ ^[yY]$ ]]; then return fi local -r artifacts_path="${WORKSPACE}/_artifacts" local -r gcs_job_path="${JENKINS_GCS_LOGS_PATH}/${JOB_NAME}" local -r gcs_build_path="${gcs_job_path}/${BUILD_NUMBER}" local -r gcs_acl="public-read" for upload_attempt in $(seq 3); do echo "Uploading to ${gcs_build_path} (attempt ${upload_attempt})" if [[ -d "${artifacts_path}" ]]; then gsutil -m -q -o "GSUtil:use_magicfile=True" cp -a "${gcs_acl}" -r -c  -z log,txt,xml "${artifacts_path}" "${gcs_build_path}/artifacts" || continue fi # Remove ANSI escape sequences from the console log before uploading. local -r filtered_console_log="${WORKSPACE}/console-log.txt" sed -r "s/x1B[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g"  "${JENKINS_HOME}/jobs/${JOB_NAME}/builds/${BUILD_NUMBER}/log"  > "${filtered_console_log}" gsutil -q cp -a "${gcs_acl}" -z txt "${filtered_console_log}" "${gcs_build_path}/" || continue # Mark this build as the latest completed. { echo "BUILD_NUMBER=${BUILD_NUMBER}" echo "GIT_COMMIT=${GIT_COMMIT:-}" } | gsutil -q -h "Content-Type:text/plain" -h "Cache-Control:private, max-age=0, no-transform"  cp -a "${gcs_acl}" - "${gcs_job_path}/latest-build.txt" || continue break  # all uploads succeeded if we hit this point done } # Automatically upload logs to GCS on exit or timeout. trap upload_logs_to_gcs EXIT SIGTERM SIGINT </ins>

Thanks for looking into it and ! Is it easy to increase the retry count? Can we do it (I agree that 5 doesn't seem low, but we still observe errors). Did you talk to GCS people? I wouldn't expect the number of errors to be such big...
No, the retry count is unfortunately not configurable. I have not yet spoke to the GCS people, but plan to.
^
did we ever follow up with the GCS folks? I'm working on a PR to upload test artifacts/logs all within itself.
sorry, no.
- why did you close this? IIUC the PR is still not merged.
Yes, sorry, you're right. I'll assign this one to to close once the PR is merged.
I enabled the new script for -scalability, but it will take a few hours to verify that it works as intended. I haven't disabled the plugin yet, but assuming everything works as intended, I'll do that during the day PDT Friday.
- thanks!
- this doesn't work at all - in fact the cluster doesn't even start after this change
- I reverted your change Can you please experiment on something different than -scalability - if this doesn't work it blocks merge bot for ~2.5 hours which is how long a single run lasts. Can you please experiment on something that is not blocking merge-bot first? FYI:
Ugh, sorry about that. It'd been working fine on a number of builds, but all of those cleaned their workspaces before running tests. should fix the issue, and I'll test it on one of the flaky builds first.


Question: }, shouldFail: false, }, <del> // TODO test rescheduling on PVC add/update </del> <ins> "reschedule on delay binding PVC add": { pod: makePod("pod-reschedule-onpvcadd", config.ns, []string{"pvc-reschedule-onpvcadd"}), pvs: []*testPV{ { name:   "pv-reschedule-onpvcadd", scMode: modeWait, node:   node1, }, }, trigger: func(config *testConfig) { pvc := makePVC("pvc-reschedule-onpvcadd", config.ns, &classWait, "") if _, err := config.client.CoreV1().PersistentVolumeClaims(config.ns).Create(pvc); err != nil { t.Fatalf("Failed to create PersistentVolumeClaim %q: %v", pvc.Name, err) } }, shouldFail: false, }, </ins> } for name, test := range cases {

Add test for "on PVC add", see Test for "on PVC update" is not needed, because it has already been tested in all volume scheduling tests.
/sig
docs:


Question: package dryrun <ins> import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" ) </ins> // IsDryRun returns true if the DryRun flag is an actual dry-run. func IsDryRun(flag []string) bool { return len(flag) > 0 } <ins> // ResetMetadata resets metadata fields that are not allowed to be set by dry-run. func ResetMetadata(originalObj, newObj runtime.Object) error { originalObjMeta, err := meta.Accessor(originalObj) if err != nil { return errors.NewInternalError(err) } newObjMeta, err := meta.Accessor(newObj) if err != nil { return errors.NewInternalError(err) } // If a resource is created with dry-run enabled where generateName is set, the // store will set the name to the generated name. We need to reset the name and restore // the generateName metadata fields in order for the returned object to match the intent // of the original template. if originalObjMeta.GetGenerateName() != "" { newObjMeta.SetName("") } newObjMeta.SetGenerateName(originalObjMeta.GetGenerateName()) // If UID is set in the dry-run output then that output cannot be used to create a resource. Reset // the UID to allow the output to be used to create resources. newObjMeta.SetUID("") // If the resourceVersion is set in the dry-run output then that output cannot be used to create // a resource. Reset the resourceVersion to allow the output to be used to create resources. newObjMeta.SetResourceVersion("") return nil } </ins>

Running kubectl create with dry-run returns unwanted uid metadata field, as well as a generated name when using generateName. This is inconsistent with the expectations as defined in which states I expected uid, and a generated name to be empty when using the dry-run create option. Expected metadata: Actual metadata: No response <detailsI've tested this from 1.19.0 through 1.23.0 </details<details</details<details</details<details</details<details</details<details</details
/sig api-machinery
/assign /triage accepted
reopening, is reverted in to make sure requests with both name+generateName are handled correctly
The Kubernetes project currently lacks enough contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Mark this issue or PR as rotten with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle stale [1]:
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle rotten [1]:
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Reopen this issue or PR with Mark this issue or PR as fresh with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at .