Question: <ins> // +build linux /* Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package mount import ( "k8s.io/utils/exec" fakeexec "k8s.io/utils/exec/testing" "testing" ) func TestGetFileSystemSize(t *testing.T) { cmdOutputSuccessXfs := ` statfs.f_bsize = 4096 statfs.f_blocks = 1832448 statfs.f_bavail = 1822366 statfs.f_files = 3670016 statfs.f_ffree = 3670012 statfs.f_flags = 0x1020 geom.bsize = 4096 geom.agcount = 4 geom.agblocks = 458752 geom.datablocks = 1835008 geom.rtblocks = 0 geom.rtextents = 0 geom.rtextsize = 1 geom.sunit = 0 geom.swidth = 0 counts.freedata = 1822372 counts.freertx = 0 counts.freeino = 61 counts.allocino = 64 ` cmdOutputNoDataXfs := ` statfs.f_bsize = 4096 statfs.f_blocks = 1832448 statfs.f_bavail = 1822366 statfs.f_files = 3670016 statfs.f_ffree = 3670012 statfs.f_flags = 0x1020 geom.agcount = 4 geom.agblocks = 458752 geom.rtblocks = 0 geom.rtextents = 0 geom.rtextsize = 1 geom.sunit = 0 geom.swidth = 0 counts.freedata = 1822372 counts.freertx = 0 counts.freeino = 61 counts.allocino = 64 ` cmdOutputSuccessExt4 := ` Filesystem volume name:   cloudimg-rootfs Last mounted on:          / Filesystem UUID:          testUUID Filesystem magic number:  0xEF53 Filesystem revision #:    1 (dynamic) Filesystem features:      has_journal ext_attr resize_inode dir_index filetype needs_recovery extent 64bit Default mount options:    user_xattr acl Filesystem state:         clean Errors behavior:          Continue Filesystem OS type:       Linux Inode count:              3840000 Block count:              5242880 Reserved block count:     0 Free blocks:              5514413 Free inodes:              3677492 First block:              0 Block size:               4096 Fragment size:            4096 Group descriptor size:    64 Reserved GDT blocks:      252 Blocks per group:         32768 Fragments per group:      32768 Inodes per group:         16000 Inode blocks per group:   1000 Flex block group size:    16 Mount count:              2 Maximum mount count:      -1 Check interval:           0 (<none>) Lifetime writes:          180 GB Reserved blocks uid:      0 (user root) Reserved blocks gid:      0 (group root) First inode:              11 Inode size:	              256 Required extra isize:     32 Desired extra isize:      32 Journal inode:            8 Default directory hash:   half_md4 Directory Hash Seed:      Test Hashing Journal backup:           inode blocks Checksum type:            crc32c Checksum:                 0x57705f62 Journal features:         journal_incompat_revoke journal_64bit journal_checksum_v3 Journal size:             64M Journal length:           16384 Journal sequence:         0x00037109 Journal start:            1 Journal checksum type:    crc32c Journal checksum:         0xb7df3c6e ` cmdOutputNoDataExt4 := `Filesystem volume name:   cloudimg-rootfs Last mounted on:          / Filesystem UUID:          testUUID Filesystem magic number:  0xEF53 Filesystem revision #:    1 (dynamic) Filesystem features:      has_journal ext_attr resize_inode dir_index filetype needs_recovery extent 64bit Default mount options:    user_xattr acl Filesystem state:         clean Errors behavior:          Continue Filesystem OS type:       Linux Inode count:              3840000 Reserved block count:     0 Free blocks:              5514413 Free inodes:              3677492 First block:              0 Fragment size:            4096 Group descriptor size:    64 Reserved GDT blocks:      252 Blocks per group:         32768 Fragments per group:      32768 Inodes per group:         16000 Inode blocks per group:   1000 Flex block group size:    16 Mount count:              2 Maximum mount count:      -1 Check interval:           0 (<none>) Lifetime writes:          180 GB Reserved blocks uid:      0 (user root) Reserved blocks gid:      0 (group root) First inode:              11 Inode size:	              256 Required extra isize:     32 Desired extra isize:      32 Journal inode:            8 Default directory hash:   half_md4 Directory Hash Seed:      Test Hashing Journal backup:           inode blocks Checksum type:            crc32c Checksum:                 0x57705f62 Journal features:         journal_incompat_revoke journal_64bit journal_checksum_v3 Journal size:             64M Journal length:           16384 Journal sequence:         0x00037109 Journal start:            1 Journal checksum type:    crc32c Journal checksum:         0xb7df3c6e ` testcases := []struct { name        string devicePath  string blocksize   uint64 blockCount  uint64 cmdOutput   string expectError bool fsType      string }{ { name:        "success parse xfs info", devicePath:  "/dev/test1", blocksize:   4096, blockCount:  1835008, cmdOutput:   cmdOutputSuccessXfs, expectError: false, fsType:      "xfs", }, { name:        "block size not present - xfs", devicePath:  "/dev/test1", blocksize:   0, blockCount:  0, cmdOutput:   cmdOutputNoDataXfs, expectError: true, fsType:      "xfs", }, { name:        "success parse ext info", devicePath:  "/dev/test1", blocksize:   4096, blockCount:  5242880, cmdOutput:   cmdOutputSuccessExt4, expectError: false, fsType:      "ext4", }, { name:        "block size not present - ext4", devicePath:  "/dev/test1", blocksize:   0, blockCount:  0, cmdOutput:   cmdOutputNoDataExt4, expectError: true, fsType:      "ext4", }, } for _, test := range testcases { t.Run(test.name, func(t *testing.T) { fcmd := fakeexec.FakeCmd{ CombinedOutputScript: []fakeexec.FakeAction{ func() ([]byte, []byte, error) { return []byte(test.cmdOutput), nil, nil }, }, } fexec := fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } resizefs := ResizeFs{exec: &fexec} var blockSize uint64 var fsSize uint64 var err error switch test.fsType { case "xfs": blockSize, fsSize, err = resizefs.getXFSSize(test.devicePath) case "ext4": blockSize, fsSize, err = resizefs.getExtSize(test.devicePath) } if blockSize != test.blocksize { t.Fatalf("Parse wrong block size value, expect %d, but got %d", test.blocksize, blockSize) } if fsSize != test.blocksize*test.blockCount { t.Fatalf("Parse wrong fs size value, expect %d, but got %d", test.blocksize*test.blockCount, fsSize) } if !test.expectError && err != nil { t.Fatalf("Expect no error but got %v", err) } }) } } func TestNeedResize(t *testing.T) { testcases := []struct { name            string devicePath      string deviceMountPath string deviceSize      string cmdOutputFsType string expectError     bool expectResult    bool }{ { name:            "False - Unsupported fs type", devicePath:      "/dev/test1", deviceMountPath: "/mnt/test1", deviceSize:      "2048", cmdOutputFsType: "TYPE=ntfs", expectError:     true, expectResult:    false, }, } for _, test := range testcases { t.Run(test.name, func(t *testing.T) { fcmd := fakeexec.FakeCmd{ CombinedOutputScript: []fakeexec.FakeAction{ func() ([]byte, []byte, error) { return []byte(test.deviceSize), nil, nil }, func() ([]byte, []byte, error) { return []byte(test.cmdOutputFsType), nil, nil }, }, } fexec := fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } resizefs := ResizeFs{exec: &fexec} needResize, err := resizefs.NeedResize(test.devicePath, test.deviceMountPath) if needResize != test.expectResult { t.Fatalf("Expect result is %v but got %v", test.expectResult, needResize) } if !test.expectError && err != nil { t.Fatalf("Expect no error but got %v", err) } }) } } </ins>

Then we just let the rest of the normal resize flow continue. No need to have provisioner set the conditions.
we agreed to set conditions too because we want to be as close as possible to external-resizer. If there is some tooling that depends on condition for resizing pending on node, we don't want to break any such tooling. currently kubelet does not depend on condition for node expansion though.
Why can't we let resize controller set the condition when it sees pvc size != pv size?
hmm good point. I think that should work but we may have to verify that flow.
I think we can do that, but only with a change to the CSI spec wording, because the current implication is that should not be necessary.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten
can you summarize the current conclusion/state of this? /remove-lifecycle rotten
Updated the issue with TODO list.
should this issue stay open to keep track of remaining issues?
/reopen
Reopened this issue. <detailsIn response to : /reopen </details
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle stale
/remove-lifecycle stale
/triage accepted
The Kubernetes project currently lacks enough contributors to adequately respond to all issues and PRs.


Question: <ins> // +build linux /* Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package mount import ( "k8s.io/utils/exec" fakeexec "k8s.io/utils/exec/testing" "testing" ) func TestGetFileSystemSize(t *testing.T) { cmdOutputSuccessXfs := ` statfs.f_bsize = 4096 statfs.f_blocks = 1832448 statfs.f_bavail = 1822366 statfs.f_files = 3670016 statfs.f_ffree = 3670012 statfs.f_flags = 0x1020 geom.bsize = 4096 geom.agcount = 4 geom.agblocks = 458752 geom.datablocks = 1835008 geom.rtblocks = 0 geom.rtextents = 0 geom.rtextsize = 1 geom.sunit = 0 geom.swidth = 0 counts.freedata = 1822372 counts.freertx = 0 counts.freeino = 61 counts.allocino = 64 ` cmdOutputNoDataXfs := ` statfs.f_bsize = 4096 statfs.f_blocks = 1832448 statfs.f_bavail = 1822366 statfs.f_files = 3670016 statfs.f_ffree = 3670012 statfs.f_flags = 0x1020 geom.agcount = 4 geom.agblocks = 458752 geom.rtblocks = 0 geom.rtextents = 0 geom.rtextsize = 1 geom.sunit = 0 geom.swidth = 0 counts.freedata = 1822372 counts.freertx = 0 counts.freeino = 61 counts.allocino = 64 ` cmdOutputSuccessExt4 := ` Filesystem volume name:   cloudimg-rootfs Last mounted on:          / Filesystem UUID:          testUUID Filesystem magic number:  0xEF53 Filesystem revision #:    1 (dynamic) Filesystem features:      has_journal ext_attr resize_inode dir_index filetype needs_recovery extent 64bit Default mount options:    user_xattr acl Filesystem state:         clean Errors behavior:          Continue Filesystem OS type:       Linux Inode count:              3840000 Block count:              5242880 Reserved block count:     0 Free blocks:              5514413 Free inodes:              3677492 First block:              0 Block size:               4096 Fragment size:            4096 Group descriptor size:    64 Reserved GDT blocks:      252 Blocks per group:         32768 Fragments per group:      32768 Inodes per group:         16000 Inode blocks per group:   1000 Flex block group size:    16 Mount count:              2 Maximum mount count:      -1 Check interval:           0 (<none>) Lifetime writes:          180 GB Reserved blocks uid:      0 (user root) Reserved blocks gid:      0 (group root) First inode:              11 Inode size:	              256 Required extra isize:     32 Desired extra isize:      32 Journal inode:            8 Default directory hash:   half_md4 Directory Hash Seed:      Test Hashing Journal backup:           inode blocks Checksum type:            crc32c Checksum:                 0x57705f62 Journal features:         journal_incompat_revoke journal_64bit journal_checksum_v3 Journal size:             64M Journal length:           16384 Journal sequence:         0x00037109 Journal start:            1 Journal checksum type:    crc32c Journal checksum:         0xb7df3c6e ` cmdOutputNoDataExt4 := `Filesystem volume name:   cloudimg-rootfs Last mounted on:          / Filesystem UUID:          testUUID Filesystem magic number:  0xEF53 Filesystem revision #:    1 (dynamic) Filesystem features:      has_journal ext_attr resize_inode dir_index filetype needs_recovery extent 64bit Default mount options:    user_xattr acl Filesystem state:         clean Errors behavior:          Continue Filesystem OS type:       Linux Inode count:              3840000 Reserved block count:     0 Free blocks:              5514413 Free inodes:              3677492 First block:              0 Fragment size:            4096 Group descriptor size:    64 Reserved GDT blocks:      252 Blocks per group:         32768 Fragments per group:      32768 Inodes per group:         16000 Inode blocks per group:   1000 Flex block group size:    16 Mount count:              2 Maximum mount count:      -1 Check interval:           0 (<none>) Lifetime writes:          180 GB Reserved blocks uid:      0 (user root) Reserved blocks gid:      0 (group root) First inode:              11 Inode size:	              256 Required extra isize:     32 Desired extra isize:      32 Journal inode:            8 Default directory hash:   half_md4 Directory Hash Seed:      Test Hashing Journal backup:           inode blocks Checksum type:            crc32c Checksum:                 0x57705f62 Journal features:         journal_incompat_revoke journal_64bit journal_checksum_v3 Journal size:             64M Journal length:           16384 Journal sequence:         0x00037109 Journal start:            1 Journal checksum type:    crc32c Journal checksum:         0xb7df3c6e ` testcases := []struct { name        string devicePath  string blocksize   uint64 blockCount  uint64 cmdOutput   string expectError bool fsType      string }{ { name:        "success parse xfs info", devicePath:  "/dev/test1", blocksize:   4096, blockCount:  1835008, cmdOutput:   cmdOutputSuccessXfs, expectError: false, fsType:      "xfs", }, { name:        "block size not present - xfs", devicePath:  "/dev/test1", blocksize:   0, blockCount:  0, cmdOutput:   cmdOutputNoDataXfs, expectError: true, fsType:      "xfs", }, { name:        "success parse ext info", devicePath:  "/dev/test1", blocksize:   4096, blockCount:  5242880, cmdOutput:   cmdOutputSuccessExt4, expectError: false, fsType:      "ext4", }, { name:        "block size not present - ext4", devicePath:  "/dev/test1", blocksize:   0, blockCount:  0, cmdOutput:   cmdOutputNoDataExt4, expectError: true, fsType:      "ext4", }, } for _, test := range testcases { t.Run(test.name, func(t *testing.T) { fcmd := fakeexec.FakeCmd{ CombinedOutputScript: []fakeexec.FakeAction{ func() ([]byte, []byte, error) { return []byte(test.cmdOutput), nil, nil }, }, } fexec := fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } resizefs := ResizeFs{exec: &fexec} var blockSize uint64 var fsSize uint64 var err error switch test.fsType { case "xfs": blockSize, fsSize, err = resizefs.getXFSSize(test.devicePath) case "ext4": blockSize, fsSize, err = resizefs.getExtSize(test.devicePath) } if blockSize != test.blocksize { t.Fatalf("Parse wrong block size value, expect %d, but got %d", test.blocksize, blockSize) } if fsSize != test.blocksize*test.blockCount { t.Fatalf("Parse wrong fs size value, expect %d, but got %d", test.blocksize*test.blockCount, fsSize) } if !test.expectError && err != nil { t.Fatalf("Expect no error but got %v", err) } }) } } func TestNeedResize(t *testing.T) { testcases := []struct { name            string devicePath      string deviceMountPath string deviceSize      string cmdOutputFsType string expectError     bool expectResult    bool }{ { name:            "False - Unsupported fs type", devicePath:      "/dev/test1", deviceMountPath: "/mnt/test1", deviceSize:      "2048", cmdOutputFsType: "TYPE=ntfs", expectError:     true, expectResult:    false, }, } for _, test := range testcases { t.Run(test.name, func(t *testing.T) { fcmd := fakeexec.FakeCmd{ CombinedOutputScript: []fakeexec.FakeAction{ func() ([]byte, []byte, error) { return []byte(test.deviceSize), nil, nil }, func() ([]byte, []byte, error) { return []byte(test.cmdOutputFsType), nil, nil }, }, } fexec := fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } resizefs := ResizeFs{exec: &fexec} needResize, err := resizefs.NeedResize(test.devicePath, test.deviceMountPath) if needResize != test.expectResult { t.Fatalf("Expect result is %v but got %v", test.expectResult, needResize) } if !test.expectError && err != nil { t.Fatalf("Expect no error but got %v", err) } }) } } </ins>

This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Mark this issue or PR as rotten with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle stale [1]:
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle rotten [1]:
/remove-lifecycle rotten
The Kubernetes project currently lacks enough contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Mark this issue or PR as rotten with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle stale [1]:
/remove-lifecycle rotten
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle rotten [1]:
The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs.


Question: <ins> // +build linux /* Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package mount import ( "k8s.io/utils/exec" fakeexec "k8s.io/utils/exec/testing" "testing" ) func TestGetFileSystemSize(t *testing.T) { cmdOutputSuccessXfs := ` statfs.f_bsize = 4096 statfs.f_blocks = 1832448 statfs.f_bavail = 1822366 statfs.f_files = 3670016 statfs.f_ffree = 3670012 statfs.f_flags = 0x1020 geom.bsize = 4096 geom.agcount = 4 geom.agblocks = 458752 geom.datablocks = 1835008 geom.rtblocks = 0 geom.rtextents = 0 geom.rtextsize = 1 geom.sunit = 0 geom.swidth = 0 counts.freedata = 1822372 counts.freertx = 0 counts.freeino = 61 counts.allocino = 64 ` cmdOutputNoDataXfs := ` statfs.f_bsize = 4096 statfs.f_blocks = 1832448 statfs.f_bavail = 1822366 statfs.f_files = 3670016 statfs.f_ffree = 3670012 statfs.f_flags = 0x1020 geom.agcount = 4 geom.agblocks = 458752 geom.rtblocks = 0 geom.rtextents = 0 geom.rtextsize = 1 geom.sunit = 0 geom.swidth = 0 counts.freedata = 1822372 counts.freertx = 0 counts.freeino = 61 counts.allocino = 64 ` cmdOutputSuccessExt4 := ` Filesystem volume name:   cloudimg-rootfs Last mounted on:          / Filesystem UUID:          testUUID Filesystem magic number:  0xEF53 Filesystem revision #:    1 (dynamic) Filesystem features:      has_journal ext_attr resize_inode dir_index filetype needs_recovery extent 64bit Default mount options:    user_xattr acl Filesystem state:         clean Errors behavior:          Continue Filesystem OS type:       Linux Inode count:              3840000 Block count:              5242880 Reserved block count:     0 Free blocks:              5514413 Free inodes:              3677492 First block:              0 Block size:               4096 Fragment size:            4096 Group descriptor size:    64 Reserved GDT blocks:      252 Blocks per group:         32768 Fragments per group:      32768 Inodes per group:         16000 Inode blocks per group:   1000 Flex block group size:    16 Mount count:              2 Maximum mount count:      -1 Check interval:           0 (<none>) Lifetime writes:          180 GB Reserved blocks uid:      0 (user root) Reserved blocks gid:      0 (group root) First inode:              11 Inode size:	              256 Required extra isize:     32 Desired extra isize:      32 Journal inode:            8 Default directory hash:   half_md4 Directory Hash Seed:      Test Hashing Journal backup:           inode blocks Checksum type:            crc32c Checksum:                 0x57705f62 Journal features:         journal_incompat_revoke journal_64bit journal_checksum_v3 Journal size:             64M Journal length:           16384 Journal sequence:         0x00037109 Journal start:            1 Journal checksum type:    crc32c Journal checksum:         0xb7df3c6e ` cmdOutputNoDataExt4 := `Filesystem volume name:   cloudimg-rootfs Last mounted on:          / Filesystem UUID:          testUUID Filesystem magic number:  0xEF53 Filesystem revision #:    1 (dynamic) Filesystem features:      has_journal ext_attr resize_inode dir_index filetype needs_recovery extent 64bit Default mount options:    user_xattr acl Filesystem state:         clean Errors behavior:          Continue Filesystem OS type:       Linux Inode count:              3840000 Reserved block count:     0 Free blocks:              5514413 Free inodes:              3677492 First block:              0 Fragment size:            4096 Group descriptor size:    64 Reserved GDT blocks:      252 Blocks per group:         32768 Fragments per group:      32768 Inodes per group:         16000 Inode blocks per group:   1000 Flex block group size:    16 Mount count:              2 Maximum mount count:      -1 Check interval:           0 (<none>) Lifetime writes:          180 GB Reserved blocks uid:      0 (user root) Reserved blocks gid:      0 (group root) First inode:              11 Inode size:	              256 Required extra isize:     32 Desired extra isize:      32 Journal inode:            8 Default directory hash:   half_md4 Directory Hash Seed:      Test Hashing Journal backup:           inode blocks Checksum type:            crc32c Checksum:                 0x57705f62 Journal features:         journal_incompat_revoke journal_64bit journal_checksum_v3 Journal size:             64M Journal length:           16384 Journal sequence:         0x00037109 Journal start:            1 Journal checksum type:    crc32c Journal checksum:         0xb7df3c6e ` testcases := []struct { name        string devicePath  string blocksize   uint64 blockCount  uint64 cmdOutput   string expectError bool fsType      string }{ { name:        "success parse xfs info", devicePath:  "/dev/test1", blocksize:   4096, blockCount:  1835008, cmdOutput:   cmdOutputSuccessXfs, expectError: false, fsType:      "xfs", }, { name:        "block size not present - xfs", devicePath:  "/dev/test1", blocksize:   0, blockCount:  0, cmdOutput:   cmdOutputNoDataXfs, expectError: true, fsType:      "xfs", }, { name:        "success parse ext info", devicePath:  "/dev/test1", blocksize:   4096, blockCount:  5242880, cmdOutput:   cmdOutputSuccessExt4, expectError: false, fsType:      "ext4", }, { name:        "block size not present - ext4", devicePath:  "/dev/test1", blocksize:   0, blockCount:  0, cmdOutput:   cmdOutputNoDataExt4, expectError: true, fsType:      "ext4", }, } for _, test := range testcases { t.Run(test.name, func(t *testing.T) { fcmd := fakeexec.FakeCmd{ CombinedOutputScript: []fakeexec.FakeAction{ func() ([]byte, []byte, error) { return []byte(test.cmdOutput), nil, nil }, }, } fexec := fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } resizefs := ResizeFs{exec: &fexec} var blockSize uint64 var fsSize uint64 var err error switch test.fsType { case "xfs": blockSize, fsSize, err = resizefs.getXFSSize(test.devicePath) case "ext4": blockSize, fsSize, err = resizefs.getExtSize(test.devicePath) } if blockSize != test.blocksize { t.Fatalf("Parse wrong block size value, expect %d, but got %d", test.blocksize, blockSize) } if fsSize != test.blocksize*test.blockCount { t.Fatalf("Parse wrong fs size value, expect %d, but got %d", test.blocksize*test.blockCount, fsSize) } if !test.expectError && err != nil { t.Fatalf("Expect no error but got %v", err) } }) } } func TestNeedResize(t *testing.T) { testcases := []struct { name            string devicePath      string deviceMountPath string deviceSize      string cmdOutputFsType string expectError     bool expectResult    bool }{ { name:            "False - Unsupported fs type", devicePath:      "/dev/test1", deviceMountPath: "/mnt/test1", deviceSize:      "2048", cmdOutputFsType: "TYPE=ntfs", expectError:     true, expectResult:    false, }, } for _, test := range testcases { t.Run(test.name, func(t *testing.T) { fcmd := fakeexec.FakeCmd{ CombinedOutputScript: []fakeexec.FakeAction{ func() ([]byte, []byte, error) { return []byte(test.deviceSize), nil, nil }, func() ([]byte, []byte, error) { return []byte(test.cmdOutputFsType), nil, nil }, }, } fexec := fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } resizefs := ResizeFs{exec: &fexec} needResize, err := resizefs.NeedResize(test.devicePath, test.deviceMountPath) if needResize != test.expectResult { t.Fatalf("Expect result is %v but got %v", test.expectResult, needResize) } if !test.expectError && err != nil { t.Fatalf("Expect no error but got %v", err) } }) } } </ins>

This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Reopen this issue or PR with Mark this issue or PR as fresh with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /close [1]:
Closing this issue. <detailsIn response to : The Kubernetes project currently lacks enough active contributors to adequately respond to all issues and PRs. After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closedReopen this issue or PR with Mark this issue or PR as fresh with Offer to help out with [Issue Triage][1] Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/remove-lifecycle stale /reopen
Reopened this issue. <detailsIn response to : /remove-lifecycle stale /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
The issue has been marked as an important bug and triaged. Such issues are automatically marked as frozen when hitting the rotten state to avoid missing important bugs. Please send feedback to sig-contributor-experience at . /lifecycle frozen
This issue is labeled with but has not been updated in over 90 days, and should be re-triaged. Important-soon issues must be staffed and worked on either currently, or very soon, ideally in time for the next release. You can: Confirm that this issue is still relevant with (org members only) Deprioritize it with or Close this issue with For more details on the triage process, see /remove-triage accepted
/triage accepted /priority long-term what is the latest status on this?
The label(s) cannot be applied, because the repository doesn't have them. <detailsIn response to : /triage accepted /priority long-term what is the latest status on this? Instructions for interacting with me using PR comments are available .


Question: <ins> // +build linux /* Copyright 2021 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package mount import ( "k8s.io/utils/exec" fakeexec "k8s.io/utils/exec/testing" "testing" ) func TestGetFileSystemSize(t *testing.T) { cmdOutputSuccessXfs := ` statfs.f_bsize = 4096 statfs.f_blocks = 1832448 statfs.f_bavail = 1822366 statfs.f_files = 3670016 statfs.f_ffree = 3670012 statfs.f_flags = 0x1020 geom.bsize = 4096 geom.agcount = 4 geom.agblocks = 458752 geom.datablocks = 1835008 geom.rtblocks = 0 geom.rtextents = 0 geom.rtextsize = 1 geom.sunit = 0 geom.swidth = 0 counts.freedata = 1822372 counts.freertx = 0 counts.freeino = 61 counts.allocino = 64 ` cmdOutputNoDataXfs := ` statfs.f_bsize = 4096 statfs.f_blocks = 1832448 statfs.f_bavail = 1822366 statfs.f_files = 3670016 statfs.f_ffree = 3670012 statfs.f_flags = 0x1020 geom.agcount = 4 geom.agblocks = 458752 geom.rtblocks = 0 geom.rtextents = 0 geom.rtextsize = 1 geom.sunit = 0 geom.swidth = 0 counts.freedata = 1822372 counts.freertx = 0 counts.freeino = 61 counts.allocino = 64 ` cmdOutputSuccessExt4 := ` Filesystem volume name:   cloudimg-rootfs Last mounted on:          / Filesystem UUID:          testUUID Filesystem magic number:  0xEF53 Filesystem revision #:    1 (dynamic) Filesystem features:      has_journal ext_attr resize_inode dir_index filetype needs_recovery extent 64bit Default mount options:    user_xattr acl Filesystem state:         clean Errors behavior:          Continue Filesystem OS type:       Linux Inode count:              3840000 Block count:              5242880 Reserved block count:     0 Free blocks:              5514413 Free inodes:              3677492 First block:              0 Block size:               4096 Fragment size:            4096 Group descriptor size:    64 Reserved GDT blocks:      252 Blocks per group:         32768 Fragments per group:      32768 Inodes per group:         16000 Inode blocks per group:   1000 Flex block group size:    16 Mount count:              2 Maximum mount count:      -1 Check interval:           0 (<none>) Lifetime writes:          180 GB Reserved blocks uid:      0 (user root) Reserved blocks gid:      0 (group root) First inode:              11 Inode size:	              256 Required extra isize:     32 Desired extra isize:      32 Journal inode:            8 Default directory hash:   half_md4 Directory Hash Seed:      Test Hashing Journal backup:           inode blocks Checksum type:            crc32c Checksum:                 0x57705f62 Journal features:         journal_incompat_revoke journal_64bit journal_checksum_v3 Journal size:             64M Journal length:           16384 Journal sequence:         0x00037109 Journal start:            1 Journal checksum type:    crc32c Journal checksum:         0xb7df3c6e ` cmdOutputNoDataExt4 := `Filesystem volume name:   cloudimg-rootfs Last mounted on:          / Filesystem UUID:          testUUID Filesystem magic number:  0xEF53 Filesystem revision #:    1 (dynamic) Filesystem features:      has_journal ext_attr resize_inode dir_index filetype needs_recovery extent 64bit Default mount options:    user_xattr acl Filesystem state:         clean Errors behavior:          Continue Filesystem OS type:       Linux Inode count:              3840000 Reserved block count:     0 Free blocks:              5514413 Free inodes:              3677492 First block:              0 Fragment size:            4096 Group descriptor size:    64 Reserved GDT blocks:      252 Blocks per group:         32768 Fragments per group:      32768 Inodes per group:         16000 Inode blocks per group:   1000 Flex block group size:    16 Mount count:              2 Maximum mount count:      -1 Check interval:           0 (<none>) Lifetime writes:          180 GB Reserved blocks uid:      0 (user root) Reserved blocks gid:      0 (group root) First inode:              11 Inode size:	              256 Required extra isize:     32 Desired extra isize:      32 Journal inode:            8 Default directory hash:   half_md4 Directory Hash Seed:      Test Hashing Journal backup:           inode blocks Checksum type:            crc32c Checksum:                 0x57705f62 Journal features:         journal_incompat_revoke journal_64bit journal_checksum_v3 Journal size:             64M Journal length:           16384 Journal sequence:         0x00037109 Journal start:            1 Journal checksum type:    crc32c Journal checksum:         0xb7df3c6e ` testcases := []struct { name        string devicePath  string blocksize   uint64 blockCount  uint64 cmdOutput   string expectError bool fsType      string }{ { name:        "success parse xfs info", devicePath:  "/dev/test1", blocksize:   4096, blockCount:  1835008, cmdOutput:   cmdOutputSuccessXfs, expectError: false, fsType:      "xfs", }, { name:        "block size not present - xfs", devicePath:  "/dev/test1", blocksize:   0, blockCount:  0, cmdOutput:   cmdOutputNoDataXfs, expectError: true, fsType:      "xfs", }, { name:        "success parse ext info", devicePath:  "/dev/test1", blocksize:   4096, blockCount:  5242880, cmdOutput:   cmdOutputSuccessExt4, expectError: false, fsType:      "ext4", }, { name:        "block size not present - ext4", devicePath:  "/dev/test1", blocksize:   0, blockCount:  0, cmdOutput:   cmdOutputNoDataExt4, expectError: true, fsType:      "ext4", }, } for _, test := range testcases { t.Run(test.name, func(t *testing.T) { fcmd := fakeexec.FakeCmd{ CombinedOutputScript: []fakeexec.FakeAction{ func() ([]byte, []byte, error) { return []byte(test.cmdOutput), nil, nil }, }, } fexec := fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } resizefs := ResizeFs{exec: &fexec} var blockSize uint64 var fsSize uint64 var err error switch test.fsType { case "xfs": blockSize, fsSize, err = resizefs.getXFSSize(test.devicePath) case "ext4": blockSize, fsSize, err = resizefs.getExtSize(test.devicePath) } if blockSize != test.blocksize { t.Fatalf("Parse wrong block size value, expect %d, but got %d", test.blocksize, blockSize) } if fsSize != test.blocksize*test.blockCount { t.Fatalf("Parse wrong fs size value, expect %d, but got %d", test.blocksize*test.blockCount, fsSize) } if !test.expectError && err != nil { t.Fatalf("Expect no error but got %v", err) } }) } } func TestNeedResize(t *testing.T) { testcases := []struct { name            string devicePath      string deviceMountPath string deviceSize      string cmdOutputFsType string expectError     bool expectResult    bool }{ { name:            "False - Unsupported fs type", devicePath:      "/dev/test1", deviceMountPath: "/mnt/test1", deviceSize:      "2048", cmdOutputFsType: "TYPE=ntfs", expectError:     true, expectResult:    false, }, } for _, test := range testcases { t.Run(test.name, func(t *testing.T) { fcmd := fakeexec.FakeCmd{ CombinedOutputScript: []fakeexec.FakeAction{ func() ([]byte, []byte, error) { return []byte(test.deviceSize), nil, nil }, func() ([]byte, []byte, error) { return []byte(test.cmdOutputFsType), nil, nil }, }, } fexec := fakeexec.FakeExec{ CommandScript: []fakeexec.FakeCommandAction{ func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, func(cmd string, args ...string) exec.Cmd { return fakeexec.InitFakeCmd(&fcmd, cmd, args...) }, }, } resizefs := ResizeFs{exec: &fexec} needResize, err := resizefs.NeedResize(test.devicePath, test.deviceMountPath) if needResize != test.expectResult { t.Fatalf("Expect result is %v but got %v", test.expectResult, needResize) } if !test.expectError && err != nil { t.Fatalf("Expect no error but got %v", err) } }) } } </ins>

If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/priority important-long-term
The label(s) cannot be applied, because the repository doesn't have them. <detailsIn response to : /priority important-long-term Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Hello I've checked what is the current status of the drivers that were involved in this effort and I think we might be able to close this as all upstream work is done. Please take a look at the table below and let me know if anything is missing.
/close
Closing this issue. <detailsIn response to : /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: return utilerrors.NewAggregate(errs) } <del> // Gets the (CPU) container the specified pid is in. </del> <ins> // getContainer returns the cgroup associated with the specified pid. // It enforces a unified hierarchy for memory and cpu cgroups. // On systemd environments, it uses the name=systemd cgroup for the specified pid. </ins> func getContainer(pid int) (string, error) { cgs, err := cgroups.ParseCgroupFile(fmt.Sprintf("/proc/%d/cgroup", pid)) if err != nil { return "", err } <del> cg, ok := cgs["cpu"] if ok { return cg, nil </del> <ins> cpu, found := cgs["cpu"] if !found { return "", cgroups.NewNotFoundError("cpu") } memory, found := cgs["memory"] if !found { return "", cgroups.NewNotFoundError("memory") } // since we use this container for accounting, we need to ensure its a unified hierarchy. if cpu != memory { return "", fmt.Errorf("cpu and memory cgroup hierarchy not unified.  cpu: %s, memory: %s", cpu, memory) } // on systemd, every pid is in a unified cgroup hierarchy (name=systemd as seen in systemd-cgls) // cpu and memory accounting is off by default, users may choose to enable it per unit or globally. // users could enable CPU and memory accounting globally via /etc/systemd/system.conf (DefaultCPUAccounting=true DefaultMemoryAccounting=true). // users could also enable CPU and memory accounting per unit via CPUAccounting=true and MemoryAccounting=true // we only warn if accounting is not enabled for CPU or memory so as to not break local development flows where kubelet is launched in a terminal. // for example, the cgroup for the user session will be something like /user.slice/user-X.slice/session-X.scope, but the cpu and memory // cgroup will be the closest ancestor where accounting is performed (most likely /) on systems that launch docker containers. // as a result, on those systems, you will not get cpu or memory accounting statistics for kubelet. // in addition, you would not get memory or cpu accounting for the runtime unless accounting was enabled on its unit (or globally). if systemd, found := cgs["name=systemd"]; found { if systemd != cpu { glog.Warningf("CPUAccounting not enabled for pid: %d", pid) } if systemd != memory { glog.Warningf("MemoryAccounting not enabled for pid: %d", pid) } return systemd, nil </ins> } <del> return "", cgroups.NewNotFoundError("cpu") </del> <ins> return cpu, nil </ins> } // Ensures the system container is created and all non-kernel threads and process 1

container manager setup node routine should ensure that cpu and memory cgroups for container runtime are of the same value, and warn if they are equal to . For background, on systemd environments, the default unit file for has CPU and Memory accounting not enabled, so the /proc/.../cgroup file reports as cpu cgroup.
/cc - I will send a bug-fix for 1.3 for this in next couple days.
/cc
Looking forward to the fix
The PR that fixes this issue was prematurely merged by the merge bot so re-opening.


Question: curNode.ObjectMeta.Labels[kubeletapis.LabelInstanceType] = instanceType } <del> // TODO(wlan0): Move this logic to the route controller using the node taint instead of condition // Since there are node taints, do we still need this? // This condition marks the node as unusable until routes are initialized in the cloud provider if cnc.cloud.ProviderName() == "gce" { curNode.Status.Conditions = append(node.Status.Conditions, v1.NodeCondition{ Type:               v1.NodeNetworkUnavailable, Status:             v1.ConditionTrue, Reason:             "NoRouteCreated", Message:            "Node created without a route", LastTransitionTime: metav1.Now(), }) } </del> if zones, ok := cnc.cloud.Zones(); ok { zone, err := getZoneByProviderIDOrName(zones, curNode) if err != nil {

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . If the matter is security related, please disclose it privately via --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: In , the controller add NodeNetworkUnavailable to the node conditions, and use to update it. But node status can not be updated by , it doesn't work because replace the node status with old status. To fix this bug, we should use to update node status. /sig cloud-provider /sig cluster-lifecycle /area cloudprovider What you expected to happen: How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others:
I agree this is a bug. As a matter of interest though, how are you exercising this bug? It seems specific to cloud controller on gce. However that controller isn't actually enabled on gce. It would be good for me to understand your use case, to make sure we don't break it later.
I built my kubernetes cluster with external cloudprovider, but I found if I stop the route controller, pod will be scheduled on node even if the route is not for that node. I read the code and found that the node condition would not be add when using other cloudproviders. At first, I change "gce" in that line code to the name of my cloudprovider, but the problem also exists. Then I found that didn't take effect actually.


Question: // Optional, no events will be sent without it etcdClient tools.EtcdClient <del> // Optional, no statistics will be available if omitted cadvisorClient CadvisorInterface </del> // Optional, defaults to simple implementaiton healthChecker health.HealthChecker // Optional, defaults to simple Docker implementation

So today, cadvisor became wedged and just stopped paying attention to its socket. Not kubelet's fault. But when I would launch all I would get would be: It took me a bit of trolling through netstat and lsof to figure out that the problem was cadvisor being completely locked up. kill -9 cadvisor and everything started working well. I wonder if we should emit more messages during startup about what operation we are about to attempt, not just the results of operations, as I was blaming docker for a while....
kubelet shouldn't block on this. I'll take a look.
Any ideas what was locking cAdvisor? and was it just not responding to requests?
I don't have any idea why it was locked. If I find another one locked I'll hang onto it to debug. Journal had: obviously the 'kill' came from me. It would not respond to systemctl cadvisor stop. netstat looking at port 4194 showed me At which point I realized cadvisor wasn't handling its sockets right since so many long dead kubelet sockets hadn't been cleaned up, so I kill -9'd it.
I cut some interesting info from the cadvisor journal log (trying to cut hostnames) You can see the last message about failed to update stats for container ... happened Oct 08 01:06:01. It was then silent until I killed it Oct 08 19:20:13, even though I'm pretty sure dnf was still being launched every hour on the hour. So it was wedged for about 18 hours.
What kernel is this on? We've seen there is a kernel bug fixed in 3.12+ that sometimes hangs us forever on exit. We're not quite sure how it gets triggered...
16.3


Question: } } <ins> // Set the connection reuse mode if val, _ := sysctl.GetSysctl(sysctlConnReuse); val != 0 { if err := sysctl.SetSysctl(sysctlConnReuse, 0); err != nil { return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlConnReuse, err) } } </ins> // Set the ip_forward sysctl we need for if val, _ := sysctl.GetSysctl(sysctlForward); val != 1 { if err := sysctl.SetSysctl(sysctlForward, 1); err != nil {

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!--What happened: Service with clusterIP and a domain , the domain resolve to ingress-nginx. Anothor service will request , when use domain it works fine. I think using clusterIP directly will get better performance because it saves some time for dns resolve and ingress-nginx proxy, so I change domain to clusterIP. But the result was not expected, it got worse, very low throughput and high latency. What you expected to happen: On the above situation, clusterIP should have better performance than ingress-nginx. How to reproduce it (as minimally and precisely as possible): mode, do test with ab Anything else we need to know?: I install an ipvs enviroment manually and do some test, and found it possiblly because ipvs will use iptables to realize ! Environment: Kubernetes version (use ): 1.11.1 Cloud provider or hardware configuration: OS (e.g. from /etc/os-release): centos 7.2 Kernel (e.g. ): 3.10 Install tools: binary Others: <!-- DO NOT EDIT BELOW THIS LINE --/kind bug
Reiterating the mentions to trigger a notification: <detailsIn response to : Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
I try to change some kernel param and found is related to this issue , default . change to then complete about 3000k request, pretty good performance. So, should I change the param in production? Does it have any side effects
/area ipvs
Can you look at it?
, default 1. change to 0then complte about 3000k request, pretty good performance. Yes, seems you find the key to the problem - it's set to 0 in my environment and I have not found any issue so far(it's about IPVS module behavior and we will not find parameter in old kernels ). BTW, you can also try to set the ipvs conntrack() to 0 - it will improve performance as well.
Thanks for reply.


Question: } } <ins> // Set the connection reuse mode if val, _ := sysctl.GetSysctl(sysctlConnReuse); val != 0 { if err := sysctl.SetSysctl(sysctlConnReuse, 0); err != nil { return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlConnReuse, err) } } </ins> // Set the ip_forward sysctl we need for if val, _ := sysctl.GetSysctl(sysctlForward); val != 1 { if err := sysctl.SetSysctl(sysctlForward, 1); err != nil {

I am using kubernetes , and after set , clusterIP can not be reachable. Which version do I need ?
oops, have you ever configured Cluster-CIDR or iptables masq-all for kube-proxy?
Yes, I configure Cluster-CIDR
Sorry, Cluster IP still needs the IPVS conntrack and nodeport does not rely on it any more - we are trying to figure it out for Cluster IP.
Got it, thanks!
cc Should we tune the kernel parameter in kube-proxy? Given that many users complain about the performance issue, probably the answer is YES.
There is a nice article describing even more kernel parameters to tune ipvs:
Side effect like this comment: <img width="790" alt="hpaprize" src="https://user-"Note: time in picture below is UTC+8. ! Abviously every time scaledown , response time will increase. Any idea for this?
Thanks for reporting. Have you ever read the article posted by In this article, some ipvs parameters can be tuned, e.g.
yes, I had test these parameters not found in centos 7.2(kernel 3.10) will not affect the result is similar with , will be low throughput According to this issue ,I guess it may be the root cause, so I upgrade my setup from 1.11.1 to 1.12.2. But did not solve this problem, and found a new problem, rs delete failed after scaledown. I had open a new issue to this problem:
/assign
Hi, , I raised a pr for this issue. Please check. : ) Looking forward for your advise.
Could you please check out the HEAD to see if your issues have been resolved?
It's not a big surprise this benchmark performs better with . uses http/1.0 so is creating a new connection for every single request. With IPVS will reuse the existing connection entry if source port matches - sending repeated requests to the same real server rather than actually load balancing. So my guess is you will see an unbalanced load with using and , but it makes the benchmark look better. I'm not sure this benchmark is a realistic scenario. I suggest using a tool like instead which uses persistent connections and see what happens.
Apart from the load balancing issue, it's also possible the overhead of is bad when creating connections in a short period of time.


Question: } } <ins> // Set the connection reuse mode if val, _ := sysctl.GetSysctl(sysctlConnReuse); val != 0 { if err := sysctl.SetSysctl(sysctlConnReuse, 0); err != nil { return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlConnReuse, err) } } </ins> // Set the ip_forward sysctl we need for if val, _ := sysctl.GetSysctl(sysctlForward); val != 1 { if err := sysctl.SetSysctl(sysctlForward, 1); err != nil {

The benchmark is basically stress testing the new connection handling of IPVS. But I'm not sure we should be optimizing for such a thing, when in reality connections will be much longer lived (like with gRPC, or http/1.1+). It's a trade off between improved load balancing (connreusemode=1 or 2) and connection creation latency (connreusemode=0).
I have a php program use unirest to request other sevice in the cluster. It not use keepalive.
I would prefer to improve the latency number and I think the scenario where one client send lots of requests at the same time is not a common case and mostly happen in test environment.
It might be possible to improve this in the kernel too. You can see the change here: bypasses the reuse checks and expiring logic for new connections.
We set in IPVS proxier in case other people don't know what kernel parameters they should tune. You are familiar with IPVS and linux kernel, but I think many users don't have enough context.
, I think this was because the the container already deleted, but kube-proxy need time to flush the rules, iptables mode should also have this problem.
Yes, I will test preStop recently, may solve the problem
Here the likely reason for the slow throughput is described: As setting to causes , is any of the options listed possible to do? I guess only option 2 might be worth looking at as conntrack is probably needed for the remaining iptables components.
Hello everyone We are very fortunate to tell you that this bug has been fixed by us and has been verified to work very well. The patch is being submitted to the Linux kernel community. You can also apply this patch to your own kernel, and then only need to set (default) and (default). As the sysctl switch is newly . You can adapt the kube-proxy by judging whether there is , if so, it means that the current kernel is the version that fixed this bug.


Question: } } <ins> // Set the connection reuse mode if val, _ := sysctl.GetSysctl(sysctlConnReuse); val != 0 { if err := sysctl.SetSysctl(sysctlConnReuse, 0); err != nil { return nil, fmt.Errorf("can't set sysctl %s: %v", sysctlConnReuse, err) } } </ins> // Set the ip_forward sysctl we need for if val, _ := sysctl.GetSysctl(sysctlForward); val != 1 { if err := sysctl.SetSysctl(sysctlForward, 1); err != nil {

That Can solve the following problems: update, IPVS keeps scheduling traffic to the destroyed Pod IPVS traffic scheduling after scaled up or rolling update IPVS low throughput issue second connection delay in masque low throughputÂ Bench can fill up ipvs service proxy in seconds 1s latency in when upgrading from on RHEL 8.1 -proxy ipvs connreuse_mode setting causes errors with high load from single client Thank you. By Yang Yuxi TencentCloudContainerTeam
Following-up on comment above for posterity. The patch mentioned above didn't make it to the kernel but there are two recently merged patches worth highlighting. One of them fixes the 1 second delay issue and the other fixes dropped packets when stale connection entries in the IPVS table are used: 1) 2)


Question: kubectl delete secret test-secret --namespace=test-secrets ### Create a tls secret <ins> if [[ "${WAIT_FOR_DELETION:-}" == "true" ]]; then kube::test::wait_object_assert 'secrets --namespace=test-secrets' "{{range.items}}{{$id_field}}:{{end}}" '' fi </ins> # Pre-condition: no SECRET exists kube::test::get_object_assert 'secrets --namespace=test-secrets' "{{range.items}}{{$id_field}}:{{end}}" '' # Command

disabled them since was flaky. We need to find and fix the issue and then reenable these tests. cc Related issues: and
Probably should be higher priority so it doesn't get dropped
The issue is that we dont wait for the resource to be deleted unless we are using a reaper: This breaks for federation since we use finalizers. kubectl should wait for the resource to be deleted. We should also set OrphanDependents = false. This will have no effect in kubernetes but will enable cascading deletion in federation. Ref cc Will send a fix.
Sent
Updated to remove the wait (GET) at the end so it no longer fixes this issue. Sent to update the test to include a wait which fixes this issue.


Question: name: cql resources: limits: <del> cpu: "1" </del> <ins> cpu: "500m" </ins> memory: 1Gi requests: <del> cpu: "1" </del> <ins> cpu: "500m" </ins> memory: 1Gi securityContext: capabilities:

https://k8s- Failed: [] [Feature:Example] [] CassandraPetSet should create petset {Kubernetes e2e suite}
https://k8s- Failed: [] [Feature:Example] [] CassandraPetSet should create petset {Kubernetes e2e suite}
https://k8s- Failed: [] [Feature:Example] [] CassandraPetSet should create petset {Kubernetes e2e suite}
sorry wrong issue.
looks like this was recently in #. can you take a look please?
https://k8s- Failed: [] [Feature:Example] [] CassandraPetSet should create petset {Kubernetes e2e suite}
This requires persistent storage. Is this flaky on master? Any guidance on how to diagnose this? I did run the e2e tests on GCE.
and is this really a p0?
it's happening pretty frequently, but I don't think we care about examples tests as much.
https://k8s- Failed: [] [Feature:Example] [] CassandraPetSet should create petset {Kubernetes e2e suite}
[FLAKE-PING] This flaky-test issue would love to have more attention.
who can assist me with this? I cannot recreate it ... Command I ran. Results that I am getting. Just ran that off of head, and it worked. I am rerunning as the full test suite.
look through the logs, from last test cassandra-2 was stuck in pending which prevents the petset from starting up (c*-3 not even created) you see events like and logs like
so the test is already slow. I am only giving it one cpu, giving it less will make it super slow. Any recommendations? .25 CPU? And where did you find this?
[FLAKE-PING] This flaky-test issue would love to have more attention.
This is going to sit for a bit. With my schedule this week, won't likely get to it.
https://k8s- Failed: [] [Feature:Example] [] CassandraPetSet should create petset {Kubernetes e2e suite}
[FLAKE-PING] This flaky-test issue would love to have more attention.


Question: name: cql resources: limits: <del> cpu: "1" </del> <ins> cpu: "500m" </ins> memory: 1Gi requests: <del> cpu: "1" </del> <ins> cpu: "500m" </ins> memory: 1Gi securityContext: capabilities:

https://k8s- Failed: [] [Feature:Example] [] CassandraPetSet should create petset {Kubernetes e2e suite}
https://k8s- Failed: [] [Feature:Example] [] CassandraPetSet should create petset {Kubernetes e2e suite}
https://k8s- Failed: [] [Feature:Example] [] CassandraPetSet should create petset {Kubernetes e2e suite}
[FLAKE-PING] This flaky-test issue would love to have more attention.
[FLAKE-PING] This flaky-test issue would love to have more attention.
https://k8s- Failed: [] [Feature:Example] [] CassandraPetSet should create petset {Kubernetes e2e suite}
https://k8s- Failed: [] [Feature:Example] [] CassandraPetSet should create petset {Kubernetes e2e suite}
https://k8s- Failed: [] [Feature:Example] [] CassandraPetSet should create petset {Kubernetes e2e suite}
[FLAKE-PING] This flaky-test issue would love to have more attention.
[FLAKE-PING] This flaky-test issue would love to have more attention.
https://k8s- Failed: [] [Feature:Example] [] CassandraPetSet should create petset {Kubernetes e2e suite}
https://k8s- Failed: [] [Feature:Example] [] CassandraPetSet should create petset {Kubernetes e2e suite}
[FLAKE-PING] This flaky-test issue would love to have more attention.
[FLAKE-PING] This flaky-test issue would love to have more attention.
https://k8s- Failed: [] [Feature:Example] [] CassandraPetSet should create petset {Kubernetes e2e suite}
https://k8s- Failed: [] [Feature:Example] [] CassandraPetSet should create petset {Kubernetes e2e suite}
https://k8s- Failed: [] [Feature:Example] [] CassandraPetSet should create petset {Kubernetes e2e suite}
https://k8s- Failed: [] [Feature:Example] [] CassandraPetSet should create petset {Kubernetes e2e suite}
[FLAKE-PING] This flaky-test issue would love to have more attention.


Question: if accountName == "" || accountKey == "" { return "", "", fmt.Errorf("Invalid %v/%v, couldn't extract azurestorageaccountname or azurestorageaccountkey", nameSpace, secretName) } <ins> accountName = strings.TrimSpace(accountName) </ins> return accountName, accountKey, nil }

<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . If this may be security issue, please disclose it privately via --Is this a BUG REPORT or FEATURE REQUEST?: What happened: When trying to mount an Azure File mount it fails, it looks as though its putting a new line between the name of the Azure Storage Account and the Azure storage domain. There are no new line characters in the secret that holds the account name. The following is the secret: The following is the declaration within the section of the Kubernetes yaml: The following is the error that's shown in `kubectl describe po/pod_name: What you expected to happen: How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: azure OS (e.g. from /etc/os-release): Ubuntu 16.04.3 LTS Kernel (e.g. ): 4.13.0-1011-azure Install tools: Others:
/sig storage
/sig azure
It seems to me that does decode to , so it seems to be behaving correctly. Maybe it's a good idea to trim the return char for and fields after ?
is specified by user, so trim the is not a good idea in the code. Another simple way is to use kubectl create secret to create azure-secret, the accountname and key won't require base64 encoded, so you won't mess up with let me know if you have any question, thanks.
Thanks !
Bumped into this issue today too on our cluster. Can confirm command worked. Is there an easy way to time the /n off the end of the echo for before using base64? Thanks.
yes it's
Of course it is! Doh!
i think would've made better sense to trim the azurestorageaccountname value when receiving it... anyways the storage account name wont have n in them right? I think that just made better developer experience...
is not allowed in azure storage account name
so if azure storage account name does not allow in the name.. then why not just trim the name instead of error... Because devs would just and that would already have the n in the resulting base64-encoded string... right?
sounds reasonable /reopen
Reopened this issue.


Question: if accountName == "" || accountKey == "" { return "", "", fmt.Errorf("Invalid %v/%v, couldn't extract azurestorageaccountname or azurestorageaccountkey", nameSpace, secretName) } <ins> accountName = strings.TrimSpace(accountName) </ins> return accountName, accountKey, nil }

<detailsIn response to : sounds reasonable /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
awesome. Thanks
so then this goes in to kube version 1.14.6 in AKS ? or how does this work?
It would only go into k8s v1.16.0 since it's not a critical bug, we don't want to do hot fix to the old release.
got it. thanks Andy.


Question: HOSTNAME_OVERRIDE=${HOSTNAME_OVERRIDE:-"127.0.0.1"} CLOUD_PROVIDER=${CLOUD_PROVIDER:-""} CLOUD_CONFIG=${CLOUD_CONFIG:-""} <del> FEATURE_GATES=${FEATURE_GATES:-"AllAlpha=true"} </del> <ins> FEATURE_GATES=${FEATURE_GATES:-"AllAlpha=false"} </ins> STORAGE_BACKEND=${STORAGE_BACKEND:-"etcd3"} # enable swagger ui ENABLE_SWAGGER_UI=${ENABLE_SWAGGER_UI:-false}

Is this a BUG REPORT or FEATURE REQUEST? (choose one): bug Kubernetes version (use ): Environment: Cloud provider or hardware configuration: N/A OS (e.g. from /etc/os-release): RHE 7.2 Kernel (e.g. ): What happened: The node resource is empty when start local cluster by . What you expected to happen: There's one node in this local cluster. How to reproduce it (as minimally and precisely as possible):
Just check the script/code, it seems kubelet will not register until dynamic config is there ( enabled all alpha features by default).
I met same problem
What do you mean "dynamic config" ?
I think there's a PR that make kubelet can update configuration dynamically, similar to what we had done in scheduler. But did not find PR :(. I'm checking the code.
Should be this one: /cc /sig node
that shouldn't be the case. As long as you can get a client and a nodeName you should be able to register. What specific error are you seeing that suggests it's dynamic config? Also why does local-up-cluster enable all alpha features by default? That seems wrong to me.
Not sure why, but it really does:
I think this should be caused by /cc
, sorry for the disturbing :). Regarding all alpha features, I prefer to enable is by default: local-up-cluster is a kind of test environment for developer, enabling all alpha features maybe helpful to catch up potential feature interactions.
I would not expect all alpha features on by default in local up cluster
Neither would I. Developers, by default, shouldn't have to wonder if bugs are their fault or if they come from unstable alpha components. Users, by default, shouldn't have to deal with unstable alpha features either.
/ , got that; alpha components is unstable, do NOT cover everything for it.
met same problem


Question: <ins> #!/bin/bash # Copyright 2014 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -o errexit set -o nounset set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE}")/.. source "${KUBE_ROOT}/hack/lib/init.sh" kube::golang::setup_env "${KUBE_ROOT}/hack/build-go.sh" cmd/gendocs # Get the absolute path of the directory component of a file, i.e. the # absolute path of the dirname of $1. get_absolute_dirname() { echo "$(cd "$(dirname "$1")" && pwd)" } # Detect the OS name/arch so that we can find our binary case "$(uname -s)" in Darwin) host_os=darwin ;; Linux) host_os=linux ;; *) echo "Unsupported host OS.  Must be Linux or Mac OS X." >&2 exit 1 ;; esac case "$(uname -m)" in x86_64*) host_arch=amd64 ;; i?86_64*) host_arch=amd64 ;; amd64*) host_arch=amd64 ;; arm*) host_arch=arm ;; i?86*) host_arch=x86 ;; *) echo "Unsupported host arch. Must be x86_64, 386 or arm." >&2 exit 1 ;; esac # Find binary locations=( "${KUBE_ROOT}/_output/dockerized/bin/${host_os}/${host_arch}/gendocs" "${KUBE_ROOT}/_output/local/bin/${host_os}/${host_arch}/gendocs" "${KUBE_ROOT}/platforms/${host_os}/${host_arch}/gendocs" ) gendocs=$( (ls -t "${locations[@]}" 2>/dev/null || true) | head -1 ) if [[ ! -x "$gendocs" ]]; then { echo "It looks as if you don't have a compiled gendocs binary" echo echo "If you are running from a clone of the git repo, please run" echo "'./hack/build-go.sh cmd/gendocs'." } >&2 exit 1 fi KUBECTL_DOC="docs/kubectl.md" echo "diffing ${KUBECTL_DOC} against generated output from ${gendocs}" "${gendocs}" | diff "${KUBE_ROOT}/${KUBECTL_DOC}" - && echo "${KUBECTL_DOC} up to date." || { echo "${KUBECTL_DOC} is out of date. Please run ${gendocs} > ${KUBECTL_DOC}" exit 1 } </ins>

Manually updated docs have a tendency to drift from actual implementation. It shouldn't be too hard to add a script to generate the markdown from a . We could then run the script manually or as a commit hook when changes are made to pkg/kubectl.
A big chunk of that is cut/paste from the help output anyway.. Brendan On Jan 9, 2015 4:20 PM, "Jeff Lowdermilk" wrote:
Why are we including the help output? Seems like cobra should do that, or we should just tell the user to ask for help if they want it.
It's useful to see generated markdown of help output for the full command tree, e.g. if you want to link to the syntax on irc or something. Anyway, this was quite easy to do, I'll have a PR shortly.


Question: Subsystem:         KubeletSubsystem, Name:              ConfigErrorKey, Help:              "This metric is true (1) if the node is experiencing a configuration-related error, false (0) otherwise.", <del> DeprecatedVersion: "1.23.0", </del> <ins> DeprecatedVersion: "1.22.0", </ins> StabilityLevel:    metrics.ALPHA, }, )

This shipped in 1.22 as deprecated in 1.22. As a hacky fix to unbreak tests on 1.23 dev, I bumped the deprecation policy one release: We will need to revert that before 1.23 ships and figure out a better solution. /kind cleanup /milestone v1.23 /sig node
/triage accepted
I want to try it. Can I take this issue?
I have explained how to fix this on PR:
/reopen reverted in as fix did not work
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/cc
The Kubernetes project currently lacks enough contributors to adequately respond to all issues and PRs. This bot triages issues and PRs according to the following rules: After 90d of inactivity, is applied After 30d of inactivity since was applied, is applied After 30d of inactivity since was applied, the issue is closed You can: Mark this issue or PR as fresh with Mark this issue or PR as rotten with Close this issue or PR with Offer to help out with [Issue Triage][1] Please send feedback to sig-contributor-experience at . /lifecycle stale [1]:
/remove-lifecycle stale
Hi, Im the bug triage shadow. Since we're in 1.23 milestone code freeze, could we move this issue to milestone 1.24?
this is critical for 1.23 and I think it just got forgotten... we need to get this fixed before test freeze
/priority important-soon


Question: "net.ipv4.ip_unprivileged_port_start", } <ins> var safeSysctlsIncludeReservedPorts = []string{ "kernel.shm_rmid_forced", "net.ipv4.ip_local_port_range", "net.ipv4.tcp_syncookies", "net.ipv4.ping_group_range", "net.ipv4.ip_unprivileged_port_start", "net.ipv4.ip_local_reserved_ports", } </ins> // SafeSysctlAllowlist returns the allowlist of safe sysctls and safe sysctl patterns (ending in *). // // A sysctl is called safe iff // - it is namespaced in the container or the pod // - it is isolated, i.e. has no influence on any other pod on the same node. func SafeSysctlAllowlist() []string { <ins> if goruntime.GOOS == "linux" { // make sure we're on a new enough kernel that the ip_local_reserved_ports sysctl is namespaced kernelVersion, err := getKernelVersion() if err != nil { klog.ErrorS(err, "Failed to get kernel version, dropping net.ipv4.ip_local_reserved_ports from safe sysctl list") return safeSysctls } if kernelVersion.LessThan(version.MustParseGeneric(ipLocalReservedPortsMinNamespacedKernelVersion)) { klog.ErrorS(nil, "Kernel version is too old, dropping net.ipv4.ip_local_reserved_ports from safe sysctl list", "kernelVersion", kernelVersion) return safeSysctls } } return safeSysctlsIncludeReservedPorts } func getKernelVersion() (*version.Version, error) { </ins> kernelVersionStr, err := ipvs.NewLinuxKernelHandler().GetKernelVersion() if err != nil { <del> klog.ErrorS(err, "Failed to get kernel version.") return safeSysctls </del> <ins> return nil, fmt.Errorf("failed to get kernel version: %w", err) </ins> } kernelVersion, err := version.ParseGeneric(kernelVersionStr) if err != nil { <del> klog.ErrorS(err, "Failed to parse kernel version.") return safeSysctls } // ip_local_reserved_ports has been changed to namesapced since kernel v3.16. // refer to https://github.com/torvalds/linux/commit/122ff243f5f104194750ecbc76d5946dd1eec934. if kernelVersion.LessThan(version.MustParseGeneric(ipLocalReservedPortsMinNamespacedKernelVersion)) { return safeSysctls } return []string{ "kernel.shm_rmid_forced", "net.ipv4.ip_local_port_range", "net.ipv4.tcp_syncookies", "net.ipv4.ping_group_range", "net.ipv4.ip_unprivileged_port_start", "net.ipv4.ip_local_reserved_ports", </del> <ins> return nil, fmt.Errorf("failed to parse kernel version: %w", err) </ins> } <ins> return kernelVersion, nil </ins> }

E0320 18:35:17. 4656 ] "Failed to get kernel version." err="error reading osrelease file "/proc/sys/kernel/osrelease": open /proc/sys/kernel/osrelease: The system cannot find the path specified." handle windows node correctly. Run kubelet v1.27 beta+ on windows. This is an example v1.27.0-beta* N/A windows <details</details<details</details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/sig node windows The issue came from
/assign
This Issue/PR has been marked as release blocker. Please have the fix merged by eod today otherwise the feature has to be reverted. For the Issue/PR owner: please have a revert/feature disable PR open today in case the fix failed to get in. Thank you! See the -- Kubernetes 1.27 release manager


Question: func TestCheckGracefulDelete(t *testing.T) { defaultGracePeriod := int64(30) tcs := []struct { <del> in          *api.Pod gracePeriod int64 </del> <ins> name              string pod               *api.Pod deleteGracePeriod *int64 gracePeriod       int64 </ins> }{ { <del> in: &api.Pod{ </del> <ins> name: "in pending phase with has node name", pod: &api.Pod{ </ins> Spec:   api.PodSpec{NodeName: "something"}, Status: api.PodStatus{Phase: api.PodPending}, }, <del> gracePeriod: defaultGracePeriod, </del> <ins> deleteGracePeriod: &defaultGracePeriod, gracePeriod:       defaultGracePeriod, </ins> }, { <del> in: &api.Pod{ </del> <ins> name: "in failed phase with has node name", pod: &api.Pod{ </ins> Spec:   api.PodSpec{NodeName: "something"}, Status: api.PodStatus{Phase: api.PodFailed}, }, <del> gracePeriod: 0, </del> <ins> deleteGracePeriod: &defaultGracePeriod, gracePeriod:       0, </ins> }, { <del> in: &api.Pod{ </del> <ins> name: "in failed phase", pod: &api.Pod{ </ins> Spec:   api.PodSpec{}, Status: api.PodStatus{Phase: api.PodPending}, }, <del> gracePeriod: 0, </del> <ins> deleteGracePeriod: &defaultGracePeriod, gracePeriod:       0, </ins> }, { <del> in: &api.Pod{ </del> <ins> name: "in succeeded phase", pod: &api.Pod{ </ins> Spec:   api.PodSpec{}, Status: api.PodStatus{Phase: api.PodSucceeded}, }, <del> gracePeriod: 0, </del> <ins> deleteGracePeriod: &defaultGracePeriod, gracePeriod:       0, </ins> }, { <del> in: &api.Pod{ </del> <ins> name: "no phase", pod: &api.Pod{ </ins> Spec:   api.PodSpec{}, Status: api.PodStatus{}, }, <del> gracePeriod: 0, </del> <ins> deleteGracePeriod: &defaultGracePeriod, gracePeriod:       0, }, { name: "has negative grace period", pod: &api.Pod{ Spec: api.PodSpec{ NodeName:                      "something", TerminationGracePeriodSeconds: utilpointer.Int64(-1), }, Status: api.PodStatus{}, }, gracePeriod: 1, </ins> }, } for _, tc := range tcs { <del> out := &metav1.DeleteOptions{GracePeriodSeconds: &defaultGracePeriod} Strategy.CheckGracefulDelete(genericapirequest.NewContext(), tc.in, out) if out.GracePeriodSeconds == nil { t.Errorf("out grace period was nil but supposed to be %v", tc.gracePeriod) } if *(out.GracePeriodSeconds) != tc.gracePeriod { t.Errorf("out grace period was %v but was expected to be %v", *out, tc.gracePeriod) } </del> <ins> t.Run(tc.name, func(t *testing.T) { out := &metav1.DeleteOptions{} if tc.deleteGracePeriod != nil { out.GracePeriodSeconds = utilpointer.Int64(*tc.deleteGracePeriod) } Strategy.CheckGracefulDelete(genericapirequest.NewContext(), tc.pod, out) if out.GracePeriodSeconds == nil { t.Errorf("out grace period was nil but supposed to be %v", tc.gracePeriod) } if *(out.GracePeriodSeconds) != tc.gracePeriod { t.Errorf("out grace period was %v but was expected to be %v", *out, tc.gracePeriod) } }) </ins> } }

<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: Some k8s client tried to evict a pod by setting negative value for deletionGracePeriodSeconds. But all eventual delete calls (and force-delete calls) made by kubelet, pod-garbage-collector, etc were not able to delete that pod, because apiserver doesnt allow deletionGracePeriodSeconds to be increased once it is set. So force-delete (which set it to 0) doesn't take effect when it's already negative. Not even kubectl is able to call delete with negative grace-period. But curl with an even more negative grace-period value worked. What you expected to happen: force-delete works. How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: I think the issue is in the code below As the comment suggest, in phase 1, the was set to a negative value (meaning was in the past), and in phase 2, has to be set even lower, otherwise, it will be considered as pending graceful deletion and not performing the immediate delete. I think there're two possible fixes. not allowing setting in the past ( should be non-negative). In that case, we could convert to 0 if negative. that could be set in the past. In that case, in phase 2, we treat negative value of the same as 0, and allow immediate delete. Thoughts? Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig api-machinery
The label(s) cannot be applied, because the repository doesn't have them <detailsIn response to : /sig api-machinery Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign
/cc /triage accepted
No component in the system should ever treat the timestamp of DeletionTimestamp as authoritative.


Question: func TestCheckGracefulDelete(t *testing.T) { defaultGracePeriod := int64(30) tcs := []struct { <del> in          *api.Pod gracePeriod int64 </del> <ins> name              string pod               *api.Pod deleteGracePeriod *int64 gracePeriod       int64 </ins> }{ { <del> in: &api.Pod{ </del> <ins> name: "in pending phase with has node name", pod: &api.Pod{ </ins> Spec:   api.PodSpec{NodeName: "something"}, Status: api.PodStatus{Phase: api.PodPending}, }, <del> gracePeriod: defaultGracePeriod, </del> <ins> deleteGracePeriod: &defaultGracePeriod, gracePeriod:       defaultGracePeriod, </ins> }, { <del> in: &api.Pod{ </del> <ins> name: "in failed phase with has node name", pod: &api.Pod{ </ins> Spec:   api.PodSpec{NodeName: "something"}, Status: api.PodStatus{Phase: api.PodFailed}, }, <del> gracePeriod: 0, </del> <ins> deleteGracePeriod: &defaultGracePeriod, gracePeriod:       0, </ins> }, { <del> in: &api.Pod{ </del> <ins> name: "in failed phase", pod: &api.Pod{ </ins> Spec:   api.PodSpec{}, Status: api.PodStatus{Phase: api.PodPending}, }, <del> gracePeriod: 0, </del> <ins> deleteGracePeriod: &defaultGracePeriod, gracePeriod:       0, </ins> }, { <del> in: &api.Pod{ </del> <ins> name: "in succeeded phase", pod: &api.Pod{ </ins> Spec:   api.PodSpec{}, Status: api.PodStatus{Phase: api.PodSucceeded}, }, <del> gracePeriod: 0, </del> <ins> deleteGracePeriod: &defaultGracePeriod, gracePeriod:       0, </ins> }, { <del> in: &api.Pod{ </del> <ins> name: "no phase", pod: &api.Pod{ </ins> Spec:   api.PodSpec{}, Status: api.PodStatus{}, }, <del> gracePeriod: 0, </del> <ins> deleteGracePeriod: &defaultGracePeriod, gracePeriod:       0, }, { name: "has negative grace period", pod: &api.Pod{ Spec: api.PodSpec{ NodeName:                      "something", TerminationGracePeriodSeconds: utilpointer.Int64(-1), }, Status: api.PodStatus{}, }, gracePeriod: 1, </ins> }, } for _, tc := range tcs { <del> out := &metav1.DeleteOptions{GracePeriodSeconds: &defaultGracePeriod} Strategy.CheckGracefulDelete(genericapirequest.NewContext(), tc.in, out) if out.GracePeriodSeconds == nil { t.Errorf("out grace period was nil but supposed to be %v", tc.gracePeriod) } if *(out.GracePeriodSeconds) != tc.gracePeriod { t.Errorf("out grace period was %v but was expected to be %v", *out, tc.gracePeriod) } </del> <ins> t.Run(tc.name, func(t *testing.T) { out := &metav1.DeleteOptions{} if tc.deleteGracePeriod != nil { out.GracePeriodSeconds = utilpointer.Int64(*tc.deleteGracePeriod) } Strategy.CheckGracefulDelete(genericapirequest.NewContext(), tc.pod, out) if out.GracePeriodSeconds == nil { t.Errorf("out grace period was nil but supposed to be %v", tc.gracePeriod) } if *(out.GracePeriodSeconds) != tc.gracePeriod { t.Errorf("out grace period was %v but was expected to be %v", *out, tc.gracePeriod) } }) </ins> } }

In Kube, components are required to calculate the time period from when they observe the event in their local clock (i.e. 30s from now), not use the deletion timestamp. A negative value of deletion grace period should be rejected by the apiserver as an invalid value (bad request / invalid field).
attempts to take a missing or positive deletion grace period negative should be rejected, I agree. if a negative value is already persisted, I'd be inclined to allow a negative value in an update and clip to 0
As I mentioned in the PR, eviction is passing negatives through to the store: In kubectl, we specify the default value for GracePeriod is -1. In the case of delete, we simply don't set the value in the deleteOptions we send to the API. In the case of eviction, we send the -1 value, and that gets passed along to the store.
I don't see the -1 getting sent to the eviction API. It looks like values < 0 result in a nil value being sent to the API: I would not expect the API to accept a negative value as-is.
Looks like you are correct. Do the deletionOptions go through their own validation or through some kind of validation for the evict API? The original bug report mentions 'evict' though it's not clear if the eviction API was used or not.
ValidateDeleteOptions is called inside pod deletion, but did not check that the field was non-negative. I don't see any eviction-specific validation.
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-contributor-experience at . /lifecycle rotten
/lifecycle rotten
Im investigating the and . Surprisingly see you here...


Question: e := &Publisher{ client: cl, rootCA: rootCA, <del> queue:  workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "root-ca-cert-publisher"), </del> <ins> queue:  workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "root_ca_cert_publisher"), </ins> } if cl.CoreV1().RESTClient().GetRateLimiter() != nil { if err := metrics.RegisterMetricAndTrackRateLimiterUsage("root_ca_cert_publisher", cl.CoreV1().RESTClient().GetRateLimiter()); err != nil {

What happened: From : What you expected to happen: Metrics should register successfully. How to reproduce it (as minimally and precisely as possible): Run kubernetes-integration test Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g. from /etc/os-release): Kernel (e.g. ): Install tools: Others:
There are no sig labels on this issue. Please add a sig label by either: a sig: e.g., to notify the contributor experience sig, OR the label manually: e.g., to apply the label Note: Method 1 will trigger an email to the group. See the . The in method 1 has to be replaced with one of these: bugs, feature-requests, pr-reviews, test-failures, proposals. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details


Question: Labels: testSvcLabels, }, Spec: v1.ServiceSpec{ <del> Type: "ClusterIP", </del> <ins> Type: "LoadBalancer", </ins> Ports: []v1.ServicePort{{ Name:       "http", Protocol:   v1.ProtocolTCP, Port:       int32(80), TargetPort: intstr.FromInt(80), }}, <ins> LoadBalancerClass: utilpointer.String("example.com/internal-vip"), </ins> }, } _, err = cs.CoreV1().Services(ns).Create(ctx, &testService, metav1.CreateOptions{})

/kind failing-test <!-- If this is a flake, please add: /kind flake --/sig network seems to be related.
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign
/triage accepted


Question: // .. and other fields we don't care about.  Expand as necessary. } <del> // parseMetadataUUID reads JSON from OpenStack metadata server and parses </del> <ins> // parseMetadata reads JSON from OpenStack metadata server and parses </ins> // instance ID out of it. func parseMetadata(r io.Reader) (*Metadata, error) { var metadata Metadata

According to the : "The scheduler bypasses any nodes that don't have any topologySpreadConstraints[].topologyKey present." However, the is:  "The scheduler bypasses any nodes that any don't have all topologySpreadConstraints[].topologyKey present." This is a bug as the implementation is not following . We expect the code implementation to align with the documentation's description. Read the documentation and the code. /sig scheduling of the on the branch, and of the on the branch <detailsNA </details<details</details<details</details<details</details<details</details
This issue is currently awaiting triage. If a SIG or subproject determines this is a relevant issue, they will accept it by applying the label and provide further guidance. The label can be by org members by writing in a comment. <detailsInstructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
pr in case documentation is upheld
I dont't think the implementation of func is not as expected. Perhaps we can optimize the document a bit. /cc
I don't understand what the issue is trying to point out. -return true if the node has all required . -bypasses a node that don't have any topologySpreadConstraints[*].topologyKey. Nothing looks wrong?
I also think there is no problem with code logic. According to doc, if one node only have one label that equal to , this node should not be bypassed. This sentence may be ambiguous. In fact, a node with labels that match all should not be bypassed by scheduler.
So, where's the ambiguousness of the doc? Yes, and it matches the doc's explanation.
This is where there might be ambiguity. means that if node only have one matched label, this node will not be bypassed. But in fact the node labels must match all(not one of) topologySpreadConstraints[].topologyKey. Maybe I misunderstood it as well because English is not my native language. Ignore me if I'm wrong.
Okay, I got what you wanted to say. This is... an interesting English question.


Question: // .. and other fields we don't care about.  Expand as necessary. } <del> // parseMetadataUUID reads JSON from OpenStack metadata server and parses </del> <ins> // parseMetadata reads JSON from OpenStack metadata server and parses </ins> // instance ID out of it. func parseMetadata(r io.Reader) (*Metadata, error) { var metadata Metadata

For example, the sentence "I don't have any of those books"; I'd interpret it as "I have no book; I don't have book-a, don't have book-b... don't have book-z". On the other hand, the sentence "If any of the following conditions are not true, the alert is triggered", I'd interpret it as "if condition-a and b is true, but condition-c isn't true, the alert is triggered". I interpreted the topology spread's doc like the second one; if any of topologySpreadConstraints[].topologyKey aren't in the Node's label, the node is bypassed. I'm not sure which interpretation is natural for the native speakers (looks like ChatGPT has the same interpretation as me though), but at least we can say it's ambiguous. I guess it'd be clearer if we rephrase it to, for example, "The scheduler bypasses any nodes that is missing at least of of the required topologySpreadConstraints[].topologyKey in its label.". Feel free to raise a PR. /remove-kind bug /kind documentation
I agree that the sentence is a little ambiguous. But this is not a bug in the code. The implementation matches the original intent of the design. I suggest opening a bug in /website so that SIG Docs can advise on better wording.
/close
Closing this issue. <detailsIn response to : /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
Thanks for your swift response! We chose to post this here because we were unsure about the expected behavior and wanted to hear developers' opinions. Now that we have clarity, so we will open a bug issue in the /website.


Question: return framework.RunKubectl("exec", fmt.Sprintf("--namespace=%s", pod.Namespace), pod.Name, "--", "/bin/sh", "-c", bashExec) } <ins> // VerifyExecInPodSucceed verifies bash cmd in target pod succeed func VerifyExecInPodSucceed(pod *v1.Pod, bashExec string) { _, err := PodExec(pod, bashExec) if err != nil { if err, ok := err.(uexec.CodeExitError); ok { exitCode := err.ExitStatus() Expect(err).NotTo(HaveOccurred(), "%q should succeed, but failed with exit code %d and error message %q", bashExec, exitCode, err) } else { Expect(err).NotTo(HaveOccurred(), "%q should succeed, but failed with error message %q", bashExec, err) } } } // VerifyExecInPodFail verifies bash cmd in target pod fail with certain exit code func VerifyExecInPodFail(pod *v1.Pod, bashExec string, exitCode int) { _, err := PodExec(pod, bashExec) if err != nil { if err, ok := err.(uexec.CodeExitError); ok { actualExitCode := err.ExitStatus() Expect(actualExitCode).To(Equal(exitCode), "%q should fail with exit code %d, but failed with exit code %d and error message %q", bashExec, exitCode, actualExitCode, err) } else { Expect(err).NotTo(HaveOccurred(), "%q should fail with exit code %d, but failed with error message %q", bashExec, exitCode, err) } } Expect(err).To(HaveOccurred(), "%q should fail with exit code %d, but exit without error", bashExec, exitCode) } </ins> // KubeletCommand performs `start`, `restart`, or `stop` on the kubelet running on the node of the target pod and waits // for the desired statues.. // - First issues the command via `systemctl`

It would better to have e2e for iSCSI persistent volume. /sig storage
/assign
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
Stale issues rot after 30d of inactivity. Mark the issue as fresh with . Rotten issues close after an additional 30d of inactivity. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle rotten /remove-lifecycle stale
Rotten issues close after 30d of inactivity. Reopen the issue with . Mark the issue as fresh with . Send feedback to sig-testing, kubernetes/test-infra and/or . /close


Question: return result, nil } if resourceVersion < oldest { <del> return nil, errors.NewInternalError(fmt.Errorf("too old resource version: %d (%d)", resourceVersion, oldest)) </del> <ins> return nil, errors.NewGone(fmt.Sprintf("too old resource version: %d (%d)", resourceVersion, oldest)) </ins> } // Binary seatch the smallest index at which resourceVersion is not smaller than

We sorta fixed this-- now it's a real status error, but we left the status code 500, so it still produces stack traces and potentially confuses clients. we should figure out the correct status code.
I think there is a lot more going on related to this in . Clients are re-listing after the error but not rewatching.
/sub
It turned out this was not causing job e2e flakes. Just sending 500s that were confusing. So lowering priority.
How about a NewBadRequest error?
that certainly sounds like an improvement.


Question: } if err := d.writeCachedFile(filename, liveResources); err != nil { <del> glog.V(3).Infof("failed to write cache to %v due to %v", filename, err) </del> <ins> glog.V(1).Infof("failed to write cache to %v due to %v", filename, err) </ins> } return liveResources, nil

/kind bug What happened: kubectl became very slow, taking 30s to list pods etc. After going deep down the rabbit hole of debugging the controller components, I ran kubectl -v=8 which showed: What you expected to happen: Since caching is required for kubectl to respond in reasonable time and the issue isn't obvious, kubectl should at least print warnings in default log level. How to reproduce it (as minimally and precisely as possible): I got into the situation by running kubectl with sudo, but just chown'ing should be enough to reproduce this. Anything else we need to know?: Environment: kubectl v1.8.3
/sig cli
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale /lifecycle freeze


Question: cd /go/src/k8s.io/kubernetes # hack/verify-client-go.sh requires all dependencies exist in the GOPATH. <del> godep restore </del> <ins> # the retry helps avoid flakes while keeping total time bounded. godep restore || godep restore </ins> ./hack/install-etcd.sh make verify

Seen in issue Logs from : At the very least, we should retry godep restore a couple times given the heavy network dependencies it has. The network dependencies are sorta necessary... but they change rarely so we could also do some clever caching of unmodified clones that we put in place ahead of time to dodge network traffic outside of new deps being . cc
We could probably do a single retry and avoid most flakes. Godep restore takes a long time though, so if we retry too many times we'll run into timeout issues (but maybe that's fine). See also , tools/godep
Actually, it looks like this is only being done for an excluded verify step (). We should not do this for an excluded step, and we should not do for every PR.
can you explain why staging/ needs to function?
verify- also runs :
eventually gets published to . IIRC, it needs because otherwise a later fails ().
If we can run the verify script in order, then we can run verify-staging-client- after verify-, then we can save one .
verify- exits out early if Godeps/ or vendor/ hasn't changed.
Hmm, ok. How often will fail?
I'm seeing ~.5% failure rate over the last few days, and averaging ~2 minutes for the godep restore. Do we have to check in _vendor for the staging client code at all?
We run to find all the dependencies of clients, including k8s packages, so _vendor is not the reason we need .


Question: computealpha "google.golang.org/api/compute/v0.alpha" computebeta "google.golang.org/api/compute/v0.beta" computev1 "google.golang.org/api/compute/v1" <ins> "k8s.io/kubernetes/pkg/cloudprovider" </ins> ) func TestReadConfigFile(t *testing.T) {

Is this a BUG REPORT or FEATURE REQUEST?: As part of the initiative for external cloud providers (), we GetZoneByProviderID and GetZoneByNodeName. See for more details. The previous PR did not actually implement those methods for existing cloud providers in . These methods should be implemented by all cloud providers in the future. cc
Could someone add the label and the label for this issue please?
/sig cluster-lifecycle
/area cloudprovider
/assign wlan0
cc We need the following label - help-wanted and add it to the 1.8(?) milestone cc to implement it for their respective clouds.
I'll try to have a look by the end of the week.
Ok, I will take care of the Rackspace and OpenStack.
Thank you !
- Is there any listing where to see the already implemented cloud provider ?
No providers have this yet. Current PRs: Openstack: Rackspace: Please contribute if you have something ready :)
Sorry for the delay guys :)
Have you implemented them for gce?
Intending to work on it tonight, but if you have something I can work on azure
Yes, I have prepared for GCE.
Azure .
Cool, than you all! Any other providers we should implement this for in time for v1.8?
I'll take a look tomorrow to see if there's something left
Let's see which providers are left before closing this...
- I guess this one should be closed ?
If you say that all providers have this now, thanks, great!