) const ( <del> testNodeMonitorGracePeriod = 40 * time.Second </del> <ins> testNodeMonitorGracePeriod = 50 * time.Second </ins> testNodeStartupGracePeriod = 60 * time.Second testNodeMonitorPeriod      = 5 * time.Second testRateLimiterQPS         = float32(100000)


}, "NodeMonitorGracePeriod": { SchemaProps: spec.SchemaProps{ <del> Description: "nodeMontiorGracePeriod is the amount of time which we allow a running node to be unresponsive before marking it unhealthy. Must be N times more than kubelet's nodeStatusUpdateFrequency, where N means number of retries allowed for kubelet to post node status.", </del> <ins> Description: "nodeMontiorGracePeriod is the amount of time which we allow a running node to be unresponsive before marking it unhealthy. Must be N times more than kubelet's nodeStatusUpdateFrequency, where N means number of retries allowed for kubelet to post node status. This value should also be greater than the sum of HTTP2_PING_TIMEOUT_SECONDS and HTTP2_READ_IDLE_TIMEOUT_SECONDS.", </ins> Ref:         ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), }, },


// nodeMontiorGracePeriod is the amount of time which we allow a running node to be // unresponsive before marking it unhealthy. Must be N times more than kubelet's // nodeStatusUpdateFrequency, where N means number of retries allowed for kubelet <del> // to post node status. </del> <ins> // to post node status. This value should also be greater than the sum of // HTTP2_PING_TIMEOUT_SECONDS and HTTP2_READ_IDLE_TIMEOUT_SECONDS. </ins> NodeMonitorGracePeriod metav1.Duration // podEvictionTimeout is the grace period for deleting pods on failed nodes. PodEvictionTimeout metav1.Duration


"fmt" "math" "strconv" <del> "sync" </del> "time" "github.com/GoogleCloudPlatform/kubernetes/pkg/api"


Expect(badEvents).NotTo(BeNumerically(">", int(math.Floor(0.01*float64(totalPods))))) }) } <del> type Scalability struct { skip          bool totalPods     int podsPerMinion int rcsPerThread  int } scalabilityTests := []Scalability{ {totalPods: 500, podsPerMinion: 10, rcsPerThread: 5, skip: true}, {totalPods: 500, podsPerMinion: 10, rcsPerThread: 25, skip: true}, } for _, testArg := range scalabilityTests { // # of threads calibrate to totalPods threads := (testArg.totalPods / (testArg.podsPerMinion * testArg.rcsPerThread)) name := fmt.Sprintf( "should be able to launch %v pods, %v per minion, in %v rcs/thread.", testArg.totalPods, testArg.podsPerMinion, testArg.rcsPerThread) if testArg.skip { name = "[Skipped] " + name } itArg := testArg It(name, func() { podsLaunched := 0 var wg sync.WaitGroup wg.Add(threads) // Create queue of pending requests on the api server. for i := 0; i < threads; i++ { go func() { defer wg.Done() for i := 0; i < itArg.rcsPerThread; i++ { name := "my-short-lived-pod" + string(util.NewUUID()) n := itArg.podsPerMinion * minionCount expectNoError(RunRC(c, name, ns, "gcr.io/google_containers/pause:go", n)) podsLaunched += n Logf("Launched %v pods so far...", podsLaunched) err := DeleteRC(c, ns, name) expectNoError(err) } }() } // Wait for all the pods from all the RC's to return. wg.Wait() Logf("%v pods out of %v launched", podsLaunched, itArg.totalPods) }) } </del> })


<ins> /* Copyright 2015 Google Inc. All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package e2e import ( "fmt" "sync" "github.com/GoogleCloudPlatform/kubernetes/pkg/client" "github.com/GoogleCloudPlatform/kubernetes/pkg/fields" "github.com/GoogleCloudPlatform/kubernetes/pkg/labels" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) // This test suite can take a long time to run, so by default it is added to // the ginkgo.skip list (see driver.go). // To run this suite you must explicitly ask for it by setting the // -t/--test flag or ginkgo.focus flag. var _ = Describe("Scale", func() { var c *client.Client var minionCount int var RCName string var ns string BeforeEach(func() { var err error c, err = loadClient() expectNoError(err) minions, err := c.Nodes().List(labels.Everything(), fields.Everything()) expectNoError(err) minionCount = len(minions.Items) Expect(minionCount).NotTo(BeZero()) nsForTesting, err := createTestingNS("scale", c) ns = nsForTesting.Name expectNoError(err) }) AfterEach(func() { // Remove any remaining pods from this test if the // replication controller still exists and the replica count // isn't 0.  This means the controller wasn't cleaned up // during the test so clean it up here rc, err := c.ReplicationControllers(ns).Get(RCName) if err == nil && rc.Spec.Replicas != 0 { By("Cleaning up the replication controller") err := DeleteRC(c, ns, RCName) expectNoError(err) } By(fmt.Sprintf("Destroying namespace for this suite %v", ns)) if err := c.Namespaces().Delete(ns); err != nil { Failf("Couldn't delete ns %s", err) } }) type Scalability struct { skip          bool totalPods     int podsPerMinion int rcsPerThread  int } scalabilityTests := []Scalability{ {totalPods: 500, podsPerMinion: 10, rcsPerThread: 5, skip: true}, {totalPods: 500, podsPerMinion: 10, rcsPerThread: 25, skip: true}, } for _, testArg := range scalabilityTests { // # of threads calibrate to totalPods threads := (testArg.totalPods / (testArg.podsPerMinion * testArg.rcsPerThread)) name := fmt.Sprintf( "should be able to launch %v pods, %v per minion, in %v rcs/thread.", testArg.totalPods, testArg.podsPerMinion, testArg.rcsPerThread) if testArg.skip { name = "[Skipped] " + name } itArg := testArg It(name, func() { podsLaunched := 0 var wg sync.WaitGroup wg.Add(threads) // Create queue of pending requests on the api server. for i := 0; i < threads; i++ { go func() { defer GinkgoRecover() defer wg.Done() for i := 0; i < itArg.rcsPerThread; i++ { name := "my-short-lived-pod" + string(util.NewUUID()) n := itArg.podsPerMinion * minionCount expectNoError(RunRC(c, name, ns, "gcr.io/google_containers/pause:go", n)) podsLaunched += n Logf("Launched %v pods so far...", podsLaunched) err := DeleteRC(c, ns, name) expectNoError(err) } }() } // Wait for all the pods from all the RC's to return. wg.Wait() Logf("%v pods out of %v launched", podsLaunched, itArg.totalPods) }) } }) </ins>


if err != nil { return nil, "", err } <del> return aggregator.FilterSpecByPathsWithoutSideEffects(result, []string{"/apis/"}), etag, nil </del> <ins> group := specInfo.apiService.Spec.Group version := specInfo.apiService.Spec.Version return aggregator.FilterSpecByPathsWithoutSideEffects(result, []string{"/apis/" + group + "/" + version}), etag, nil </ins> }, cached.Result[*spec.Swagger]{Value: result, Etag: etag, Err: err}) specInfo.spec.Store(filteredResult) return err


SwaggerProps: spec.SwaggerProps{ Paths: &spec.Paths{ Paths: map[string]spec.PathItem{ <del> "/apis/apiservicegroup/v1": {}, </del> <ins> "/apis/apiservicegroup/v1/path1": {}, </ins> }, }, },


t.Error(err) } <del> expectPath(t, swagger, "/apis/apiservicegroup/v1") </del> <ins> expectPath(t, swagger, "/apis/apiservicegroup/v1/path1") </ins> expectPath(t, swagger, "/apis/apiregistration.k8s.io/v1") t.Log("Update APIService OpenAPI")


SwaggerProps: spec.SwaggerProps{ Paths: &spec.Paths{ Paths: map[string]spec.PathItem{ <del> "/apis/apiservicegroup/v2": {}, </del> <ins> "/apis/apiservicegroup/v1/path2": {}, </ins> }, }, },


} // Ensure that the if the APIService OpenAPI is updated, the // aggregated OpenAPI is also updated. <ins> expectPath(t, swagger, "/apis/apiservicegroup/v1/path2") expectNoPath(t, swagger, "/apis/apiservicegroup/v1/path1") expectPath(t, swagger, "/apis/apiregistration.k8s.io/v1") } // Tests that an APIService that registers OpenAPI will only have the OpenAPI // for its specific group version served registered. // See https://github.com/kubernetes/kubernetes/pull/123570 for full context. func TestAPIServiceOpenAPIServiceMismatch(t *testing.T) { mux := http.NewServeMux() var delegationHandlers []http.Handler delegate1 := &openAPIHandler{openapi: &spec.Swagger{ SwaggerProps: spec.SwaggerProps{ Paths: &spec.Paths{ Paths: map[string]spec.PathItem{ "/apis/foo/v1": {}, }, }, }, }} delegationHandlers = append(delegationHandlers, delegate1) s := buildAndRegisterSpecAggregator(delegationHandlers, mux) apiService := &v1.APIService{ Spec: v1.APIServiceSpec{ Group:   "apiservicegroup", Version: "v1", Service: &v1.ServiceReference{Name: "dummy"}, }, } apiService.Name = "apiservice" apiService2 := &v1.APIService{ Spec: v1.APIServiceSpec{ Group:   "apiservicegroup", Version: "v2", Service: &v1.ServiceReference{Name: "dummy2"}, }, } apiService2.Name = "apiservice2" handler := &openAPIHandler{openapi: &spec.Swagger{ SwaggerProps: spec.SwaggerProps{ Paths: &spec.Paths{ Paths: map[string]spec.PathItem{ "/apis/apiservicegroup/v1": {}, }, }, }, }} handler2 := &openAPIHandler{openapi: &spec.Swagger{ SwaggerProps: spec.SwaggerProps{ Paths: &spec.Paths{ Paths: map[string]spec.PathItem{ "/apis/a":                  {}, "/apis/apiservicegroup/v1": {}, "/apis/apiservicegroup/v2": {}, }, }, }, }} if err := s.AddUpdateAPIService(apiService, handler); err != nil { t.Error(err) } if err := s.UpdateAPIServiceSpec(apiService.Name); err != nil { t.Error(err) } if err := s.AddUpdateAPIService(apiService2, handler2); err != nil { t.Error(err) } if err := s.UpdateAPIServiceSpec(apiService2.Name); err != nil { t.Error(err) } swagger, err := fetchOpenAPI(mux) if err != nil { t.Error(err) } expectPath(t, swagger, "/apis/apiservicegroup/v1") </ins> expectPath(t, swagger, "/apis/apiservicegroup/v2") <ins> expectPath(t, swagger, "/apis/apiregistration.k8s.io/v1") t.Logf("Remove APIService %s", apiService.Name) s.RemoveAPIService(apiService.Name) swagger, err = fetchOpenAPI(mux) if err != nil { t.Error(err) } // Ensure that the if the APIService is added then removed, the OpenAPI disappears from the aggregated OpenAPI as well. </ins> expectNoPath(t, swagger, "/apis/apiservicegroup/v1") expectPath(t, swagger, "/apis/apiregistration.k8s.io/v1") }


apiServiceSuccess := &v1.APIService{ Spec: v1.APIServiceSpec{ <ins> Group:   "success", Version: "v1", </ins> Service: &v1.ServiceReference{Name: "dummy2"}, }, }


apiService := &v1.APIService{ Spec: v1.APIServiceSpec{ <ins> Group:   "apiservicegroup", Version: "v1", </ins> Service: &v1.ServiceReference{Name: "dummy"}, }, }


apiServiceFailed := &v1.APIService{ Spec: v1.APIServiceSpec{ <ins> Group:   "failed", Version: "v1", </ins> Service: &v1.ServiceReference{Name: "dummy"}, }, }


<ins> #!/usr/bin/env bash # Copyright 2024 The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. set -o errexit set -o nounset set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. cd "${KUBE_ROOT}" source hack/lib/init.sh # NOTE: Please do NOT add any to this list!! # # We are aiming to consolidate on: registry.k8s.io/e2e-test-images/agnhost # The sources for which are in test/images/agnhost. # If agnhost is missing functionality for your tests, please reach out to SIG Testing. kube::util::read-array PERMITTED_IMAGES < <(sed '/^#/d' ./test/images/.permitted-images) # get current list of images, ignoring tags echo "Getting e2e image list ..." make WHAT=test/e2e/e2e.test e2e_test="$(kube::util::find-binary e2e.test)" kube::util::read-array IMAGES < <("${e2e_test}" --list-images | sed -E 's/^(.+):[^:]+$/1/' | LC_ALL=C sort -u) # diff versus known permitted images ret=0 >&2 echo "Diffing e2e image list ..." diff -Naupr <(printf '%sn' "${IMAGES[@]}") <(printf '%sn' "${PERMITTED_IMAGES[@]}") || ret=$? if [[ $ret -eq 0 ]]; then >&2 echo "PASS: e2e images used are OK." else >&2 echo "FAIL: e2e images do not match the approved list!" >&2 echo "" >&2 echo "Please use registry.k8s.io/e2e-test-images/agnhost wherever possible, we are consolidating test images." >&2 echo "See: test/images/agnhost/README.md" >&2 echo "" >&2 echo "You can reach out to https://git.k8s.io/community/sig-testing for help." exit 1 fi </ins>


<ins> # NOTE: Please do NOT add any to this list!! # # We are aiming to consolidate on: registry.k8s.io/e2e-test-images/agnhost # The sources for which are in test/images/agnhost. # If agnhost is missing functionality for your tests, please reach out to SIG Testing. gcr.io/authenticated-image-pulling/alpine gcr.io/authenticated-image-pulling/windows-nanoserver gcr.io/k8s-authenticated-test/agnhost invalid.registry.k8s.io/invalid/alpine registry.k8s.io/build-image/distroless-iptables registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver registry.k8s.io/e2e-test-images/agnhost registry.k8s.io/e2e-test-images/apparmor-loader registry.k8s.io/e2e-test-images/busybox registry.k8s.io/e2e-test-images/cuda-vector-add registry.k8s.io/e2e-test-images/httpd registry.k8s.io/e2e-test-images/ipc-utils registry.k8s.io/e2e-test-images/jessie-dnsutils registry.k8s.io/e2e-test-images/kitten registry.k8s.io/e2e-test-images/nautilus registry.k8s.io/e2e-test-images/nginx registry.k8s.io/e2e-test-images/node-perf/npb-ep registry.k8s.io/e2e-test-images/node-perf/npb-is registry.k8s.io/e2e-test-images/node-perf/tf-wide-deep registry.k8s.io/e2e-test-images/nonewprivs registry.k8s.io/e2e-test-images/nonroot registry.k8s.io/e2e-test-images/perl registry.k8s.io/e2e-test-images/redis registry.k8s.io/e2e-test-images/regression-issue-74839 registry.k8s.io/e2e-test-images/resource-consumer registry.k8s.io/e2e-test-images/sample-apiserver registry.k8s.io/e2e-test-images/volume/iscsi registry.k8s.io/e2e-test-images/volume/nfs registry.k8s.io/etcd registry.k8s.io/pause registry.k8s.io/prometheus-dummy-exporter registry.k8s.io/prometheus-to-sd registry.k8s.io/sd-dummy-exporter registry.k8s.io/sig-storage/csi-attacher registry.k8s.io/sig-storage/csi-external-health-monitor-controller registry.k8s.io/sig-storage/csi-node-driver-registrar registry.k8s.io/sig-storage/csi-provisioner registry.k8s.io/sig-storage/csi-resizer registry.k8s.io/sig-storage/csi-snapshotter registry.k8s.io/sig-storage/hello-populator registry.k8s.io/sig-storage/hostpathplugin registry.k8s.io/sig-storage/livenessprobe registry.k8s.io/sig-storage/nfs-provisioner registry.k8s.io/sig-storage/volume-data-source-validator </ins>


}, []string{"resource"}, ) <ins> WatchCacheReadWait = compbasemetrics.NewHistogramVec( &compbasemetrics.HistogramOpts{ Namespace:      namespace, Subsystem:      subsystem, Name:           "read_wait_seconds", Help:           "Histogram of time spent waiting for a watch cache to become fresh.", StabilityLevel: compbasemetrics.ALPHA, Buckets:        []float64{0.005, 0.025, 0.05, 0.1, 0.2, 0.4, 0.6, 0.8, 1.0, 1.25, 1.5, 2, 3}, }, []string{"resource"}) </ins> ) var registerMetrics sync.Once


legacyregistry.MustRegister(watchCacheCapacityDecreaseTotal) legacyregistry.MustRegister(WatchCacheCapacity) legacyregistry.MustRegister(WatchCacheInitializations) <ins> legacyregistry.MustRegister(WatchCacheReadWait) </ins> }) }


// You HAVE TO explicitly call w.RUnlock() after this function. func (w *watchCache) waitUntilFreshAndBlock(ctx context.Context, resourceVersion uint64) error { startTime := w.clock.Now() <ins> defer func() { if resourceVersion > 0 { metrics.WatchCacheReadWait.WithContext(ctx).WithLabelValues(w.groupResource.String()).Observe(w.clock.Since(startTime).Seconds()) } }() </ins> // In case resourceVersion is 0, we accept arbitrarily stale result. // As a result, the condition in the below for loop will never be


"k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/storage" <ins> "k8s.io/apiserver/pkg/storage/cacher/metrics" </ins> utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/tools/cache" featuregatetesting "k8s.io/component-base/featuregate/testing" <ins> k8smetrics "k8s.io/component-base/metrics" "k8s.io/component-base/metrics/testutil" </ins> "k8s.io/utils/clock" testingclock "k8s.io/utils/clock/testing" )


store.updateCache(add) } } <ins> func TestHistogramCacheReadWait(t *testing.T) { registry := k8smetrics.NewKubeRegistry() if err := registry.Register(metrics.WatchCacheReadWait); err != nil { t.Errorf("unexpected error: %v", err) } ctx := context.Background() testedMetrics := "apiserver_watch_cache_read_wait_seconds" store := newTestWatchCache(2, &cache.Indexers{}) defer store.Stop() // In background, update the store. go func() { if err := store.Add(makeTestPod("foo", 2)); err != nil { t.Errorf("unexpected error: %v", err) } if err := store.Add(makeTestPod("bar", 5)); err != nil { t.Errorf("unexpected error: %v", err) } }() testCases := []struct { desc            string resourceVersion uint64 want            string }{ { desc:            "resourceVersion is non-zero", resourceVersion: 5, want: ` # HELP apiserver_watch_cache_read_wait_seconds [ALPHA] Histogram of time spent waiting for a watch cache to become fresh. # TYPE apiserver_watch_cache_read_wait_seconds histogram apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.005"} 1 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.025"} 1 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.05"} 1 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.1"} 1 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.2"} 1 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.4"} 1 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.6"} 1 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="0.8"} 1 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="1"} 1 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="1.25"} 1 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="1.5"} 1 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="2"} 1 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="3"} 1 apiserver_watch_cache_read_wait_seconds_bucket{resource="pods",le="+Inf"} 1 apiserver_watch_cache_read_wait_seconds_sum{resource="pods"} 0 apiserver_watch_cache_read_wait_seconds_count{resource="pods"} 1 `, }, { desc:            "resourceVersion is 0", resourceVersion: 0, want:            ``, }, } for _, test := range testCases { t.Run(test.desc, func(t *testing.T) { defer registry.Reset() if _, _, _, err := store.WaitUntilFreshAndGet(ctx, test.resourceVersion, "prefix/ns/bar"); err != nil { t.Errorf("unexpected error: %v", err) } if err := testutil.GatherAndCompare(registry, strings.NewReader(test.want), testedMetrics); err != nil { t.Errorf("unexpected error: %v", err) } }) } } </ins>


MINION_TAG="${INSTANCE_PREFIX}-minion" MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}})) MINION_IP_RANGES=($(eval echo "10.245.{1..${NUM_MINIONS}}.0/24")) <del> MINION_SCOPES="" </del> <ins> MINION_SCOPES="storage-ro,compute-rw" </ins> # Increase the sleep interval value if concerned about API rate limits. 3, in seconds, is the default. POLL_SLEEP_INTERVAL=3 PORTAL_NET="10.0.0.0/16"


<ins> apiVersion: v1beta1 desiredState: manifest: containers: - image: kubernetes/pause name: testpd volumeMounts: - mountPath: "/testpd" name: "testpd" id: testpd version: v1beta1 volumes: - name: testpd source: persistentDisk: # This GCE PD must already exist and be formatted ext4 pdName: "%insert_pd_name_here%" fsType: ext4 id: testpd kind: Pod labels: test: testpd </ins>


<ins> #!/bin/bash # Copyright 2014 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Launches a container and verifies it can be reached. Assumes that # we're being called by hack/e2e-test.sh (we use some env vars it sets up). # TODO: Convert uses of gcutil to gcloud set -o errexit set -o nounset set -o pipefail KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. source "${KUBE_ROOT}/cluster/kube-env.sh" source "${KUBE_ROOT}/cluster/$KUBERNETES_PROVIDER/util.sh" if [[ "$KUBERNETES_PROVIDER" != "gce" ]]; then echo "PD test is only run for GCE" return 0 fi disk_name="e2e-$(date +%H-%M-%s)" config="/tmp/${disk_name}.yaml" function teardown() { echo "Cleaning up test artifacts" ${KUBECFG} delete pods/testpd rm -rf ${config} echo "Waiting for disk to become unmounted" sleep 20 gcutil deletedisk -f --zone=${ZONE} ${disk_name} } trap "teardown" EXIT perl -p -e "s/%.*%/${disk_name}/g" ${KUBE_ROOT}/examples/gce-pd/testpd.yaml > ${config} # Create and mount the disk. gcutil adddisk --size_gb=10 --zone=${ZONE} ${disk_name} gcutil attachdisk --disk ${disk_name} ${MASTER_NAME} gcutil ssh ${MASTER_NAME} sudo rm -rf /mnt/tmp gcutil ssh ${MASTER_NAME} sudo mkdir /mnt/tmp gcutil ssh ${MASTER_NAME} sudo /usr/share/google/safe_format_and_mount /dev/disk/by-id/google-${disk_name} /mnt/tmp gcutil ssh ${MASTER_NAME} sudo umount /mnt/tmp gcloud compute instances detach-disk --disk ${disk_name} --zone ${ZONE} ${MASTER_NAME} ${KUBECFG} -c ${config} create pods pod_id_list=$($KUBECFG '-template={{range.items}}{{.id}} {{end}}' -l test=testpd list pods) # Pod turn up on a clean cluster can take a while for the docker image pull. all_running=0 for i in $(seq 1 24); do echo "Waiting for pods to come up." sleep 5 all_running=1 for id in $pod_id_list; do current_status=$($KUBECFG -template '{{.currentState.status}}' get pods/$id) || true if [[ "$current_status" != "Running" ]]; then all_running=0 break fi done if [[ "${all_running}" == 1 ]]; then break fi done if [[ "${all_running}" == 0 ]]; then echo "Pods did not come up in time" exit 1 fi </ins>


"regexp" "strings" "syscall" <ins> "github.com/golang/glog" </ins> ) const MOUNT_MS_BIND = syscall.MS_BIND


// Wraps syscall.Mount() func (mounter *DiskMounter) Mount(source string, target string, fstype string, flags uintptr, data string) error { <ins> glog.V(5).Infof("Mounting %s %s %s %d %s", source, target, fstype, flags, data) </ins> return syscall.Mount(source, target, fstype, flags, data) }


// Update uncertain volumes - the new markVolumeOpts may have updated information. // Especially reconstructed volumes (marked as uncertain during reconstruction) need // an update. <del> updateUncertainVolume = utilfeature.DefaultFeatureGate.Enabled(features.SELinuxMountReadWriteOncePod) && podObj.volumeMountStateForPod == operationexecutor.VolumeMountUncertain </del> <ins> updateUncertainVolume = utilfeature.DefaultFeatureGate.Enabled(features.NewVolumeManagerReconstruction) && podObj.volumeMountStateForPod == operationexecutor.VolumeMountUncertain </ins> } if !podExists || updateUncertainVolume { // Add new mountedPod or update existing one.


utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/kubernetes" "k8s.io/client-go/transport" <ins> "k8s.io/component-base/tracing" </ins> "k8s.io/component-base/version" v1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" v1helper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper"


// rejectForwardingRedirects is whether to allow to forward redirect response rejectForwardingRedirects bool <ins> // tracerProvider is used to wrap the proxy transport and handler with tracing tracerProvider tracing.TracerProvider </ins> } // Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.


openAPIV3Config:            c.GenericConfig.OpenAPIV3Config, proxyCurrentCertKeyContent: func() (bytes []byte, bytes2 []byte) { return nil, nil }, rejectForwardingRedirects:  c.ExtraConfig.RejectForwardingRedirects, <ins> tracerProvider:             c.GenericConfig.TracerProvider, </ins> } // used later  to filter the served resource by those that have expired.


proxyTransportDial:         s.proxyTransportDial, serviceResolver:            s.serviceResolver, rejectForwardingRedirects:  s.rejectForwardingRedirects, <ins> tracerProvider:             s.tracerProvider, </ins> } proxyHandler.updateAPIService(apiService) if s.openAPIAggregationController != nil {


"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" endpointmetrics "k8s.io/apiserver/pkg/endpoints/metrics" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" <ins> genericfeatures "k8s.io/apiserver/pkg/features" utilfeature "k8s.io/apiserver/pkg/util/feature" </ins> utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol" apiserverproxyutil "k8s.io/apiserver/pkg/util/proxy" "k8s.io/apiserver/pkg/util/x509metrics" "k8s.io/client-go/transport" <ins> "k8s.io/component-base/tracing" </ins> "k8s.io/klog/v2" apiregistrationv1api "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" apiregistrationv1apihelper "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1/helper"


// reject to forward redirect response rejectForwardingRedirects bool <ins> // tracerProvider is used to wrap the proxy transport and handler with tracing tracerProvider tracing.TracerProvider </ins> } type proxyHandlingInfo struct {


proxyRoundTripper = transport.NewAuthProxyRoundTripper(user.GetName(), user.GetGroups(), user.GetExtra(), proxyRoundTripper) <ins> if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServerTracing) && !upgrade { tracingWrapper := tracing.WrapperFor(r.tracerProvider) proxyRoundTripper = tracingWrapper(proxyRoundTripper) } </ins> // If we are upgrading, then the upgrade path tries to use this request with the TLS config we provide, but it does // NOT use the proxyRoundTripper.  It's a direct dial that bypasses the proxyRoundTripper.  This means that we have to // attach the "correct" user headers to the request ahead of time.


"golang.org/x/net/websocket" <ins> "go.opentelemetry.io/otel/propagation" sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/sdk/trace/tracetest" "go.opentelemetry.io/otel/trace" </ins> metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/proxy" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/authentication/user" <ins> "k8s.io/apiserver/pkg/endpoints/filters" </ins> genericapirequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/server/egressselector" utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"


"k8s.io/component-base/metrics/legacyregistry" apiregistration "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" "k8s.io/utils/pointer" <ins> "k8s.io/utils/ptr" </ins> ) type targetHTTPHandler struct {


} <ins> func TestTracerProvider(t *testing.T) { fakeRecorder := tracetest.NewSpanRecorder() otelTracer := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(fakeRecorder)) target := &targetHTTPHandler{} user := &user.DefaultInfo{ Name:   "username", Groups: []string{"one", "two"}, } path := "/request/path" apiService := &apiregistration.APIService{ ObjectMeta: metav1.ObjectMeta{Name: "v1.foo"}, Spec: apiregistration.APIServiceSpec{ Service:  &apiregistration.ServiceReference{Name: "test-service", Namespace: "test-ns", Port: ptr.To(int32(443))}, Group:    "foo", Version:  "v1", CABundle: testCACrt, }, Status: apiregistration.APIServiceStatus{ Conditions: []apiregistration.APIServiceCondition{ {Type: apiregistration.Available, Status: apiregistration.ConditionTrue}, }, }, } targetServer := httptest.NewUnstartedServer(target) serviceCert := svcCrt if cert, err := tls.X509KeyPair(serviceCert, svcKey); err != nil { t.Fatalf("TestTracerProvider: failed to parse key pair: %v", err) } else { targetServer.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} } targetServer.StartTLS() defer targetServer.Close() serviceResolver := &mockedRouter{destinationHost: targetServer.Listener.Addr().String()} handler := &proxyHandler{ localDelegate:              http.NewServeMux(), serviceResolver:            serviceResolver, proxyCurrentCertKeyContent: func() ([]byte, []byte) { return emptyCert(), emptyCert() }, tracerProvider:             otelTracer, } server := httptest.NewServer(contextHandler(filters.WithTracing(handler, otelTracer), user)) defer server.Close() handler.updateAPIService(apiService) curr := handler.handlingInfo.Load().(proxyHandlingInfo) handler.handlingInfo.Store(curr) var propagator propagation.TraceContext req, err := http.NewRequest(http.MethodGet, server.URL+path, nil) if err != nil { t.Errorf("expected new request: %v", err) return } t.Logf("Sending request: %v", req) _, err = http.DefaultClient.Do(req) if err != nil { t.Errorf("http request failed: %v", err) return } t.Log("Ensure the target server received the traceparent header") id, ok := target.headers["Traceparent"] if !ok { t.Error("expected traceparent header") return } t.Log("Get the span context from the traceparent header") h := http.Header{ "Traceparent": id, } ctx := propagator.Extract(context.Background(), propagation.HeaderCarrier(h)) span := trace.SpanFromContext(ctx) t.Log("Ensure that the span context is valid and remote") if !span.SpanContext().IsValid() { t.Error("expected valid span context") return } if !span.SpanContext().IsRemote() { t.Error("expected remote span context") return } t.Log("Ensure that the span ID and trace ID match the expected values") expectedSpanCtx := fakeRecorder.Ended()[0].SpanContext() if expectedSpanCtx.TraceID() != span.SpanContext().TraceID() { t.Errorf("expected trace id to match. expected: %v, but got %v", expectedSpanCtx.TraceID(), span.SpanContext().TraceID()) return } if expectedSpanCtx.SpanID() != span.SpanContext().SpanID() { t.Errorf("expected span id to match. expected: %v, but got: %v", expectedSpanCtx.SpanID(), span.SpanContext().SpanID()) return } t.Log("Ensure that the expected spans were recorded when sending a request through the proxy") expectedSpanNames := []string{"HTTP GET", "GET"} spanNames := []string{} for _, span := range fakeRecorder.Ended() { spanNames = append(spanNames, span.Name()) } if e, a := expectedSpanNames, spanNames; !reflect.DeepEqual(e, a) { t.Errorf("expected span names %v, got %v", e, a) return } } </ins> func TestNewRequestForProxyWithAuditID(t *testing.T) { tests := []struct { name    string


"fmt" "math/rand" "sync" <ins> "sync/atomic" </ins> "testing" "time"


close(stopCh) } <ins> func TestTransformingInformerRace(t *testing.T) { // source simulates an apiserver object endpoint. source := fcache.NewFakeControllerSource() label := "to-be-transformed" makePod := func(name string) *v1.Pod { return &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:      name, Namespace: "namespace", Labels:    map[string]string{label: "true"}, }, Spec: v1.PodSpec{ Hostname: "hostname", }, } } badTransform := atomic.Bool{} podTransformer := func(obj interface{}) (interface{}, error) { pod, ok := obj.(*v1.Pod) if !ok { return nil, fmt.Errorf("unexpected object type: %T", obj) } if pod.ObjectMeta.Labels[label] != "true" { badTransform.Store(true) return nil, fmt.Errorf("object already transformed: %#v", obj) } pod.ObjectMeta.Labels[label] = "false" return pod, nil } numObjs := 5 for i := 0; i < numObjs; i++ { source.Add(makePod(fmt.Sprintf("pod-%d", i))) } type event struct{} events := make(chan event, numObjs) recordEvent := func(eventType watch.EventType, previous, current interface{}) { events <- event{} } checkEvents := func(count int) { for i := 0; i < count; i++ { <-events } } store, controller := NewTransformingInformer( source, &v1.Pod{}, 5*time.Millisecond, ResourceEventHandlerDetailedFuncs{ AddFunc:    func(obj interface{}, isInInitialList bool) { recordEvent(watch.Added, nil, obj) }, UpdateFunc: func(oldObj, newObj interface{}) { recordEvent(watch.Modified, oldObj, newObj) }, DeleteFunc: func(obj interface{}) { recordEvent(watch.Deleted, obj, nil) }, }, podTransformer, ) stopCh := make(chan struct{}) go controller.Run(stopCh) checkEvents(numObjs) // Periodically fetch objects to ensure no access races. wg := sync.WaitGroup{} errors := make(chan error, numObjs) for i := 0; i < numObjs; i++ { wg.Add(1) go func(index int) { defer wg.Done() key := fmt.Sprintf("namespace/pod-%d", index) for { select { case <-stopCh: return default: } obj, ok, err := store.GetByKey(key) if !ok || err != nil { errors <- fmt.Errorf("couldn't get the object for %v", key) return } pod := obj.(*v1.Pod) if pod.ObjectMeta.Labels[label] != "false" { errors <- fmt.Errorf("unexpected object: %#v", pod) return } } }(i) } // Let resyncs to happen for some time. time.Sleep(time.Second) close(stopCh) wg.Wait() close(errors) for err := range errors { t.Error(err) } if badTransform.Load() { t.Errorf("unexpected transformation happened") } } </ins> func TestDeletionHandlingObjectToName(t *testing.T) { cm := &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{


} // TransformFunc allows for transforming an object before it will be processed. <del> // TransformFunc (similarly to ResourceEventHandler functions) should be able // to correctly handle the tombstone of type cache.DeletedFinalStateUnknown. // // New in v1.27: In such cases, the contained object will already have gone // through the transform object separately (when it was added / updated prior // to the delete), so the TransformFunc can likely safely ignore such objects // (i.e., just return the input object). </del> // // The most common usage pattern is to clean-up some parts of the object to // reduce component memory usage if a given component doesn't care about them. // <del> // New in v1.27: unless the object is a DeletedFinalStateUnknown, TransformFunc // sees the object before any other actor, and it is now safe to mutate the // object in place instead of making a copy. </del> <ins> // New in v1.27: TransformFunc sees the object before any other actor, and it // is now safe to mutate the object in place instead of making a copy. // // It's recommended for the TransformFunc to be idempotent. // It MUST be idempotent if objects already present in the cache are passed to // the Replace() to avoid re-mutating them. Default informers do not pass // existing objects to Replace though. </ins> // // Note that TransformFunc is called while inserting objects into the // notification queue and is therefore extremely performance sensitive; please


// queueActionLocked appends to the delta list for the object. // Caller must lock first. func (f *DeltaFIFO) queueActionLocked(actionType DeltaType, obj interface{}) error { <ins> return f.queueActionInternalLocked(actionType, actionType, obj) } // queueActionInternalLocked appends to the delta list for the object. // The actionType is emitted and must honor emitDeltaTypeReplaced. // The internalActionType is only used within this function and must // ignore emitDeltaTypeReplaced. // Caller must lock first. func (f *DeltaFIFO) queueActionInternalLocked(actionType, internalActionType DeltaType, obj interface{}) error { </ins> id, err := f.KeyOf(obj) if err != nil { return KeyError{obj, err} } // Every object comes through this code path once, so this is a good <del> // place to call the transform func.  If obj is a // DeletedFinalStateUnknown tombstone, then the containted inner object // will already have gone through the transformer, but we document that // this can happen. In cases involving Replace(), such an object can // come through multiple times. </del> <ins> // place to call the transform func. // // If obj is a DeletedFinalStateUnknown tombstone or the action is a Sync, // then the object have already gone through the transformer. // // If the objects already present in the cache are passed to Replace(), // the transformer must be idempotent to avoid re-mutating them, // or coordinate with all readers from the cache to avoid data races. // Default informers do not pass existing objects to Replace. </ins> if f.transformer != nil { <del> var err error obj, err = f.transformer(obj) if err != nil { return err </del> <ins> _, isTombstone := obj.(DeletedFinalStateUnknown) if !isTombstone && internalActionType != Sync { var err error obj, err = f.transformer(obj) if err != nil { return err } </ins> } }


return KeyError{item, err} } keys.Insert(key) <del> if err := f.queueActionLocked(action, item); err != nil { </del> <ins> if err := f.queueActionInternalLocked(action, Replaced, item); err != nil { </ins> return fmt.Errorf("couldn't enqueue object: %v", err) } }


return fmt.Errorf("failed to try resolving symlinks in path %q: %v", path, err) } path = evaluated <del> f, err := os.Open(path) </del> <ins> f, err := openFileShareDelete(path) </ins> if err != nil { return fmt.Errorf("failed to open log file %q: %v", path, err) }


return err } if recreated { <del> newF, err := os.Open(path) </del> <ins> newF, err := openFileShareDelete(path) </ins> if err != nil { if os.IsNotExist(err) { continue


<ins> //go:build !windows // +build !windows /* Copyright 2024 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package logs import ( "os" ) func openFileShareDelete(path string) (*os.File, error) { // Noop. Only relevant for Windows. return os.Open(path) } </ins>


"io" "os" "path/filepath" <del> goruntime "runtime" </del> "testing" "time"


} func TestReadRotatedLog(t *testing.T) { <del> if goruntime.GOOS == "windows" { // TODO: remove skip once the failing test has been fixed. t.Skip("Skip failing test on Windows.") } </del> tmpDir := t.TempDir() file, err := os.CreateTemp(tmpDir, "logfile") if err != nil {


} } <ins> // Finished writing into the file, close it, so we can delete it later. err = file.Close() assert.NoErrorf(t, err, "could not close file.") </ins> time.Sleep(20 * time.Millisecond) // Make the function ReadLogs end. fakeRuntimeService.Lock()


<ins> //go:build windows // +build windows /* Copyright 2024 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package logs import ( "os" "syscall" ) // Based on Windows implementation of Windows' syscall.Open // https://cs.opensource.google/go/go/+/refs/tags/go1.22.2:src/syscall/syscall_windows.go;l=342 // In addition to syscall.Open, this function also adds the syscall.FILE_SHARE_DELETE flag to sharemode, // which will allow us to read from the file without blocking the file from being deleted or renamed. // This is essential for Log Rotation which is done by renaming the open file. Without this, the file rename would fail. func openFileShareDelete(path string) (*os.File, error) { pathp, err := syscall.UTF16PtrFromString(path) if err != nil { return nil, err } var access uint32 = syscall.GENERIC_READ var sharemode uint32 = syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE | syscall.FILE_SHARE_DELETE var createmode uint32 = syscall.OPEN_EXISTING var attrs uint32 = syscall.FILE_ATTRIBUTE_NORMAL handle, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, attrs, 0) if err != nil { return nil, err } return os.NewFile(uintptr(handle), path), nil } </ins>


o.Prefix = true return o }, <del> expectedOutSubstrings: []string{"[pod/test-sts-0/test-container] test log content for pod test-sts-0n[pod/test-sts-1/test-container] test log content for pod test-sts-1n"}, </del> <ins> expectedOutSubstrings: []string{ "[pod/test-sts-0/test-container] test log content for pod test-sts-0n", "[pod/test-sts-1/test-container] test log content for pod test-sts-1n", }, </ins> }, { name: "pod logs with prefix: init container",


// latency for a pod. InitialAttemptTimestamp *time.Time // UnschedulablePlugins records the plugin names that the Pod failed with Unschedulable or UnschedulableAndUnresolvable status. <del> // It's registered only when the Pod is rejected in PreFilter, Filter, Reserve, or Permit (WaitOnPermit). </del> <ins> // It's registered only when the Pod is rejected in PreFilter, Filter, Reserve, PreBind or Permit (WaitOnPermit). </ins> UnschedulablePlugins sets.Set[string] // PendingPlugins records the plugin names that the Pod failed with Pending status. PendingPlugins sets.Set[string]


// Run "prebind" plugins. if status := fwk.RunPreBindPlugins(ctx, state, assumedPod, scheduleResult.SuggestedHost); !status.IsSuccess() { <ins> if status.IsRejected() { fitErr := &framework.FitError{ NumAllNodes: 1, Pod:         assumedPodInfo.Pod, Diagnosis: framework.Diagnosis{ NodeToStatusMap:      framework.NodeToStatusMap{scheduleResult.SuggestedHost: status}, UnschedulablePlugins: sets.New(status.Plugin()), }, } return framework.NewStatus(status.Code()).WithError(fitErr) } </ins> return status }


ginkgo.It("should reject a pod with an AppArmor profile", func(ctx context.Context) { status := runAppArmorTest(ctx, f, false, v1.DeprecatedAppArmorBetaProfileRuntimeDefault) <del> expectSoftRejection(status) </del> <ins> expectRejection(status) </ins> }) }) }


return e2epod.NewPodClient(f).Create(ctx, pod) } <del> func expectSoftRejection(status v1.PodStatus) { </del> <ins> func expectRejection(status v1.PodStatus) { </ins> args := []interface{}{"PodStatus: %+v", status} <del> gomega.Expect(status.Phase).To(gomega.Equal(v1.PodPending), args...) </del> <ins> gomega.Expect(status.Phase).To(gomega.Equal(v1.PodFailed), args...) </ins> gomega.Expect(status.Reason).To(gomega.Equal("AppArmor"), args...) gomega.Expect(status.Message).To(gomega.ContainSubstring("AppArmor"), args...) <del> gomega.Expect(status.ContainerStatuses[0].State.Waiting.Reason).To(gomega.Equal("Blocked"), args...) </del> } func isAppArmorEnabled() bool {


// When specified, it should match one the container or initContainer // names in the pod template. // +optional <del> ContainerName *string `json:"containerName" protobuf:"bytes,1,opt,name=containerName"` </del> <ins> ContainerName *string `json:"containerName,omitempty" protobuf:"bytes,1,opt,name=containerName"` </ins> // Represents the relationship between the container exit code(s) and the // specified values. Containers completed with success (exit code 0) are


// Represents the requirement on the container exit codes. // +optional <del> OnExitCodes *PodFailurePolicyOnExitCodesRequirement `json:"onExitCodes" protobuf:"bytes,2,opt,name=onExitCodes"` </del> <ins> OnExitCodes *PodFailurePolicyOnExitCodesRequirement `json:"onExitCodes,omitempty" protobuf:"bytes,2,opt,name=onExitCodes"` </ins> // Represents the requirement on the pod conditions. The requirement is represented // as a list of pod condition patterns. The requirement is satisfied if at // least one pattern matches an actual pod condition. At most 20 elements are allowed. // +listType=atomic // +optional <del> OnPodConditions []PodFailurePolicyOnPodConditionsPattern `json:"onPodConditions" protobuf:"bytes,3,opt,name=onPodConditions"` </del> <ins> OnPodConditions []PodFailurePolicyOnPodConditionsPattern `json:"onPodConditions,omitempty" protobuf:"bytes,3,opt,name=onPodConditions"` </ins> } // PodFailurePolicy describes how failed pods influence the backoffLimit.


e2eoutput.TestContainerOutput(ctx, f, "check pod SecurityContext username", pod, 1, []string{"ContainerAdministrator"}) }) <del> ginkgo.It("should ignore SELinux Specific SecurityContext if set", func(ctx context.Context) { </del> <ins> ginkgo.It("should ignore Linux Specific SecurityContext if set", func(ctx context.Context) { </ins> ginkgo.By("Creating a pod with SELinux options") // It is sufficient to show that the pod comes up here. Since we're stripping the SELinux and other linux // security contexts in apiserver and not updating the pod object in the apiserver, we cannot validate the


f.Namespace.Name), "failed to wait for pod %s to be running", windowsPodWithSELinux.Name) }) <del> ginkgo.It("should ignore ProcMount Specific SecurityContext if set", func(ctx context.Context) { ginkgo.By("Creating a pod with ProcMount options") // It is sufficient to show that the pod comes up here. Since we're stripping the SELinux and other linux // security contexts in apiserver and not updating the pod object in the apiserver, we cannot validate the // pod object to not have those security contexts. However the pod coming to running state is a sufficient // enough condition for us to validate since prior to https://github.com/kubernetes/kubernetes/pull/93475 // the pod would have failed to come up. windowsPodWithSELinux := createTestPod(f, imageutils.GetE2EImage(imageutils.Agnhost), windowsOS) windowsPodWithSELinux.Spec.Containers[0].Args = []string{"test-webserver-with-selinux"} windowsPodWithSELinux.Spec.SecurityContext = &v1.PodSecurityContext{} pmt := v1.UnmaskedProcMount containerUserName := "ContainerAdministrator" windowsPodWithSELinux.Spec.Containers[0].SecurityContext = &v1.SecurityContext{ ProcMount:      &pmt, WindowsOptions: &v1.WindowsSecurityContextOptions{RunAsUserName: &containerUserName}} windowsPodWithSELinux.Spec.Tolerations = []v1.Toleration{{Key: "os", Value: "Windows"}} windowsPodWithSELinux, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, windowsPodWithSELinux, metav1.CreateOptions{}) framework.ExpectNoError(err) framework.Logf("Created pod %v", windowsPodWithSELinux) framework.ExpectNoError(e2epod.WaitForPodNameRunningInNamespace(ctx, f.ClientSet, windowsPodWithSELinux.Name, f.Namespace.Name), "failed to wait for pod %s to be running", windowsPodWithSELinux.Name) }) </del> ginkgo.It("should not be able to create pods with containers running as ContainerAdministrator when runAsNonRoot is true", func(ctx context.Context) { ginkgo.By("Creating a pod")


// TODO: document the feature (owning SIG, when to use this feature for a test) PodResources = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("PodResources")) <ins> // ResourceHealthStatus (SIG-node, resource health Status for device plugins and DRA <https://kep.k8s.io/4680>) ResourceHealthStatus = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("ResourceHealthStatus")) </ins> // TODO: document the feature (owning SIG, when to use this feature for a test) ProcMountType = framework.WithNodeFeature(framework.ValidNodeFeatures.Add("ProcMountType"))


v1 "k8s.io/api/core/v1" kubeletdevicepluginv1beta1 "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" <ins> "k8s.io/kubernetes/test/e2e/nodefeature" </ins> imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/uuid" <del> "k8s.io/kubernetes/pkg/features" </del> "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e_node/testdeviceplugin" ) <del> var _ = SIGDescribe("Device Plugin Failures Pod Status:", framework.WithFeatureGate(features.ResourceHealthStatus), func() { </del> <ins> var _ = SIGDescribe("Device Plugin Failures Pod Status", nodefeature.ResourceHealthStatus, func() { </ins> f := framework.NewDefaultFramework("device-plugin-failures") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged


"k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/images" "k8s.io/kubernetes/test/e2e/framework" <ins> e2enode "k8s.io/kubernetes/test/e2e/framework/node" </ins> e2epod "k8s.io/kubernetes/test/e2e/framework/pod" e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" "k8s.io/kubernetes/test/e2e/nodefeature"


e2eskipper.SkipUnlessFeatureGateEnabled(features.ImageVolume) }) <del> createPod := func(ctx context.Context, volumes []v1.Volume, volumeMounts []v1.VolumeMount) { var selinuxOptions *v1.SELinuxOptions if selinux.GetEnabled() { selinuxOptions = &v1.SELinuxOptions{ User:  defaultSELinuxUser, Role:  defaultSELinuxRole, Type:  defaultSELinuxType, Level: defaultSELinuxLevel, } ginkgo.By(fmt.Sprintf("Using SELinux on pod: %v", selinuxOptions)) } </del> <ins> createPod := func(ctx context.Context, podName, nodeName string, volumes []v1.Volume, volumeMounts []v1.VolumeMount, selinuxOptions *v1.SELinuxOptions) { </ins> pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:      podName, Namespace: f.Namespace.Name, }, Spec: v1.PodSpec{ <ins> NodeName:      nodeName, </ins> RestartPolicy: v1.RestartPolicyAlways, SecurityContext: &v1.PodSecurityContext{ SELinuxOptions: selinuxOptions,


}, } <del> ginkgo.By(fmt.Sprintf("Creating a pod (%v/%v)", f.Namespace.Name, podName)) </del> <ins> ginkgo.By(fmt.Sprintf("Creating a pod (%s/%s)", f.Namespace.Name, podName)) </ins> e2epod.NewPodClient(f).Create(ctx, pod) } f.It("should succeed with pod and pull policy of Always", func(ctx context.Context) { <ins> var selinuxOptions *v1.SELinuxOptions if selinux.GetEnabled() { selinuxOptions = &v1.SELinuxOptions{ User:  defaultSELinuxUser, Role:  defaultSELinuxRole, Type:  defaultSELinuxType, Level: defaultSELinuxLevel, } ginkgo.By(fmt.Sprintf("Using SELinux on pod: %v", selinuxOptions)) } </ins> createPod(ctx, <ins> podName, "", </ins> []v1.Volume{{Name: volumeName, VolumeSource: v1.VolumeSource{Image: &v1.ImageVolumeSource{Reference: validImageRef, PullPolicy: v1.PullAlways}}}}, []v1.VolumeMount{{Name: volumeName, MountPath: volumePathPrefix}}, <ins> selinuxOptions, </ins> ) <del> ginkgo.By(fmt.Sprintf("Waiting for the pod (%v/%v) to be running", f.Namespace.Name, podName)) </del> <ins> ginkgo.By(fmt.Sprintf("Waiting for the pod (%s/%s) to be running", f.Namespace.Name, podName)) </ins> err := e2epod.WaitForPodNameRunningInNamespace(ctx, f.ClientSet, podName, f.Namespace.Name) <del> framework.ExpectNoError(err, "Failed to await for the pod to be running: (%v/%v)", f.Namespace.Name, podName) </del> <ins> framework.ExpectNoError(err, "Failed to await for the pod to be running: (%s/%s)", f.Namespace.Name, podName) </ins> ginkgo.By(fmt.Sprintf("Verifying the volume mount contents for path: %s", volumePathPrefix))


}) f.It("should succeed with pod and multiple volumes", func(ctx context.Context) { <ins> var selinuxOptions *v1.SELinuxOptions if selinux.GetEnabled() { selinuxOptions = &v1.SELinuxOptions{ User:  defaultSELinuxUser, Role:  defaultSELinuxRole, Type:  defaultSELinuxType, Level: defaultSELinuxLevel, } ginkgo.By(fmt.Sprintf("Using SELinux on pod: %v", selinuxOptions)) } </ins> createPod(ctx, <ins> podName, "", </ins> []v1.Volume{ {Name: volumeName + "-0", VolumeSource: v1.VolumeSource{Image: &v1.ImageVolumeSource{Reference: validImageRef}}}, {Name: volumeName + "-1", VolumeSource: v1.VolumeSource{Image: &v1.ImageVolumeSource{Reference: validImageRef}}},


{Name: volumeName + "-0", MountPath: volumePathPrefix + "-0"}, {Name: volumeName + "-1", MountPath: volumePathPrefix + "-1"}, }, <ins> selinuxOptions, </ins> ) <del> ginkgo.By(fmt.Sprintf("Waiting for the pod (%v/%v) to be running", f.Namespace.Name, podName)) </del> <ins> ginkgo.By(fmt.Sprintf("Waiting for the pod (%s/%s) to be running", f.Namespace.Name, podName)) </ins> err := e2epod.WaitForPodNameRunningInNamespace(ctx, f.ClientSet, podName, f.Namespace.Name) <del> framework.ExpectNoError(err, "Failed to await for the pod to be running: (%v/%v)", f.Namespace.Name, podName) </del> <ins> framework.ExpectNoError(err, "Failed to await for the pod to be running: (%s/%s)", f.Namespace.Name, podName) </ins> for i := range 2 { volumePath := fmt.Sprintf("%s-%d", volumePathPrefix, i)


}) f.It("should fail if image volume is not existing", func(ctx context.Context) { <ins> var selinuxOptions *v1.SELinuxOptions if selinux.GetEnabled() { selinuxOptions = &v1.SELinuxOptions{ User:  defaultSELinuxUser, Role:  defaultSELinuxRole, Type:  defaultSELinuxType, Level: defaultSELinuxLevel, } ginkgo.By(fmt.Sprintf("Using SELinux on pod: %v", selinuxOptions)) } </ins> createPod(ctx, <ins> podName, "", </ins> []v1.Volume{{Name: volumeName, VolumeSource: v1.VolumeSource{Image: &v1.ImageVolumeSource{Reference: invalidImageRef}}}}, []v1.VolumeMount{{Name: volumeName, MountPath: volumePathPrefix}}, <ins> selinuxOptions, </ins> ) <del> ginkgo.By(fmt.Sprintf("Waiting for the pod (%v/%v) to fail", f.Namespace.Name, podName)) </del> <ins> ginkgo.By(fmt.Sprintf("Waiting for the pod (%s/%s) to fail", f.Namespace.Name, podName)) </ins> err := e2epod.WaitForPodContainerToFail(ctx, f.ClientSet, f.Namespace.Name, podName, 0, images.ErrImagePullBackOff.Error(), time.Minute) <del> framework.ExpectNoError(err, "Failed to await for the pod to be running: (%v/%v)", f.Namespace.Name, podName) </del> <ins> framework.ExpectNoError(err, "Failed to await for the pod to be running: (%s/%s)", f.Namespace.Name, podName) </ins> }) f.It("should succeed if image volume is not existing but unused", func(ctx context.Context) { <ins> var selinuxOptions *v1.SELinuxOptions if selinux.GetEnabled() { selinuxOptions = &v1.SELinuxOptions{ User:  defaultSELinuxUser, Role:  defaultSELinuxRole, Type:  defaultSELinuxType, Level: defaultSELinuxLevel, } ginkgo.By(fmt.Sprintf("Using SELinux on pod: %v", selinuxOptions)) } </ins> createPod(ctx, <ins> podName, "", </ins> []v1.Volume{{Name: volumeName, VolumeSource: v1.VolumeSource{Image: &v1.ImageVolumeSource{Reference: invalidImageRef}}}}, nil, <ins> selinuxOptions, </ins> ) <del> ginkgo.By(fmt.Sprintf("Waiting for the pod (%v/%v) to be running", f.Namespace.Name, podName)) </del> <ins> ginkgo.By(fmt.Sprintf("Waiting for the pod (%s/%s) to be running", f.Namespace.Name, podName)) </ins> err := e2epod.WaitForPodNameRunningInNamespace(ctx, f.ClientSet, podName, f.Namespace.Name) <del> framework.ExpectNoError(err, "Failed to await for the pod to be running: (%v/%v)", f.Namespace.Name, podName) </del> <ins> framework.ExpectNoError(err, "Failed to await for the pod to be running: (%s/%s)", f.Namespace.Name, podName) </ins> ginkgo.By(fmt.Sprintf("Verifying the volume mount is not used for path: %s", volumePathPrefix)) output := e2epod.ExecCommandInContainer(f, podName, containerName, "/bin/ls", filepath.Dir(volumePathPrefix)) gomega.Expect(output).NotTo(gomega.ContainSubstring(strings.TrimPrefix(volumePathPrefix, "/"))) }) <ins> f.It("should succeed with multiple pods and same image on the same node", func(ctx context.Context) { node, err := e2enode.GetRandomReadySchedulableNode(ctx, f.ClientSet) framework.ExpectNoError(err, "Failed to get a ready schedulable node") baseName := "test-pod" anotherSELinuxLevel := "s0:c100,c200" for i := range 2 { podName := fmt.Sprintf("%s-%d", baseName, i) var selinuxOptions *v1.SELinuxOptions if selinux.GetEnabled() { if i == 0 { selinuxOptions = &v1.SELinuxOptions{ User:  defaultSELinuxUser, Role:  defaultSELinuxRole, Type:  defaultSELinuxType, Level: defaultSELinuxLevel, } } else { selinuxOptions = &v1.SELinuxOptions{ User:  defaultSELinuxUser, Role:  defaultSELinuxRole, Type:  defaultSELinuxType, Level: anotherSELinuxLevel, } } ginkgo.By(fmt.Sprintf("Using SELinux on pod %q: %v", podName, selinuxOptions)) } createPod(ctx, podName, node.Name, []v1.Volume{{Name: volumeName, VolumeSource: v1.VolumeSource{Image: &v1.ImageVolumeSource{Reference: validImageRef}}}}, []v1.VolumeMount{{Name: volumeName, MountPath: volumePathPrefix}}, selinuxOptions, ) ginkgo.By(fmt.Sprintf("Waiting for the pod (%s/%s) to be running", f.Namespace.Name, podName)) err := e2epod.WaitForPodNameRunningInNamespace(ctx, f.ClientSet, podName, f.Namespace.Name) framework.ExpectNoError(err, "Failed to await for the pod to be running: (%s/%s)", f.Namespace.Name, podName) ginkgo.By(fmt.Sprintf("Verifying the volume mount contents for path: %s", volumePathPrefix)) firstFileContents := e2epod.ExecCommandInContainer(f, podName, containerName, "/bin/cat", filepath.Join(volumePathPrefix, "dir", "file")) gomega.Expect(firstFileContents).To(gomega.Equal("1")) secondFileContents := e2epod.ExecCommandInContainer(f, podName, containerName, "/bin/cat", filepath.Join(volumePathPrefix, "file")) gomega.Expect(secondFileContents).To(gomega.Equal("2")) } podName := baseName + "-0" ginkgo.By(fmt.Sprintf("Rechecking the pod (%s/%s) after another pod is running as expected", f.Namespace.Name, podName)) err = e2epod.WaitForPodNameRunningInNamespace(ctx, f.ClientSet, podName, f.Namespace.Name) framework.ExpectNoError(err, "Failed to await for the pod to be running: (%s/%s)", f.Namespace.Name, podName) ginkgo.By(fmt.Sprintf("Verifying the volume mount contents for path: %s", volumePathPrefix)) firstFileContents := e2epod.ExecCommandInContainer(f, podName, containerName, "/bin/cat", filepath.Join(volumePathPrefix, "dir", "file")) gomega.Expect(firstFileContents).To(gomega.Equal("1")) secondFileContents := e2epod.ExecCommandInContainer(f, podName, containerName, "/bin/cat", filepath.Join(volumePathPrefix, "file")) gomega.Expect(secondFileContents).To(gomega.Equal("2")) }) </ins> })