testCannotConnect(f, f.Namespace, "client-b", service, 81) }) }) <ins> It("should allow egress access on one named port [Feature:NetworkPolicy]", func() { clientPodName := "client-a" protocolUDP := v1.ProtocolUDP policy := &networkingv1.NetworkPolicy{ ObjectMeta: metav1.ObjectMeta{ Name: "allow-client-a-via-named-port-egress-rule", }, Spec: networkingv1.NetworkPolicySpec{ // Apply this policy to client-a PodSelector: metav1.LabelSelector{ MatchLabels: map[string]string{ "pod-name": clientPodName, }, }, // Allow traffic to only one named port: "serve-80". Egress: []networkingv1.NetworkPolicyEgressRule{{ Ports: []networkingv1.NetworkPolicyPort{ { Port: &intstr.IntOrString{Type: intstr.String, StrVal: "serve-80"}, }, // Allow DNS look-ups { Protocol: &protocolUDP, Port:     &intstr.IntOrString{Type: intstr.Int, IntVal: 53}, }, }, }}, }, } policy, err := f.ClientSet.NetworkingV1().NetworkPolicies(f.Namespace.Name).Create(policy) Expect(err).NotTo(HaveOccurred()) defer cleanupNetworkPolicy(f, policy) By("Creating client-a which should be able to contact the server.", func() { testCanConnect(f, f.Namespace, clientPodName, service, 80) }) By("Creating client-a which should not be able to contact the server on port 81.", func() { testCannotConnect(f, f.Namespace, clientPodName, service, 81) }) }) </ins> }) })


// hostname or not, we only check for the suffix match. if strings.HasSuffix(image, tag) || strings.HasSuffix(tag, image) { return true <ins> } else { // TODO: We need to remove this hack when project atomic based // docker distro(s) like centos/fedora/rhel image fix problems on // their end. // Say the tag is "docker.io/busybox:latest" // and the image is "docker.io/library/busybox:latest" t, err := dockerref.ParseNormalizedNamed(tag) if err != nil { continue } // the parsed/normalized tag will look like // reference.taggedReference { // 	 namedRepository: reference.repository { // 	   domain: "docker.io", // 	   path: "library/busybox" //	}, // 	tag: "latest" // } // If it does not have tags then we bail out t2, ok := t.(dockerref.Tagged) if !ok { continue } // normalized tag would look like "docker.io/library/busybox:latest" // note the library get added in the string normalizedTag := t2.String() if normalizedTag == "" { continue } if strings.HasSuffix(image, normalizedTag) || strings.HasSuffix(normalizedTag, image) { return true } </ins> } } }


Output: true, }, { <ins> Inspected: dockertypes.ImageInspect{ ID:       "sha256:9bbdf247c91345f0789c10f50a57e36a667af1189687ad1de88a6243d05a2227", RepoTags: []string{"docker.io/busybox:latest"}, }, Image:  "docker.io/library/busybox:latest", Output: true, }, { </ins> // RepoDigest match is is required Inspected: dockertypes.ImageInspect{ ID:          "",


} updated := accessor.GetInitializers() <ins> // controllers deployed with an empty initializers.pending have their initializers set to nil // but should be able to update without changing their manifest if updated != nil && len(updated.Pending) == 0 && updated.Result == nil { accessor.SetInitializers(nil) updated = nil } </ins> existingAccessor, err := meta.Accessor(a.GetOldObject()) if err != nil { // if the old object does not have an accessor, but the new one does, error out


glog.V(5).Infof("Modifying uninitialized resource %s", a.GetResource()) <del> if updated != nil && len(updated.Pending) == 0 && updated.Result == nil { accessor.SetInitializers(nil) } </del> // because we are called before validation, we need to ensure the update transition is valid. if errs := validation.ValidateInitializersUpdate(updated, existing, initializerFieldPath); len(errs) > 0 { return errors.NewInvalid(a.GetKind().GroupKind(), a.GetName(), errs)


newInitializers: &metav1.Initializers{Pending: []metav1.Initializer{{Name: "init.k8s.io"}}}, err:             "field is immutable once initialization has completed", }, <ins> { name:            "empty initializer list is treated as nil initializer", oldInitializers: nil, newInitializers: &metav1.Initializers{}, verifyUpdatedObj: func(obj runtime.Object) (bool, string) { accessor, err := meta.Accessor(obj) if err != nil { return false, "cannot get accessor" } if accessor.GetInitializers() != nil { return false, "expect nil initializers" } return true, "" }, err: "", }, </ins> } plugin := initializer{


"flag" "fmt" "net" <ins> "sort" </ins> "strings" "github.com/golang/glog"


// String is the method to format the flag's value, part of the flag.Value interface. func (c *cidrs) String() string { <del> return strings.Join(c.ipn.StringSlice(), ",") </del> <ins> s := c.ipn.StringSlice() sort.Strings(s) return strings.Join(s, ",") </ins> } // Set supports a value of CSV or the flag repeated multiple times


for format := range jsonFormats { formats = append(formats, format) } <ins> sort.Strings(formats) </ins> return formats }


printFlags := JSONPathPrintFlags{ TemplateArgument: templateArg, } <ins> if !sort.StringsAreSorted(printFlags.AllowedFormats()) { t.Fatalf("allowed formats are not sorted") } </ins> p, err := printFlags.ToPrinter(tc.outputFormat) if tc.expectNoMatch {


TemplateArgument: &tc.templateArg, AllowMissingKeys: tc.allowMissingKeys, } <ins> if !sort.StringsAreSorted(printFlags.AllowedFormats()) { t.Fatalf("allowed formats are not sorted") } </ins> outputFormat := "jsonpath" p, err := printFlags.ToPrinter(outputFormat)


import ( "fmt" "io/ioutil" <ins> "sort" </ins> "strings" "github.com/spf13/cobra"


for format := range templateFormats { formats = append(formats, format) } <ins> sort.Strings(formats) </ins> return formats }


"fmt" "io/ioutil" "os" <ins> "sort" </ins> "strings" "testing"


printFlags := GoTemplatePrintFlags{ TemplateArgument: templateArg, } <ins> if !sort.StringsAreSorted(printFlags.AllowedFormats()) { t.Fatalf("allowed formats are not sorted") } </ins> p, err := printFlags.ToPrinter(tc.outputFormat) if tc.expectNoMatch {


TemplateArgument: &tc.templateArg, AllowMissingKeys: tc.allowMissingKeys, } <ins> if !sort.StringsAreSorted(printFlags.AllowedFormats()) { t.Fatalf("allowed formats are not sorted") } </ins> outputFormat := "template" p, err := printFlags.ToPrinter(outputFormat)


ETCD_QUORUM_READ: $(yaml-quote ${ETCD_QUORUM_READ}) EOF fi <ins> if [ -n "${CLUSTER_SIGNING_DURATION:-}" ]; then cat >>$file <<EOF CLUSTER_SIGNING_DURATION: $(yaml-quote ${CLUSTER_SIGNING_DURATION}) EOF fi </ins> else # Node-only env vars.


# Optional: [Experiment Only] Run kube-proxy as a DaemonSet if set to true, run as static pods otherwise. KUBE_PROXY_DAEMONSET="${KUBE_PROXY_DAEMONSET:-false}" # true, false <ins> # Optional: duration of cluster signed certificates. CLUSTER_SIGNING_DURATION="${CLUSTER_SIGNING_DURATION:-}" </ins> # Optional: enable pod priority ENABLE_POD_PRIORITY="${ENABLE_POD_PRIORITY:-}" if [[ "${ENABLE_POD_PRIORITY}" == "true" ]]; then


if [[ -n "${VOLUME_PLUGIN_DIR:-}" ]]; then params+=" --flex-volume-plugin-dir=${VOLUME_PLUGIN_DIR}" fi <ins> if [[ -n "${CLUSTER_SIGNING_DURATION:-}" ]]; then params+=" --experimental-cluster-signing-duration=$CLUSTER_SIGNING_DURATION" fi </ins> local -r kube_rc_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-controller-manager.docker_tag) local container_env="" if [[ -n "${ENABLE_CACHE_MUTATION_DETECTOR:-}" ]]; then


if [[ -z "${KUBE_MASTER_IP-}" ]]; then local master_address_name="${MASTER_NAME}-ip" echo "Looking for address '${master_address_name}'" >&2 <del> KUBE_MASTER_IP=$(gcloud compute addresses describe "${master_address_name}"  --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') fi if [[ -z "${KUBE_MASTER_IP-}" ]]; then echo "Could not detect Kubernetes master node.  Make sure you've launched a cluster with 'kube-up.sh'" >&2 exit 1 </del> <ins> if ! KUBE_MASTER_IP=$(gcloud compute addresses describe "${master_address_name}"  --project "${PROJECT}" --region "${REGION}" -q --format='value(address)') ||  [[ -z "${KUBE_MASTER_IP-}" ]]; then echo "Could not detect Kubernetes master node.  Make sure you've launched a cluster with 'kube-up.sh'" >&2 exit 1 fi </ins> fi echo "Using master: $KUBE_MASTER (external IP: $KUBE_MASTER_IP)" >&2 }


} // update our local observations based on the amount reported to have been reclaimed. // note: this is optimistic, other things could have been still consuming the pressured resource in the interim. <del> signal := resourceToSignal[resourceToReclaim] value, ok := observations[signal] if !ok { glog.Errorf("eviction manager: unable to find value associated with signal %v", signal) continue </del> <ins> for _, signal := range resourceClaimToSignal[resourceToReclaim] { value, ok := observations[signal] if !ok { glog.Errorf("eviction manager: unable to find value associated with signal %v", signal) continue } value.available.Add(*reclaimed) </ins> } <del> value.available.Add(*reclaimed) </del> // evaluate all current thresholds to see if with adjusted observations, we think we have met min reclaim goals if len(thresholdsMet(m.thresholdsMet, observations, true)) == 0 { return true


} } } <ins> // TestAllocatableNodeFsPressure func TestAllocatableNodeFsPressure(t *testing.T) { podMaker := makePodWithDiskStats summaryStatsMaker := makeDiskStats podsToMake := []podToMake{ {name: "guaranteed-low", requests: newEphemeralStorageResourceList("200Mi", "100m", "1Gi"), limits: newEphemeralStorageResourceList("200Mi", "100m", "1Gi"), rootFsUsed: "200Mi"}, {name: "guaranteed-high", requests: newEphemeralStorageResourceList("800Mi", "100m", "1Gi"), limits: newEphemeralStorageResourceList("800Mi", "100m", "1Gi"), rootFsUsed: "800Mi"}, {name: "burstable-low", requests: newEphemeralStorageResourceList("300Mi", "100m", "100Mi"), limits: newEphemeralStorageResourceList("300Mi", "200m", "1Gi"), logsFsUsed: "300Mi"}, {name: "burstable-high", requests: newEphemeralStorageResourceList("800Mi", "100m", "100Mi"), limits: newEphemeralStorageResourceList("800Mi", "200m", "1Gi"), rootFsUsed: "800Mi"}, {name: "best-effort-low", requests: newEphemeralStorageResourceList("300Mi", "", ""), limits: newEphemeralStorageResourceList("300Mi", "", ""), logsFsUsed: "300Mi"}, {name: "best-effort-high", requests: newEphemeralStorageResourceList("800Mi", "", ""), limits: newEphemeralStorageResourceList("800Mi", "", ""), rootFsUsed: "800Mi"}, } pods := []*v1.Pod{} podStats := map[*v1.Pod]statsapi.PodStats{} for _, podToMake := range podsToMake { pod, podStat := podMaker(podToMake.name, podToMake.requests, podToMake.limits, podToMake.rootFsUsed, podToMake.logsFsUsed, podToMake.perLocalVolumeUsed) pods = append(pods, pod) podStats[pod] = podStat } podToEvict := pods[5] activePodsFunc := func() []*v1.Pod { return pods } fakeClock := clock.NewFakeClock(time.Now()) podKiller := &mockPodKiller{} diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: false} capacityProvider := newMockCapacityProvider(newEphemeralStorageResourceList("6Gi", "1000m", "10Gi"), newEphemeralStorageResourceList("1Gi", "1000m", "10Gi")) diskGC := &mockDiskGC{imageBytesFreed: int64(0), err: nil} nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""} config := Config{ MaxPodGracePeriodSeconds: 5, PressureTransitionPeriod: time.Minute * 5, Thresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalAllocatableNodeFsAvailable, Operator: evictionapi.OpLessThan, Value: evictionapi.ThresholdValue{ Quantity: quantityMustParse("1Ki"), }, }, }, } summaryProvider := &fakeSummaryProvider{result: summaryStatsMaker("3Gi", "6Gi", podStats)} manager := &managerImpl{ clock:           fakeClock, killPodFunc:     podKiller.killPodNow, imageGC:         diskGC, containerGC:     diskGC, config:          config, recorder:        &record.FakeRecorder{}, summaryProvider: summaryProvider, nodeRef:         nodeRef, nodeConditionsLastObservedAt: nodeConditionsObservedAt{}, thresholdsFirstObservedAt:    thresholdsObservedAt{}, } // create a best effort pod to test admission bestEffortPodToAdmit, _ := podMaker("best-admit", newEphemeralStorageResourceList("", "", ""), newEphemeralStorageResourceList("", "", ""), "0Gi", "", "") burstablePodToAdmit, _ := podMaker("burst-admit", newEphemeralStorageResourceList("1Gi", "", ""), newEphemeralStorageResourceList("1Gi", "", ""), "1Gi", "", "") // synchronize manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider) // we should not have disk pressure if manager.IsUnderDiskPressure() { t.Fatalf("Manager should not report disk pressure") } // try to admit our pods (they should succeed) expected := []bool{true, true} for i, pod := range []*v1.Pod{bestEffortPodToAdmit, burstablePodToAdmit} { if result := manager.Admit(&lifecycle.PodAdmitAttributes{Pod: pod}); expected[i] != result.Admit { t.Fatalf("Admit pod: %v, expected: %v, actual: %v", pod, expected[i], result.Admit) } } // induce disk pressure! fakeClock.Step(1 * time.Minute) pod, podStat := podMaker("guaranteed-high-2", newEphemeralStorageResourceList("2000Mi", "100m", "1Gi"), newEphemeralStorageResourceList("2000Mi", "100m", "1Gi"), "2000Mi", "", "") podStats[pod] = podStat pods = append(pods, pod) summaryProvider.result = summaryStatsMaker("6Gi", "6Gi", podStats) manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider) // we should have disk pressure if !manager.IsUnderDiskPressure() { t.Fatalf("Manager should report disk pressure") } // check the right pod was killed if podKiller.pod != podToEvict { t.Fatalf("Manager chose to kill pod: %v, but should have chosen %v", podKiller.pod, podToEvict.Name) } observedGracePeriod := *podKiller.gracePeriodOverride if observedGracePeriod != int64(0) { t.Fatalf("Manager chose to kill pod with incorrect grace period.  Expected: %d, actual: %d", 0, observedGracePeriod) } // reset state podKiller.pod = nil podKiller.gracePeriodOverride = nil // try to admit our pod (should fail) expected = []bool{false, false} for i, pod := range []*v1.Pod{bestEffortPodToAdmit, burstablePodToAdmit} { if result := manager.Admit(&lifecycle.PodAdmitAttributes{Pod: pod}); expected[i] != result.Admit { t.Fatalf("Admit pod: %v, expected: %v, actual: %v", pod, expected[i], result.Admit) } } // reduce disk pressure fakeClock.Step(1 * time.Minute) pods[5] = pods[len(pods)-1] pods = pods[:len(pods)-1] // we should have disk pressure (because transition period not yet met) if !manager.IsUnderDiskPressure() { t.Fatalf("Manager should report disk pressure") } // try to admit our pod (should fail) expected = []bool{false, false} for i, pod := range []*v1.Pod{bestEffortPodToAdmit, burstablePodToAdmit} { if result := manager.Admit(&lifecycle.PodAdmitAttributes{Pod: pod}); expected[i] != result.Admit { t.Fatalf("Admit pod: %v, expected: %v, actual: %v", pod, expected[i], result.Admit) } } // move the clock past transition period to ensure that we stop reporting pressure fakeClock.Step(5 * time.Minute) manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider) // we should not have disk pressure (because transition period met) if manager.IsUnderDiskPressure() { t.Fatalf("Manager should not report disk pressure") } // no pod should have been killed if podKiller.pod != nil { t.Fatalf("Manager chose to kill pod: %v when no pod should have been killed", podKiller.pod.Name) } // all pods should admit now expected = []bool{true, true} for i, pod := range []*v1.Pod{bestEffortPodToAdmit, burstablePodToAdmit} { if result := manager.Admit(&lifecycle.PodAdmitAttributes{Pod: pod}); expected[i] != result.Admit { t.Fatalf("Admit pod: %v, expected: %v, actual: %v", pod, expected[i], result.Admit) } } } func TestNodeReclaimForAllocatableFuncs(t *testing.T) { podMaker := makePodWithDiskStats summaryStatsMaker := makeDiskStats podsToMake := []podToMake{ {name: "guaranteed-low", requests: newEphemeralStorageResourceList("200Mi", "100m", "1Gi"), limits: newEphemeralStorageResourceList("200Mi", "100m", "1Gi"), rootFsUsed: "200Mi"}, {name: "guaranteed-high", requests: newEphemeralStorageResourceList("800Mi", "100m", "1Gi"), limits: newEphemeralStorageResourceList("800Mi", "100m", "1Gi"), rootFsUsed: "800Mi"}, {name: "burstable-low", requests: newEphemeralStorageResourceList("300Mi", "100m", "100Mi"), limits: newEphemeralStorageResourceList("300Mi", "200m", "1Gi"), logsFsUsed: "300Mi"}, {name: "burstable-high", requests: newEphemeralStorageResourceList("800Mi", "100m", "100Mi"), limits: newEphemeralStorageResourceList("800Mi", "200m", "1Gi"), rootFsUsed: "800Mi"}, {name: "best-effort-low", requests: newEphemeralStorageResourceList("300Mi", "", ""), limits: newEphemeralStorageResourceList("300Mi", "", ""), logsFsUsed: "300Mi"}, {name: "best-effort-high", requests: newEphemeralStorageResourceList("800Mi", "", ""), limits: newEphemeralStorageResourceList("800Mi", "", ""), rootFsUsed: "800Mi"}, } pods := []*v1.Pod{} podStats := map[*v1.Pod]statsapi.PodStats{} for _, podToMake := range podsToMake { pod, podStat := podMaker(podToMake.name, podToMake.requests, podToMake.limits, podToMake.rootFsUsed, podToMake.logsFsUsed, podToMake.perLocalVolumeUsed) pods = append(pods, pod) podStats[pod] = podStat } podToEvict := pods[5] activePodsFunc := func() []*v1.Pod { return pods } fakeClock := clock.NewFakeClock(time.Now()) podKiller := &mockPodKiller{} diskInfoProvider := &mockDiskInfoProvider{dedicatedImageFs: false} capacityProvider := newMockCapacityProvider(newEphemeralStorageResourceList("6Gi", "1000m", "10Gi"), newEphemeralStorageResourceList("1Gi", "1000m", "10Gi")) imageGcFree := resource.MustParse("800Mi") diskGC := &mockDiskGC{imageBytesFreed: imageGcFree.Value(), err: nil} nodeRef := &v1.ObjectReference{Kind: "Node", Name: "test", UID: types.UID("test"), Namespace: ""} config := Config{ MaxPodGracePeriodSeconds: 5, PressureTransitionPeriod: time.Minute * 5, Thresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalAllocatableNodeFsAvailable, Operator: evictionapi.OpLessThan, Value: evictionapi.ThresholdValue{ Quantity: quantityMustParse("10Mi"), }, }, }, } summaryProvider := &fakeSummaryProvider{result: summaryStatsMaker("6Gi", "6Gi", podStats)} manager := &managerImpl{ clock:           fakeClock, killPodFunc:     podKiller.killPodNow, imageGC:         diskGC, containerGC:     diskGC, config:          config, recorder:        &record.FakeRecorder{}, summaryProvider: summaryProvider, nodeRef:         nodeRef, nodeConditionsLastObservedAt: nodeConditionsObservedAt{}, thresholdsFirstObservedAt:    thresholdsObservedAt{}, } // synchronize manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider) // we should not have disk pressure if manager.IsUnderDiskPressure() { t.Errorf("Manager should not report disk pressure") } // induce hard threshold fakeClock.Step(1 * time.Minute) pod, podStat := podMaker("guaranteed-high-2", newEphemeralStorageResourceList("2000Mi", "100m", "1Gi"), newEphemeralStorageResourceList("2000Mi", "100m", "1Gi"), "2000Mi", "", "") podStats[pod] = podStat pods = append(pods, pod) summaryProvider.result = summaryStatsMaker("6Gi", "6Gi", podStats) manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider) // we should have disk pressure if !manager.IsUnderDiskPressure() { t.Fatalf("Manager should report disk pressure since soft threshold was met") } // verify image gc was invoked if !diskGC.imageGCInvoked || !diskGC.containerGCInvoked { t.Fatalf("Manager should have invoked image gc") } // verify no pod was killed because image gc was sufficient if podKiller.pod == nil { t.Fatalf("Manager should have killed a pod, but not killed") } // check the right pod was killed if podKiller.pod != podToEvict { t.Fatalf("Manager chose to kill pod: %v, but should have chosen %v", podKiller.pod, podToEvict.Name) } observedGracePeriod := *podKiller.gracePeriodOverride if observedGracePeriod != int64(0) { t.Fatalf("Manager chose to kill pod with incorrect grace period.  Expected: %d, actual: %d", 0, observedGracePeriod) } // reset state diskGC.imageGCInvoked = false diskGC.containerGCInvoked = false podKiller.pod = nil podKiller.gracePeriodOverride = nil // reduce disk pressure fakeClock.Step(1 * time.Minute) pods[5] = pods[len(pods)-1] pods = pods[:len(pods)-1] // we should have disk pressure (because transition period not yet met) if !manager.IsUnderDiskPressure() { t.Fatalf("Manager should report disk pressure") } // move the clock past transition period to ensure that we stop reporting pressure fakeClock.Step(5 * time.Minute) manager.synchronize(diskInfoProvider, activePodsFunc, capacityProvider) // we should not have disk pressure (because transition period met) if manager.IsUnderDiskPressure() { t.Fatalf("Manager should not report disk pressure") } // no pod should have been killed if podKiller.pod != nil { t.Fatalf("Manager chose to kill pod: %v when no pod should have been killed", podKiller.pod.Name) } // no image gc should have occurred if diskGC.imageGCInvoked || diskGC.containerGCInvoked { t.Errorf("Manager chose to perform image gc when it was not neeed") } // no pod should have been killed if podKiller.pod != nil { t.Errorf("Manager chose to kill pod: %v when no pod should have been killed", podKiller.pod.Name) } } </ins>


signalToNodeCondition map[evictionapi.Signal]v1.NodeConditionType // signalToResource maps a Signal to its associated Resource. signalToResource map[evictionapi.Signal]v1.ResourceName <del> // resourceToSignal maps a Resource to its associated Signal resourceToSignal map[v1.ResourceName]evictionapi.Signal </del> <ins> // resourceClaimToSignal maps a Resource that can be reclaimed to its associated Signal resourceClaimToSignal map[v1.ResourceName][]evictionapi.Signal </ins> ) func init() {


signalToResource[evictionapi.SignalNodeFsAvailable] = resourceNodeFs signalToResource[evictionapi.SignalNodeFsInodesFree] = resourceNodeFsInodes <del> resourceToSignal = map[v1.ResourceName]evictionapi.Signal{} for key, value := range signalToResource { resourceToSignal[value] = key } // Hard-code here to make sure resourceNodeFs maps to evictionapi.SignalNodeFsAvailable // (TODO) resourceToSignal is a map from resource name to a list of signals resourceToSignal[resourceNodeFs] = evictionapi.SignalNodeFsAvailable </del> <ins> // maps resource to signals (the following resource could be reclaimed) resourceClaimToSignal = map[v1.ResourceName][]evictionapi.Signal{} resourceClaimToSignal[resourceNodeFs] = []evictionapi.Signal{evictionapi.SignalNodeFsAvailable} resourceClaimToSignal[resourceImageFs] = []evictionapi.Signal{evictionapi.SignalImageFsAvailable} resourceClaimToSignal[resourceNodeFsInodes] = []evictionapi.Signal{evictionapi.SignalNodeFsInodesFree} resourceClaimToSignal[resourceImageFsInodes] = []evictionapi.Signal{evictionapi.SignalImageFsInodesFree} </ins> } // validSignal returns true if the signal is supported.


return res } <ins> func newEphemeralStorageResourceList(ephemeral, cpu, memory string) v1.ResourceList { res := v1.ResourceList{} if ephemeral != "" { res[v1.ResourceEphemeralStorage] = resource.MustParse(ephemeral) } if cpu != "" { res[v1.ResourceCPU] = resource.MustParse(cpu) } if memory != "" { res[v1.ResourceMemory] = resource.MustParse("1Mi") } return res } </ins> func newResourceRequirements(requests, limits v1.ResourceList) v1.ResourceRequirements { res := v1.ResourceRequirements{} res.Requests = requests


ginkgo := filepath.Join(outputDir, "ginkgo") test := filepath.Join(outputDir, "e2e_node.test") <del> if *systemSpecName == "" { runCommand(ginkgo, *ginkgoFlags, test, "--", *testFlags) return </del> <ins> args := []string{*ginkgoFlags, test, "--", *testFlags} if *systemSpecName != "" { rootDir, err := builder.GetK8sRootDir() if err != nil { glog.Fatalf("Failed to get k8s root directory: %v", err) } systemSpecFile := filepath.Join(rootDir, systemSpecPath, *systemSpecName+".yaml") args = append(args, fmt.Sprintf("--system-spec-name=%s --system-spec-file=%s", *systemSpecName, systemSpecFile)) </ins> } <del> rootDir, err := builder.GetK8sRootDir() if err != nil { glog.Fatalf("Failed to get k8s root directory: %v", err) </del> <ins> if err := runCommand(ginkgo, args...); err != nil { glog.Exitf("Test failed: %v", err) </ins> } <del> systemSpecFile := filepath.Join(rootDir, systemSpecPath, *systemSpecName+".yaml") runCommand(ginkgo, *ginkgoFlags, test, "--", fmt.Sprintf("--system-spec-name=%s --system-spec-file=%s", *systemSpecName, systemSpecFile), *testFlags) </del> return }


"//vendor/google.golang.org/api/googleapi:go_default_library", "//vendor/k8s.io/api/batch/v1:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", <ins> "//vendor/k8s.io/api/policy/v1beta1:go_default_library", </ins> "//vendor/k8s.io/api/rbac/v1beta1:go_default_library", "//vendor/k8s.io/api/storage/v1:go_default_library", "//vendor/k8s.io/api/storage/v1beta1:go_default_library",


. "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "k8s.io/api/core/v1" <ins> policy "k8s.io/api/policy/v1beta1" </ins> "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/uuid" <ins> "k8s.io/apimachinery/pkg/util/wait" </ins> clientset "k8s.io/client-go/kubernetes" v1core "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/kubernetes/pkg/api/testapi"


gcePDDetachPollTime = 10 * time.Second nodeStatusTimeout   = 10 * time.Minute nodeStatusPollTime  = 1 * time.Second <ins> podEvictTimeout     = 2 * time.Minute </ins> maxReadRetry        = 3 minNodes            = 2 ) var _ = SIGDescribe("Pod Disks", func() { var ( <ins> ns         string cs         clientset.Interface </ins> podClient  v1core.PodInterface nodeClient v1core.NodeInterface host0Name  types.NodeName


BeforeEach(func() { framework.SkipUnlessNodeCountIsAtLeast(minNodes) <ins> cs = f.ClientSet ns = f.Namespace.Name </ins> <del> podClient = f.ClientSet.Core().Pods(f.Namespace.Name) nodeClient = f.ClientSet.Core().Nodes() nodes = framework.GetReadySchedulableNodesOrDie(f.ClientSet) </del> <ins> podClient = cs.Core().Pods(ns) nodeClient = cs.Core().Nodes() nodes = framework.GetReadySchedulableNodesOrDie(cs) </ins> Expect(len(nodes.Items)).To(BeNumerically(">=", minNodes), fmt.Sprintf("Requires at least %d nodes", minNodes)) host0Name = types.NodeName(nodes.Items[0].ObjectMeta.Name) host1Name = types.NodeName(nodes.Items[1].ObjectMeta.Name)


} }) <del> Context("detach from a disrupted node [Slow] [Disruptive]", func() { </del> <ins> Context("detach in a disrupted environment [Slow] [Disruptive]", func() { </ins> const ( deleteNode    = 1 // delete physical node deleteNodeObj = 2 // delete node's api object only <ins> evictPod      = 3 // evict host0Pod on node0 </ins> ) type testT struct { <del> descr  string // It description nodeOp int    // disruptive operation performed on target node </del> <ins> descr     string // It description disruptOp int    // disruptive operation performed on target node </ins> } tests := []testT{ { <del> descr:  "node is deleted", nodeOp: deleteNode, </del> <ins> descr:     "node is deleted", disruptOp: deleteNode, </ins> }, { <del> descr:  "node's API object is deleted", nodeOp: deleteNodeObj, </del> <ins> descr:     "node's API object is deleted", disruptOp: deleteNodeObj, }, { descr:     "pod is evicted", disruptOp: evictPod, </ins> }, } for _, t := range tests { <del> nodeOp := t.nodeOp </del> <ins> disruptOp := t.disruptOp </ins> It(fmt.Sprintf("when %s", t.descr), func() { framework.SkipUnlessProviderIs("gce") origNodeCnt := len(nodes.Items) // healhy nodes running kublet


diskName, err := framework.CreatePDWithRetry() framework.ExpectNoError(err, "Error creating a pd") <del> targetNode := &nodes.Items[0] </del> <ins> targetNode := &nodes.Items[0] // for node delete ops </ins> host0Pod := testPDPod([]string{diskName}, host0Name, false, 1) containerName := "mycontainer" defer func() { By("defer: cleaning up PD-RW test env") framework.Logf("defer cleanup errors can usually be ignored") <del> if nodeOp == deleteNode { podClient.Delete(host0Pod.Name, metav1.NewDeleteOptions(0)) } </del> <ins> By("defer: delete host0Pod") podClient.Delete(host0Pod.Name, metav1.NewDeleteOptions(0)) By("defer: detach and delete PDs") </ins> detachAndDeletePDs(diskName, []types.NodeName{host0Name}) <del> if nodeOp == deleteNodeObj { targetNode.ObjectMeta.SetResourceVersion("0") // need to set the resource version or else the Create() fails _, err := nodeClient.Create(targetNode) framework.ExpectNoError(err, "defer: Unable to re-create the deleted node") </del> <ins> if disruptOp == deleteNode || disruptOp == deleteNodeObj { if disruptOp == deleteNodeObj { targetNode.ObjectMeta.SetResourceVersion("0") // need to set the resource version or else the Create() fails By("defer: re-create host0 node object") _, err := nodeClient.Create(targetNode) framework.ExpectNoError(err, fmt.Sprintf("defer: Unable to re-create the deleted node object %q", targetNode.Name)) } By("defer: verify the number of ready nodes") numNodes := countReadyNodes(cs, host0Name) // if this defer is reached due to an Expect then nested // Expects are lost, so use Failf here if numNodes != origNodeCnt { framework.Failf("defer: Requires current node count (%d) to return to original node count (%d)", numNodes, origNodeCnt) } </ins> } <del> numNodes := countReadyNodes(f.ClientSet, host0Name) Expect(numNodes).To(Equal(origNodeCnt), fmt.Sprintf("defer: Requires current node count (%d) to return to original node count (%d)", numNodes, origNodeCnt)) </del> }() By("creating host0Pod on node0") _, err = podClient.Create(host0Pod) framework.ExpectNoError(err, fmt.Sprintf("Failed to create host0Pod: %v", err)) <ins> By("waiting for host0Pod to be running") </ins> framework.ExpectNoError(f.WaitForPodRunningSlow(host0Pod.Name)) By("writing content to host0Pod")


By("verifying PD is present in node0's VolumeInUse list") framework.ExpectNoError(waitForPDInVolumesInUse(nodeClient, diskName, host0Name, nodeStatusTimeout, true /* should exist*/)) <del> if nodeOp == deleteNode { </del> <ins> if disruptOp == deleteNode { </ins> By("getting gce instances") gceCloud, err := framework.GetGCECloud() framework.ExpectNoError(err, fmt.Sprintf("Unable to create gcloud client err=%v", err))


By("deleting host0") resp, err := gceCloud.DeleteInstance(framework.TestContext.CloudConfig.ProjectID, framework.TestContext.CloudConfig.Zone, string(host0Name)) framework.ExpectNoError(err, fmt.Sprintf("Failed to delete host0Pod: err=%v response=%#v", err, resp)) <del> By("expecting host0 node to be recreated") numNodes := countReadyNodes(f.ClientSet, host0Name) </del> <ins> By("expecting host0 node to be re-created") numNodes := countReadyNodes(cs, host0Name) </ins> Expect(numNodes).To(Equal(origNodeCnt), fmt.Sprintf("Requires current node count (%d) to return to original node count (%d)", numNodes, origNodeCnt)) output, err = gceCloud.ListInstanceNames(framework.TestContext.CloudConfig.ProjectID, framework.TestContext.CloudConfig.Zone) framework.ExpectNoError(err, fmt.Sprintf("Unable to get list of node instances err=%v output=%s", err, output)) Expect(false, strings.Contains(string(output), string(host0Name))) <del> } else if nodeOp == deleteNodeObj { </del> <ins> } else if disruptOp == deleteNodeObj { </ins> By("deleting host0's node api object") framework.ExpectNoError(nodeClient.Delete(string(host0Name), metav1.NewDeleteOptions(0)), "Unable to delete host0's node object") By("deleting host0Pod") framework.ExpectNoError(podClient.Delete(host0Pod.Name, metav1.NewDeleteOptions(0)), "Unable to delete host0Pod") <ins> } else if disruptOp == evictPod { evictTarget := &policy.Eviction{ ObjectMeta: metav1.ObjectMeta{ Name:      host0Pod.Name, Namespace: ns, }, } By("evicting host0Pod") err = wait.PollImmediate(framework.Poll, podEvictTimeout, func() (bool, error) { err = cs.CoreV1().Pods(ns).Evict(evictTarget) if err != nil { return false, nil } else { return true, nil } }) Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("failed to evict host0Pod after %v", podEvictTimeout)) </ins> } By("waiting for pd to detach from host0")


RUN echo CACHEBUST>/dev/null && clean-install  iptables  <ins> e2fsprogs  </ins> ebtables  ethtool  ca-certificates


REGISTRY?=gcr.io/google-containers IMAGE?=debian-hyperkube-base <del> TAG=0.2 </del> <ins> TAG=0.3 </ins> ARCH?=amd64 CACHEBUST?=1


ARCH?=amd64 HYPERKUBE_BIN?=_output/dockerized/bin/linux/$(ARCH)/hyperkube <del> BASEIMAGE=gcr.io/google-containers/debian-hyperkube-base-$(ARCH):0.2 </del> <ins> BASEIMAGE=gcr.io/google-containers/debian-hyperkube-base-$(ARCH):0.3 </ins> TEMP_DIR:=$(shell mktemp -d -t hyperkubeXXXXXX) all: build


} if len(o.Output) > 0 { <del> return o.PrintObject(o.Cmd, o.Local, o.Mapper, obj, o.Out) </del> <ins> if err := o.PrintObject(o.Cmd, o.Local, o.Mapper, obj, o.Out); err != nil { return err } continue </ins> } cmdutil.PrintSuccess(o.Mapper, o.ShortOutput, o.Out, info.Mapping.Resource, info.Name, false, "env updated")


t.Errorf("did not set env: %s", buf.String()) } } <ins> func TestSetMultiResourcesEnvLocal(t *testing.T) { f, tf, codec, ns := cmdtesting.NewAPIFactory() tf.Client = &fake.RESTClient{ GroupVersion:         legacyscheme.Registry.GroupOrDie(api.GroupName).GroupVersion, NegotiatedSerializer: ns, Client: fake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) { t.Fatalf("unexpected request: %s %#vn%#v", req.Method, req.URL, req) return nil, nil }), } tf.Namespace = "test" tf.ClientConfig = &restclient.Config{ContentConfig: restclient.ContentConfig{GroupVersion: &legacyscheme.Registry.GroupOrDie(api.GroupName).GroupVersion}} buf := bytes.NewBuffer([]byte{}) cmd := NewCmdEnv(f, os.Stdin, buf, buf) cmd.SetOutput(buf) cmd.Flags().Set("output", "name") cmd.Flags().Set("local", "true") mapper, typer := f.Object() tf.Printer = &printers.NamePrinter{Decoders: []runtime.Decoder{codec}, Typer: typer, Mapper: mapper} opts := EnvOptions{FilenameOptions: resource.FilenameOptions{ Filenames: []string{"../../../../test/fixtures/pkg/kubectl/cmd/set/multi-resource-yaml.yaml"}}, Out:   buf, Local: true} err := opts.Complete(f, cmd, []string{"env=prod"}) if err == nil { err = opts.RunEnv(f) } if err != nil { t.Fatalf("unexpected error: %v", err) } expectedOut := "replicationcontrollers/first-rcnreplicationcontrollers/second-rcn" if buf.String() != expectedOut { t.Errorf("expected out:n%snbut got:n%s", expectedOut, buf.String()) } } </ins>


} } } <ins> func TestSetMultiResourcesImageLocal(t *testing.T) { f, tf, codec, ns := cmdtesting.NewAPIFactory() tf.Client = &fake.RESTClient{ GroupVersion:         legacyscheme.Registry.GroupOrDie(api.GroupName).GroupVersion, NegotiatedSerializer: ns, Client: fake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) { t.Fatalf("unexpected request: %s %#vn%#v", req.Method, req.URL, req) return nil, nil }), } tf.Namespace = "test" tf.ClientConfig = &restclient.Config{ContentConfig: restclient.ContentConfig{GroupVersion: &legacyscheme.Registry.GroupOrDie(api.GroupName).GroupVersion}} buf := bytes.NewBuffer([]byte{}) cmd := NewCmdImage(f, buf, buf) cmd.SetOutput(buf) cmd.Flags().Set("output", "name") cmd.Flags().Set("local", "true") mapper, typer := f.Object() tf.Printer = &printers.NamePrinter{Decoders: []runtime.Decoder{codec}, Typer: typer, Mapper: mapper} opts := ImageOptions{FilenameOptions: resource.FilenameOptions{ Filenames: []string{"../../../../test/fixtures/pkg/kubectl/cmd/set/multi-resource-yaml.yaml"}}, Out:   buf, Local: true} err := opts.Complete(f, cmd, []string{"*=thingy"}) if err == nil { err = opts.Validate() } if err == nil { err = opts.Run() } if err != nil { t.Fatalf("unexpected error: %v", err) } expectedOut := "replicationcontrollers/first-rcnreplicationcontrollers/second-rcn" if buf.String() != expectedOut { t.Errorf("expected out:n%snbut got:n%s", expectedOut, buf.String()) } } </ins>


} if o.Local || cmdutil.GetDryRunFlag(o.Cmd) { <del> return o.PrintObject(o.Cmd, o.Local, o.Mapper, info.Object, o.Out) </del> <ins> if err := o.PrintObject(o.Cmd, o.Local, o.Mapper, info.Object, o.Out); err != nil { return err } continue </ins> } obj, err := resource.NewHelper(info.Client, info.Mapping).Patch(info.Namespace, info.Name, types.StrategicMergePatchType, patch.Patch)


t.Errorf("did not set resources: %s", buf.String()) } } <ins> func TestSetMultiResourcesLimitsLocal(t *testing.T) { f, tf, codec, ns := cmdtesting.NewAPIFactory() tf.Client = &fake.RESTClient{ GroupVersion:         legacyscheme.Registry.GroupOrDie(api.GroupName).GroupVersion, NegotiatedSerializer: ns, Client: fake.CreateHTTPClient(func(req *http.Request) (*http.Response, error) { t.Fatalf("unexpected request: %s %#vn%#v", req.Method, req.URL, req) return nil, nil }), } tf.Namespace = "test" tf.ClientConfig = &restclient.Config{ContentConfig: restclient.ContentConfig{GroupVersion: &legacyscheme.Registry.GroupOrDie(api.GroupName).GroupVersion}} buf := bytes.NewBuffer([]byte{}) cmd := NewCmdResources(f, buf, buf) cmd.SetOutput(buf) cmd.Flags().Set("output", "name") cmd.Flags().Set("local", "true") mapper, typer := f.Object() tf.Printer = &printers.NamePrinter{Decoders: []runtime.Decoder{codec}, Typer: typer, Mapper: mapper} opts := ResourcesOptions{FilenameOptions: resource.FilenameOptions{ Filenames: []string{"../../../../test/fixtures/pkg/kubectl/cmd/set/multi-resource-yaml.yaml"}}, Out:               buf, Local:             true, Limits:            "cpu=200m,memory=512Mi", Requests:          "cpu=200m,memory=512Mi", ContainerSelector: "*"} err := opts.Complete(f, cmd, []string{}) if err == nil { err = opts.Validate() } if err == nil { err = opts.Run() } if err != nil { t.Fatalf("unexpected error: %v", err) } expectedOut := "replicationcontrollers/first-rcnreplicationcontrollers/second-rcn" if buf.String() != expectedOut { t.Errorf("expected out:n%snbut got:n%s", expectedOut, buf.String()) } } </ins>


} if o.Local || o.DryRun { <del> return o.PrintObject(o.Mapper, info.Object, o.Out) </del> <ins> if err := o.PrintObject(o.Mapper, info.Object, o.Out); err != nil { return err } continue </ins> } obj, err := resource.NewHelper(info.Client, info.Mapping).Patch(info.Namespace, info.Name, types.StrategicMergePatchType, patch.Patch)


<ins> apiVersion: v1 kind: ReplicationController metadata: name: first-rc spec: replicas: 1 selector: app: mock template: metadata: labels: app: mock spec: containers: - name: mock-container image: gcr.io/google-containers/pause:2.0 --- apiVersion: v1 kind: ReplicationController metadata: name: second-rc spec: replicas: 1 selector: app: mock template: metadata: labels: app: mock spec: containers: - name: mock-container image: gcr.io/google-containers/pause:2.0 </ins>  No newline at end of file


name = "all-srcs", srcs = [ ":package-srcs", <ins> "//staging/src/k8s.io/apiserver/plugin/pkg/audit/buffered:all-srcs", </ins> "//staging/src/k8s.io/apiserver/plugin/pkg/audit/log:all-srcs", "//staging/src/k8s.io/apiserver/plugin/pkg/audit/webhook:all-srcs", ],


<ins> load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", srcs = [ "buffered.go", "doc.go", ], importpath = "k8s.io/apiserver/plugin/pkg/audit/buffered", visibility = ["//visibility:public"], deps = [ "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apiserver/pkg/apis/audit:go_default_library", "//vendor/k8s.io/apiserver/pkg/audit:go_default_library", "//vendor/k8s.io/client-go/util/flowcontrol:go_default_library", ], ) go_test( name = "go_default_test", srcs = ["buffered_test.go"], embed = [":go_default_library"], deps = [ "//vendor/github.com/stretchr/testify/require:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apiserver/pkg/apis/audit:go_default_library", "//vendor/k8s.io/apiserver/pkg/audit:go_default_library", ], ) filegroup( name = "package-srcs", srcs = glob(["**"]), tags = ["automanaged"], visibility = ["//visibility:private"], ) filegroup( name = "all-srcs", srcs = [":package-srcs"], tags = ["automanaged"], visibility = ["//visibility:public"], ) </ins>


<ins> /* Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package buffered import ( "fmt" "sync" "time" "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" auditinternal "k8s.io/apiserver/pkg/apis/audit" "k8s.io/apiserver/pkg/audit" "k8s.io/client-go/util/flowcontrol" ) // The plugin name reported in error metrics. const pluginName = "buffered" const ( // Default configuration values for ModeBatch. defaultBatchBufferSize = 10000            // Buffer up to 10000 events before starting discarding. defaultBatchMaxSize    = 400              // Only send up to 400 events at a time. defaultBatchMaxWait    = 30 * time.Second // Send events at least twice a minute. defaultBatchThrottleQPS   = 10 // Limit the send rate by 10 QPS. defaultBatchThrottleBurst = 15 // Allow up to 15 QPS burst. ) // BatchConfig represents batching delegate audit backend configuration. type BatchConfig struct { // BufferSize defines a size of the buffering queue. BufferSize int // MaxBatchSize defines maximum size of a batch. MaxBatchSize int // MaxBatchWait indicates the maximum interval between two batches. MaxBatchWait time.Duration // ThrottleEnable defines whether throttling will be applied to the batching process. ThrottleEnable bool // ThrottleQPS defines the allowed rate of batches per second sent to the delegate backend. ThrottleQPS float32 // ThrottleBurst defines the maximum rate of batches per second sent to the delegate backend in case // the capacity defined by ThrottleQPS was not utilized. ThrottleBurst int } // NewDefaultBatchConfig returns new Config objects populated by default values. func NewDefaultBatchConfig() BatchConfig { return BatchConfig{ BufferSize:   defaultBatchBufferSize, MaxBatchSize: defaultBatchMaxSize, MaxBatchWait: defaultBatchMaxWait, ThrottleEnable: true, ThrottleQPS:    defaultBatchThrottleQPS, ThrottleBurst:  defaultBatchThrottleBurst, } } type bufferedBackend struct { // The delegate backend that actually exports events. delegateBackend audit.Backend // Channel to buffer events before sending to the delegate backend. buffer chan *auditinternal.Event // Maximum number of events in a batch sent to the delegate backend. maxBatchSize int // Amount of time to wait after sending a batch to the delegate backend before sending another one. // // Receiving maxBatchSize events will always trigger sending a batch, regardless of the amount of time passed. maxBatchWait time.Duration // Channel to signal that the batching routine has processed all remaining events and exited. // Once `shutdownCh` is closed no new events will be sent to the delegate backend. shutdownCh chan struct{} // WaitGroup to control the concurrency of sending batches to the delegate backend. // Worker routine calls Add before sending a batch and // then spawns a routine that calls Done after batch was processed by the delegate backend. // This WaitGroup is used to wait for all sending routines to finish before shutting down audit backend. wg sync.WaitGroup // Limits the number of batches sent to the delegate backend per second. throttle flowcontrol.RateLimiter } var _ audit.Backend = &bufferedBackend{} // NewBackend returns a buffered audit backend that wraps delegate backend. func NewBackend(delegate audit.Backend, config BatchConfig) audit.Backend { var throttle flowcontrol.RateLimiter if config.ThrottleEnable { throttle = flowcontrol.NewTokenBucketRateLimiter(config.ThrottleQPS, config.ThrottleBurst) } return &bufferedBackend{ delegateBackend: delegate, buffer:          make(chan *auditinternal.Event, config.BufferSize), maxBatchSize:    config.MaxBatchSize, maxBatchWait:    config.MaxBatchWait, shutdownCh:      make(chan struct{}), wg:              sync.WaitGroup{}, throttle:        throttle, } } func (b *bufferedBackend) Run(stopCh <-chan struct{}) error { go func() { // Signal that the working routine has exited. defer close(b.shutdownCh) b.processIncomingEvents(stopCh) // Handle the events that were received after the last buffer // scraping and before this line. Since the buffer is closed, no new // events will come through. allEventsProcessed := false timer := make(chan time.Time) for !allEventsProcessed { allEventsProcessed = func() bool { // Recover from any panic in order to try to process all remaining events. // Note, that in case of a panic, the return value will be false and // the loop execution will continue. defer runtime.HandleCrash() events := b.collectEvents(timer, wait.NeverStop) b.processEvents(events) return len(events) == 0 }() } }() return b.delegateBackend.Run(stopCh) } // Shutdown blocks until stopCh passed to the Run method is closed and all // events added prior to that moment are batched and sent to the delegate backend. func (b *bufferedBackend) Shutdown() { // Wait until the routine spawned in Run method exits. <-b.shutdownCh // Wait until all sending routines exit. b.wg.Wait() b.delegateBackend.Shutdown() } // processIncomingEvents runs a loop that collects events from the buffer. When // b.stopCh is closed, processIncomingEvents stops and closes the buffer. func (b *bufferedBackend) processIncomingEvents(stopCh <-chan struct{}) { defer close(b.buffer) t := time.NewTimer(b.maxBatchWait) defer t.Stop() for { func() { // Recover from any panics caused by this function so a panic in the // goroutine can't bring down the main routine. defer runtime.HandleCrash() t.Reset(b.maxBatchWait) b.processEvents(b.collectEvents(t.C, stopCh)) }() select { case <-stopCh: return default: } } } // collectEvents attempts to collect some number of events in a batch. // // The following things can cause collectEvents to stop and return the list // of events: // //   * Maximum number of events for a batch. //   * Timer has passed. //   * Buffer channel is closed and empty. //   * stopCh is closed. func (b *bufferedBackend) collectEvents(timer <-chan time.Time, stopCh <-chan struct{}) []*auditinternal.Event { var events []*auditinternal.Event L: for i := 0; i < b.maxBatchSize; i++ { select { case ev, ok := <-b.buffer: // Buffer channel was closed and no new events will follow. if !ok { break L } events = append(events, ev) case <-timer: // Timer has expired. Send currently accumulated batch. break L case <-stopCh: // Backend has been stopped. Send currently accumulated batch. break L } } return events } // processEvents process the batch events in a goroutine using delegateBackend's ProcessEvents. func (b *bufferedBackend) processEvents(events []*auditinternal.Event) { if len(events) == 0 { return } // TODO(audit): Should control the number of active goroutines // if one goroutine takes 5 seconds to finish, the number of goroutines can be 5 * defaultBatchThrottleQPS if b.throttle != nil { b.throttle.Accept() } b.wg.Add(1) go func() { defer b.wg.Done() defer runtime.HandleCrash() // Execute the real processing in a goroutine to keep it from blocking. // This lets the batching routine continue draining the queue immediately. b.delegateBackend.ProcessEvents(events...) }() } func (b *bufferedBackend) ProcessEvents(ev ...*auditinternal.Event) { // The following mechanism is in place to support the situation when audit // events are still coming after the backend was stopped. var sendErr error var evIndex int // If the delegateBackend was shutdown and the buffer channel was closed, an // attempt to add an event to it will result in panic that we should // recover from. defer func() { if err := recover(); err != nil { sendErr = fmt.Errorf("panic when processing events: %v", err) } if sendErr != nil { audit.HandlePluginError(pluginName, sendErr, ev[evIndex:]...) } }() for i, e := range ev { evIndex = i // Per the audit.Backend interface these events are reused after being // sent to the Sink. Deep copy and send the copy to the queue. event := e.DeepCopy() select { case b.buffer <- event: default: sendErr = fmt.Errorf("audit buffer queue blocked") return } } } </ins>


<ins> /* Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package buffered import ( "testing" "time" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/util/wait" auditinternal "k8s.io/apiserver/pkg/apis/audit" "k8s.io/apiserver/pkg/audit" ) var ( closedStopCh = func() <-chan struct{} { ch := make(chan struct{}) close(ch) return ch }() infiniteTimeCh <-chan time.Time = make(chan time.Time) closedTimeCh                    = func() <-chan time.Time { ch := make(chan time.Time) close(ch) return ch }() ) func newEvents(number int) []*auditinternal.Event { events := make([]*auditinternal.Event, number) for i := range events { events[i] = &auditinternal.Event{} } return events } func TestBufferedBackendCollectEvents(t *testing.T) { config := NewDefaultBatchConfig() testCases := []struct { desc          string timer         <-chan time.Time stopCh        <-chan struct{} numEvents     int wantBatchSize int }{ { desc:          "max batch size encountered", timer:         infiniteTimeCh, stopCh:        wait.NeverStop, numEvents:     config.MaxBatchSize + 1, wantBatchSize: config.MaxBatchSize, }, { desc:   "timer expired", timer:  closedTimeCh, stopCh: wait.NeverStop, }, { desc:   "chanel closed", timer:  infiniteTimeCh, stopCh: closedStopCh, }, } for _, tc := range testCases { tc := tc t.Run(tc.desc, func(t *testing.T) { t.Parallel() backend := NewBackend(&fakeBackend{}, config).(*bufferedBackend) backend.ProcessEvents(newEvents(tc.numEvents)...) batch := backend.collectEvents(tc.timer, tc.stopCh) require.Equal(t, tc.wantBatchSize, len(batch), "unexpected batch size") }) } } func TestBufferedBackendProcessEventsAfterStop(t *testing.T) { t.Parallel() backend := NewBackend(&fakeBackend{}, NewDefaultBatchConfig()).(*bufferedBackend) backend.Run(closedStopCh) backend.Shutdown() backend.ProcessEvents(newEvents(1)...) batch := backend.collectEvents(infiniteTimeCh, wait.NeverStop) require.Equal(t, 0, len(batch), "processed events after the backed has been stopped") } func TestBufferedBackendProcessEventsBufferFull(t *testing.T) { t.Parallel() config := NewDefaultBatchConfig() config.BufferSize = 1 backend := NewBackend(&fakeBackend{}, config).(*bufferedBackend) backend.ProcessEvents(newEvents(2)...) require.Equal(t, 1, len(backend.buffer), "buffed contains more elements than it should") } func TestBufferedBackendShutdownWaitsForDelegatedCalls(t *testing.T) { t.Parallel() delegatedCallStartCh := make(chan struct{}) delegatedCallEndCh := make(chan struct{}) delegateBackend := &fakeBackend{ onRequest: func(_ []*auditinternal.Event) { close(delegatedCallStartCh) <-delegatedCallEndCh }, } config := NewDefaultBatchConfig() backend := NewBackend(delegateBackend, config) // Run backend, process events, wait for them to be batched and for delegated call to start. stopCh := make(chan struct{}) backend.Run(stopCh) backend.ProcessEvents(newEvents(config.MaxBatchSize)...) <-delegatedCallStartCh // Start shutdown procedure. shutdownEndCh := make(chan struct{}) go func() { close(stopCh) backend.Shutdown() close(shutdownEndCh) }() // Wait for some time and then check whether Shutdown has exited. Can give false positive, // but never false negative. time.Sleep(100 * time.Millisecond) select { case <-shutdownEndCh: t.Fatalf("Shutdown exited before delegated call ended") default: } // Wait for Shutdown to exit after delegated call has exited. close(delegatedCallEndCh) <-shutdownEndCh } type fakeBackend struct { onRequest func(events []*auditinternal.Event) } var _ audit.Backend = &fakeBackend{} func (b *fakeBackend) Run(stopCh <-chan struct{}) error { return nil } func (b *fakeBackend) Shutdown() { return } func (b *fakeBackend) ProcessEvents(ev ...*auditinternal.Event) { if b.onRequest != nil { b.onRequest(ev) } } </ins>


<ins> /* Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Package buffered provides an implementation for the audit.Backend interface // that batches incoming audit events and sends batches to the delegate audit.Backend. package buffered // import "k8s.io/apiserver/plugin/pkg/audit/buffered" </ins>


// ServiceAnnotationLoadBalancerInternal is the annotation used on the service // to indicate that we want an internal loadbalancer service. <del> // If the value of ServiceAnnotationLoadBalancerInternal is false, it indicates that we want an external loadbalancer service. Default to true. </del> <ins> // If the value of ServiceAnnotationLoadBalancerInternal is false, it indicates that we want an external loadbalancer service. Default to false. </ins> ServiceAnnotationLoadBalancerInternal = "service.beta.kubernetes.io/openstack-internal-load-balancer" )


glog.V(4).Infof("EnsureLoadBalancer using floatingPool: %v", floatingPool) var internalAnnotation bool <del> internal := getStringFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerInternal, "true") </del> <ins> internal := getStringFromServiceAnnotation(apiService, ServiceAnnotationLoadBalancerInternal, "false") </ins> switch internal { case "true": glog.V(4).Infof("Ensure an internal loadbalancer service.")


glog.V(4).Infof("Ensure an external loadbalancer service.") internalAnnotation = false } else { <del> return nil, fmt.Errorf("floating-network-id or loadbalancer.openstack.org/floating-network-id should be specified when service.beta.kubernetes.io/openstack-internal-load-balancer is false") </del> <ins> return nil, fmt.Errorf("floating-network-id or loadbalancer.openstack.org/floating-network-id should be specified when ensuring an external loadbalancer service.") </ins> } default: return nil, fmt.Errorf("unknow service.beta.kubernetes.io/openstack-internal-load-balancer annotation: %v, specify "true" or "false".",


pod, err = c.CoreV1().Pods(ns).Get(pod.Name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) <del> // Wait for `VolumeStatsAggPeriod' to grab metrics time.Sleep(1 * time.Minute) // Grab kubelet metrics from the node the pod was scheduled on kubeMetrics, err := metricsGrabber.GrabFromKubelet(pod.Spec.NodeName) Expect(err).NotTo(HaveOccurred(), "Error getting kubelet metrics : %v", err) framework.Logf("Deleting pod %q/%q", pod.Namespace, pod.Name) framework.ExpectNoError(framework.DeletePodWithWait(f, c, pod)) </del> // Verify volume stat metrics were collected for the referenced PVC volumeStatKeys := []string{ kubeletmetrics.VolumeStatsUsedBytesKey,


kubeletmetrics.VolumeStatsInodesFreeKey, kubeletmetrics.VolumeStatsInodesUsedKey, } <ins> // Poll kubelet metrics waiting for the volume to be picked up // by the volume stats collector var kubeMetrics metrics.KubeletMetrics waitErr := wait.Poll(30*time.Second, 5*time.Minute, func() (bool, error) { framework.Logf("Grabbing Kubelet metrics") // Grab kubelet metrics from the node the pod was scheduled on var err error kubeMetrics, err = metricsGrabber.GrabFromKubelet(pod.Spec.NodeName) if err != nil { framework.Logf("Error fetching kubelet metrics") return false, err } key := volumeStatKeys[0] kubeletKeyName := fmt.Sprintf("%s_%s", kubeletmetrics.KubeletSubsystem, key) if !findVolumeStatMetric(kubeletKeyName, pvc.Namespace, pvc.Name, kubeMetrics) { return false, nil } return true, nil }) Expect(waitErr).NotTo(HaveOccurred(), "Error finding volume metrics : %v", waitErr) </ins> for _, key := range volumeStatKeys { kubeletKeyName := fmt.Sprintf("%s_%s", kubeletmetrics.KubeletSubsystem, key) <del> verifyVolumeStatMetric(kubeletKeyName, pvc.Namespace, pvc.Name, kubeMetrics) </del> <ins> found := findVolumeStatMetric(kubeletKeyName, pvc.Namespace, pvc.Name, kubeMetrics) Expect(found).To(BeTrue(), "PVC %s, Namespace %s not found for %s", pvc.Name, pvc.Namespace, kubeletKeyName) </ins> } <ins> framework.Logf("Deleting pod %q/%q", pod.Namespace, pod.Name) framework.ExpectNoError(framework.DeletePodWithWait(f, c, pod)) </ins> }) })


return result } <del> // Verifies the specified metrics are in `kubeletMetrics` func verifyVolumeStatMetric(metricKeyName string, namespace string, pvcName string, kubeletMetrics metrics.KubeletMetrics) { </del> <ins> // Finds the sample in the specified metric from `KubeletMetrics` tagged with // the specified namespace and pvc name func findVolumeStatMetric(metricKeyName string, namespace string, pvcName string, kubeletMetrics metrics.KubeletMetrics) bool { </ins> found := false errCount := 0 <ins> framework.Logf("Looking for sample in metric `%s` tagged with namespace `%s`, PVC `%s`", metricKeyName, namespace, pvcName) </ins> if samples, ok := kubeletMetrics[metricKeyName]; ok { for _, sample := range samples { <ins> framework.Logf("Found sample %s", sample.String()) </ins> samplePVC, ok := sample.Metric["persistentvolumeclaim"] if !ok { framework.Logf("Error getting pvc for metric %s, sample %s", metricKeyName, sample.String())


} } Expect(errCount).To(Equal(0), "Found invalid samples") <del> Expect(found).To(BeTrue(), "PVC %s, Namespace %s not found for %s", pvcName, namespace, metricKeyName) </del> <ins> return found </ins> }


PortOpenCheck{port: 10252}, HTTPProxyCheck{Proto: "https", Host: cfg.API.AdvertiseAddress, Port: int(cfg.API.BindPort)}, DirAvailableCheck{Path: filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName)}, <del> DirAvailableCheck{Path: "/var/lib/kubelet"}, </del> FileContentCheck{Path: bridgenf, Content: []byte{'1'}}, SwapCheck{}, InPathCheck{executable: "ip", mandatory: true},


ServiceCheck{Service: "docker", CheckIfActive: true}, PortOpenCheck{port: 10250}, DirAvailableCheck{Path: filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.ManifestsSubDirName)}, <del> DirAvailableCheck{Path: "/var/lib/kubelet"}, </del> FileAvailableCheck{Path: cfg.CACertPath}, FileAvailableCheck{Path: filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.KubeletKubeConfigFileName)}, FileContentCheck{Path: bridgenf, Content: []byte{'1'}},


RequireKubeConfig:       false, KubeConfig:              flag.NewStringFlag("/var/lib/kubelet/kubeconfig"), ContainerRuntimeOptions: *NewContainerRuntimeOptions(), <del> CertDirectory:           "/var/run/kubernetes", </del> <ins> CertDirectory:           "/var/lib/kubelet/pki", </ins> RootDirectory:           v1alpha1.DefaultRootDir, // DEPRECATED: auto detecting cloud providers goes against the initiative // for out-of-tree cloud providers as we'll now depend on cAdvisor integrations


"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", <ins> "//vendor/k8s.io/apimachinery/pkg/types:go_default_library", </ins> "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//vendor/k8s.io/apimachinery/pkg/watch:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",


metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" <ins> "k8s.io/apimachinery/pkg/types" </ins> "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/dynamic"


} } <ins> func TestPatch(t *testing.T) { stopCh, apiExtensionClient, clientPool, err := testserver.StartDefaultServer() if err != nil { t.Fatal(err) } defer close(stopCh) noxuDefinition := testserver.NewNoxuCustomResourceDefinition(apiextensionsv1beta1.ClusterScoped) noxuVersionClient, err := testserver.CreateNewCustomResourceDefinition(noxuDefinition, apiExtensionClient, clientPool) if err != nil { t.Fatal(err) } ns := "not-the-default" noxuNamespacedResourceClient := noxuVersionClient.Resource(&metav1.APIResource{ Name:       noxuDefinition.Spec.Names.Plural, Namespaced: true, }, ns) noxuInstanceToCreate := testserver.NewNoxuInstance(ns, "foo") createdNoxuInstance, err := noxuNamespacedResourceClient.Create(noxuInstanceToCreate) if err != nil { t.Fatal(err) } patch := []byte(`{"num": {"num2":999}}`) createdNoxuInstance, err = noxuNamespacedResourceClient.Patch("foo", types.MergePatchType, patch) if err != nil { t.Fatalf("unexpected error: %v", err) } // a patch with no change createdNoxuInstance, err = noxuNamespacedResourceClient.Patch("foo", types.MergePatchType, patch) if err != nil { t.Fatalf("unexpected error: %v", err) } // an empty patch createdNoxuInstance, err = noxuNamespacedResourceClient.Patch("foo", types.MergePatchType, []byte(`{}`)) if err != nil { t.Fatalf("unexpected error: %v", err) } originalJSON, err := runtime.Encode(unstructured.UnstructuredJSONScheme, createdNoxuInstance) if err != nil { t.Fatalf("unexpected error: %v", err) } gottenNoxuInstance, err := runtime.Decode(unstructured.UnstructuredJSONScheme, originalJSON) if err != nil { t.Fatalf("unexpected error: %v", err) } // Check if int is preserved. unstructuredObj := gottenNoxuInstance.(*unstructured.Unstructured).Object num := unstructuredObj["num"].(map[string]interface{}) num1 := num["num1"].(int64) num2 := num["num2"].(int64) if num1 != 9223372036854775807 || num2 != 999 { t.Errorf("Expected %v, got %v, %v", `9223372036854775807, 999`, num1, num2) } } </ins> func TestCrossNamespaceListWatch(t *testing.T) { stopCh, apiExtensionClient, clientPool, err := testserver.StartDefaultServer() if err != nil {


if err != nil { return err } <del> mustCheckData = false continue </del> <ins> if !bytes.Equal(data, origState.data) { // original data changed, restart loop mustCheckData = false continue } </ins> } return decode(s.codec, s.versioner, origState.data, out, origState.rev) }


CPUManager utilfeature.Feature = "CPUManager" // owner: @derekwaynecarr <del> // alpha: v1.8 </del> <ins> // beta: v1.10 </ins> // // Enable pods to consume pre-allocated huge pages of varying page sizes HugePages utilfeature.Feature = "HugePages"


RotateKubeletClientCertificate:              {Default: true, PreRelease: utilfeature.Beta}, PersistentLocalVolumes:                      {Default: false, PreRelease: utilfeature.Alpha}, LocalStorageCapacityIsolation:               {Default: false, PreRelease: utilfeature.Alpha}, <del> HugePages:                                   {Default: false, PreRelease: utilfeature.Alpha}, </del> <ins> HugePages:                                   {Default: true, PreRelease: utilfeature.Beta}, </ins> DebugContainers:                             {Default: false, PreRelease: utilfeature.Alpha}, PodPriority:                                 {Default: false, PreRelease: utilfeature.Alpha}, EnableEquivalenceClassCache:                 {Default: false, PreRelease: utilfeature.Alpha},


RUN ln -s /bin/sh /bin/bash RUN echo CACHEBUST>/dev/null && clean-install  <del> iptables  </del> <ins> ca-certificates  ceph-common  cifs-utils  conntrack  </ins> e2fsprogs  ebtables  ethtool  <del> kmod  ca-certificates  conntrack  util-linux  socat  </del> git  <ins> glusterfs-client  iptables  </ins> jq  <ins> kmod  openssh-client  </ins> nfs-common  <del> glusterfs-client  cifs-utils  ceph-common </del> <ins> socat  util-linux </ins> COPY cni-bin/bin /opt/cni/bin


REGISTRY?=gcr.io/google-containers IMAGE?=debian-hyperkube-base <del> TAG=0.5 </del> <ins> TAG=0.6 </ins> ARCH?=amd64 CACHEBUST?=1


docker_pull( name = "debian-hyperkube-base-amd64", <del> digest = "sha256:d216b425004fcb6d8047f74e81b30e7ead55f73e73511ca53a329c358786b6c9", </del> <ins> digest = "sha256:10546d592e58d5fdb2e25d79f291b8ac62c8d3a3d83337ad7309cca766dbebce", </ins> registry = "gcr.io", repository = "google-containers/debian-hyperkube-base-amd64", <del> tag = "0.5",  # ignored, but kept here for documentation </del> <ins> tag = "0.6",  # ignored, but kept here for documentation </ins> ) docker_pull(


ARCH?=amd64 HYPERKUBE_BIN?=_output/dockerized/bin/linux/$(ARCH)/hyperkube <del> BASEIMAGE=gcr.io/google-containers/debian-hyperkube-base-$(ARCH):0.5 </del> <ins> BASEIMAGE=gcr.io/google-containers/debian-hyperkube-base-$(ARCH):0.6 </ins> TEMP_DIR:=$(shell mktemp -d -t hyperkubeXXXXXX) all: build


var _ volume.Mounter = &azureDiskMounter{} func (m *azureDiskMounter) GetAttributes() volume.Attributes { <del> volumeSource, _ := getVolumeSource(m.spec) </del> <ins> readOnly := false volumeSource, err := getVolumeSource(m.spec) if err != nil && volumeSource.ReadOnly != nil { readOnly = *volumeSource.ReadOnly } </ins> return volume.Attributes{ <del> ReadOnly:        *volumeSource.ReadOnly, Managed:         !*volumeSource.ReadOnly, </del> <ins> ReadOnly:        readOnly, Managed:         !readOnly, </ins> SupportsSELinux: true, } }


options := []string{"bind"} <del> if *volumeSource.ReadOnly { </del> <ins> if volumeSource.ReadOnly != nil && *volumeSource.ReadOnly { </ins> options = append(options, "ro") }


return mountErr } <del> if !*volumeSource.ReadOnly { </del> <ins> if volumeSource.ReadOnly == nil || !*volumeSource.ReadOnly { </ins> volume.SetVolumeOwnership(m, fsGroup) }


annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: <del> affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 podAffinityTerm: labelSelector: matchExpressions: - key: k8s-app operator: In values: ["kube-dns"] topologyKey: kubernetes.io/hostname </del> tolerations: - key: "CriticalAddonsOnly" operator: "Exists"


CreateLoadBalancer(*elb.CreateLoadBalancerInput) (*elb.CreateLoadBalancerOutput, error) DeleteLoadBalancer(*elb.DeleteLoadBalancerInput) (*elb.DeleteLoadBalancerOutput, error) DescribeLoadBalancers(*elb.DescribeLoadBalancersInput) (*elb.DescribeLoadBalancersOutput, error) <ins> AddTags(*elb.AddTagsInput) (*elb.AddTagsOutput, error) </ins> RegisterInstancesWithLoadBalancer(*elb.RegisterInstancesWithLoadBalancerInput) (*elb.RegisterInstancesWithLoadBalancerOutput, error) DeregisterInstancesFromLoadBalancer(*elb.DeregisterInstancesFromLoadBalancerInput) (*elb.DeregisterInstancesFromLoadBalancerOutput, error) CreateLoadBalancerPolicy(*elb.CreateLoadBalancerPolicyInput) (*elb.CreateLoadBalancerPolicyOutput, error)


return ret, nil } <ins> func (c *Cloud) addLoadBalancerTags(loadBalancerName string, requested map[string]string) error { var tags []*elb.Tag for k, v := range requested { tag := &elb.Tag{ Key:   aws.String(k), Value: aws.String(v), } tags = append(tags, tag) } request := &elb.AddTagsInput{} request.LoadBalancerNames = []*string{&loadBalancerName} request.Tags = tags _, err := c.elb.AddTags(request) if err != nil { return fmt.Errorf("error adding tags to load balancer: %v", err) } return nil } </ins> // Retrieves instance's vpc id from metadata func (c *Cloud) findVPCID() (string, error) { macs, err := c.metadata.GetMetadata("network/interfaces/macs/")


panic("Not implemented") } <ins> func (elb *FakeELB) AddTags(input *elb.AddTagsInput) (*elb.AddTagsOutput, error) { panic("Not implemented") } </ins> func (elb *FakeELB) RegisterInstancesWithLoadBalancer(*elb.RegisterInstancesWithLoadBalancerInput) (*elb.RegisterInstancesWithLoadBalancerOutput, error) { panic("Not implemented") }


} } } <ins> { // Add additional tags glog.V(2).Infof("Creating additional load balancer tags for %s", loadBalancerName) tags := getLoadBalancerAdditionalTags(annotations) if len(tags) > 0 { err := c.addLoadBalancerTags(loadBalancerName, tags) if err != nil { return nil, fmt.Errorf("unable to create additional load balancer tags: %v", err) } } } </ins> } // Whether the ELB was new or existing, sync attributes regardless. This accounts for things


}) } <ins> func (m *MockedFakeELB) AddTags(input *elb.AddTagsInput) (*elb.AddTagsOutput, error) { args := m.Called(input) return args.Get(0).(*elb.AddTagsOutput), nil } </ins> func TestReadAWSCloudConfig(t *testing.T) { tests := []struct { name string


} } <ins> // Test that we can add a load balancer tag func TestAddLoadBalancerTags(t *testing.T) { loadBalancerName := "test-elb" awsServices := newMockedFakeAWSServices(TestClusterId) c, _ := newAWSCloud(strings.NewReader("[global]"), awsServices) want := make(map[string]string) want["tag1"] = "val1" expectedAddTagsRequest := &elb.AddTagsInput{ LoadBalancerNames: []*string{&loadBalancerName}, Tags: []*elb.Tag{ { Key:   aws.String("tag1"), Value: aws.String("val1"), }, }, } awsServices.elb.(*MockedFakeELB).On("AddTags", expectedAddTagsRequest).Return(&elb.AddTagsOutput{}) err := c.addLoadBalancerTags(loadBalancerName, want) assert.Nil(t, err, "Error adding load balancer tags: %v", err) awsServices.elb.(*MockedFakeELB).AssertExpectations(t) } </ins> func newMockedFakeAWSServices(id string) *FakeAWSServices { s := NewFakeAWSServices(id) s.ec2 = &MockedFakeEC2{FakeEC2Impl: s.ec2.(*FakeEC2Impl)}


ec2Device := "/dev/xvd" + string(mountDevice) if !alreadyAttached { <ins> available, err := c.checkIfAvailable(disk, "attaching", awsInstance.awsID) if !available { attachEnded = true return "", err } </ins> request := &ec2.AttachVolumeInput{ Device:     aws.String(ec2Device), InstanceId: aws.String(awsInstance.awsID),


if err != nil { return false, err } <ins> available, err := c.checkIfAvailable(awsDisk, "deleting", "") if !available { return false, err } </ins> return awsDisk.deleteVolume() } <ins> func (c *Cloud) checkIfAvailable(disk *awsDisk, opName string, instance string) (bool, error) { info, err := disk.describeVolume() if err != nil { glog.Errorf("Error describing volume %q: %q", disk.awsID, err) // if for some reason we can not describe volume we will return error return false, err } volumeState := aws.StringValue(info.State) opError := fmt.Sprintf("Error %s EBS volume %q", opName, disk.awsID) if len(instance) != 0 { opError = fmt.Sprintf("%q to instance %q", opError, instance) } // Only available volumes can be attached or deleted if volumeState != "available" { // Volume is attached somewhere else and we can not attach it here if len(info.Attachments) > 0 { attachment := info.Attachments[0] attachErr := fmt.Errorf("%s since volume is currently attached to %q", opError, aws.StringValue(attachment.InstanceId)) glog.Error(attachErr) return false, attachErr } attachErr := fmt.Errorf("%s since volume is in %q state", opError, volumeState) glog.Error(attachErr) return false, attachErr } return true, nil } </ins> func (c *Cloud) GetLabelsForVolume(pv *v1.PersistentVolume) (map[string]string, error) { // Ignore any volumes that are being provisioned if pv.Spec.AWSElasticBlockStore.VolumeID == volume.ProvisionedVolumeName {


"//pkg/kubelet/container:go_default_library", "//pkg/kubelet/events:go_default_library", "//pkg/kubelet/images:go_default_library", <ins> "//pkg/kubelet/kuberuntime/logs:go_default_library", </ins> "//pkg/kubelet/lifecycle:go_default_library", "//pkg/kubelet/metrics:go_default_library", "//pkg/kubelet/prober/results:go_default_library",


"//pkg/util/tail:go_default_library", "//pkg/util/version:go_default_library", "//vendor/github.com/armon/circbuf:go_default_library", <del> "//vendor/github.com/docker/docker/pkg/jsonlog:go_default_library", "//vendor/github.com/fsnotify/fsnotify:go_default_library", </del> "//vendor/github.com/golang/glog:go_default_library", "//vendor/github.com/google/cadvisor/info/v1:go_default_library", "//vendor/google.golang.org/grpc:go_default_library",


"kuberuntime_container_test.go", "kuberuntime_gc_test.go", "kuberuntime_image_test.go", <del> "kuberuntime_logs_test.go", </del> "kuberuntime_manager_test.go", "kuberuntime_sandbox_test.go", "labels_test.go",


filegroup( name = "all-srcs", <del> srcs = [":package-srcs"], </del> <ins> srcs = [ ":package-srcs", "//pkg/kubelet/kuberuntime/logs:all-srcs", ], </ins> tags = ["automanaged"], )


package kuberuntime import ( <del> "bufio" "bytes" "encoding/json" "errors" "fmt" </del> "io" <del> "math" "os" </del> "time" <del> "github.com/docker/docker/pkg/jsonlog" "github.com/fsnotify/fsnotify" "github.com/golang/glog" </del> "k8s.io/api/core/v1" <del> runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime" "k8s.io/kubernetes/pkg/util/tail" </del> <ins> "k8s.io/kubernetes/pkg/kubelet/kuberuntime/logs" </ins> ) <del> // Notice that the current kuberuntime logs implementation doesn't handle // log rotation. // * It will not retrieve logs in rotated log file. // * If log rotation happens when following the log: //   * If the rotation is using create mode, we'll still follow the old file. //   * If the rotation is using copytruncate, we'll be reading at the original position and get nothing. // TODO(random-liu): Support log rotation. // streamType is the type of the stream. type streamType string const ( stderrType streamType = "stderr" stdoutType streamType = "stdout" // timeFormat is the time format used in the log. timeFormat = time.RFC3339Nano // blockSize is the block size used in tail. blockSize = 1024 // stateCheckPeriod is the period to check container state while following // the container log. Kubelet should not keep following the log when the // container is not running. stateCheckPeriod = 5 * time.Second ) var ( // eol is the end-of-line sign in the log. eol = []byte{'n'} // delimiter is the delimiter for timestamp and streamtype in log line. delimiter = []byte{' '} ) // logMessage is the internal log type. type logMessage struct { timestamp time.Time stream    streamType log       []byte } // reset resets the log to nil. func (l *logMessage) reset() { l.timestamp = time.Time{} l.stream = "" l.log = nil } // logOptions is the internal type of all log options. type logOptions struct { tail      int64 bytes     int64 since     time.Time follow    bool timestamp bool } // newLogOptions convert the v1.PodLogOptions to internal logOptions. func newLogOptions(apiOpts *v1.PodLogOptions, now time.Time) *logOptions { opts := &logOptions{ tail:      -1, // -1 by default which means read all logs. bytes:     -1, // -1 by default which means read all logs. follow:    apiOpts.Follow, timestamp: apiOpts.Timestamps, } if apiOpts.TailLines != nil { opts.tail = *apiOpts.TailLines } if apiOpts.LimitBytes != nil { opts.bytes = *apiOpts.LimitBytes } if apiOpts.SinceSeconds != nil { opts.since = now.Add(-time.Duration(*apiOpts.SinceSeconds) * time.Second) } if apiOpts.SinceTime != nil && apiOpts.SinceTime.After(opts.since) { opts.since = apiOpts.SinceTime.Time } return opts } </del> // ReadLogs read the container log and redirect into stdout and stderr. // Note that containerID is only needed when following the log, or else // just pass in empty string "". func (m *kubeGenericRuntimeManager) ReadLogs(path, containerID string, apiOpts *v1.PodLogOptions, stdout, stderr io.Writer) error { <del> f, err := os.Open(path) if err != nil { return fmt.Errorf("failed to open log file %q: %v", path, err) } defer f.Close() </del> // Convert v1.PodLogOptions into internal log options. <del> opts := newLogOptions(apiOpts, time.Now()) // Search start point based on tail line. start, err := tail.FindTailLineStartIndex(f, opts.tail) if err != nil { return fmt.Errorf("failed to tail %d lines of log file %q: %v", opts.tail, path, err) } if _, err := f.Seek(start, os.SEEK_SET); err != nil { return fmt.Errorf("failed to seek %d in log file %q: %v", start, path, err) } // Start parsing the logs. r := bufio.NewReader(f) // Do not create watcher here because it is not needed if `Follow` is false. var watcher *fsnotify.Watcher var parse parseFunc writer := newLogWriter(stdout, stderr, opts) msg := &logMessage{} for { l, err := r.ReadBytes(eol[0]) if err != nil { if err != io.EOF { // This is an real error return fmt.Errorf("failed to read log file %q: %v", path, err) } if !opts.follow { // Return directly when reading to the end if not follow. if len(l) > 0 { glog.Warningf("Incomplete line in log file %q: %q", path, l) } glog.V(2).Infof("Finish parsing log file %q", path) return nil } // Reset seek so that if this is an incomplete line, // it will be read again. if _, err := f.Seek(-int64(len(l)), os.SEEK_CUR); err != nil { return fmt.Errorf("failed to reset seek in log file %q: %v", path, err) } if watcher == nil { // Intialize the watcher if it has not been initialized yet. if watcher, err = fsnotify.NewWatcher(); err != nil { return fmt.Errorf("failed to create fsnotify watcher: %v", err) } defer watcher.Close() if err := watcher.Add(f.Name()); err != nil { return fmt.Errorf("failed to watch file %q: %v", f.Name(), err) } } // Wait until the next log change. if found, err := m.waitLogs(containerID, watcher); !found { return err } continue } if parse == nil { // Intialize the log parsing function. parse, err = getParseFunc(l) if err != nil { return fmt.Errorf("failed to get parse function: %v", err) } } // Parse the log line. msg.reset() if err := parse(l, msg); err != nil { glog.Errorf("Failed with err %v when parsing log for log file %q: %q", err, path, l) continue } // Write the log line into the stream. if err := writer.write(msg); err != nil { if err == errMaximumWrite { glog.V(2).Infof("Finish parsing log file %q, hit bytes limit %d(bytes)", path, opts.bytes) return nil } glog.Errorf("Failed with err %v when writing log for log file %q: %+v", err, path, msg) return err } } } // waitLogs wait for the next log write. It returns a boolean and an error. The boolean // indicates whether a new log is found; the error is error happens during waiting new logs. func (m *kubeGenericRuntimeManager) waitLogs(id string, w *fsnotify.Watcher) (bool, error) { errRetry := 5 for { select { case e := <-w.Events: switch e.Op { case fsnotify.Write: return true, nil default: glog.Errorf("Unexpected fsnotify event: %v, retrying...", e) } case err := <-w.Errors: glog.Errorf("Fsnotify watch error: %v, %d error retries remaining", err, errRetry) if errRetry == 0 { return false, err } errRetry-- case <-time.After(stateCheckPeriod): s, err := m.runtimeService.ContainerStatus(id) if err != nil { return false, err } // Only keep following container log when it is running. if s.State != runtimeapi.ContainerState_CONTAINER_RUNNING { glog.Errorf("Container %q is not running (state=%q)", id, s.State) // Do not return error because it's normal that the container stops // during waiting. return false, nil } } } } // parseFunc is a function parsing one log line to the internal log type. // Notice that the caller must make sure logMessage is not nil. type parseFunc func([]byte, *logMessage) error var parseFuncs []parseFunc = []parseFunc{ parseCRILog,        // CRI log format parse function parseDockerJSONLog, // Docker JSON log format parse function } // parseCRILog parses logs in CRI log format. CRI Log format example: //   2016-10-06T00:17:09.669794202Z stdout log content 1 //   2016-10-06T00:17:09.669794203Z stderr log content 2 func parseCRILog(log []byte, msg *logMessage) error { var err error // Parse timestamp idx := bytes.Index(log, delimiter) if idx < 0 { return fmt.Errorf("timestamp is not found") } msg.timestamp, err = time.Parse(timeFormat, string(log[:idx])) if err != nil { return fmt.Errorf("unexpected timestamp format %q: %v", timeFormat, err) } // Parse stream type log = log[idx+1:] idx = bytes.Index(log, delimiter) if idx < 0 { return fmt.Errorf("stream type is not found") } msg.stream = streamType(log[:idx]) if msg.stream != stdoutType && msg.stream != stderrType { return fmt.Errorf("unexpected stream type %q", msg.stream) } // Get log content msg.log = log[idx+1:] return nil } // parseDockerJSONLog parses logs in Docker JSON log format. Docker JSON log format // example: //   {"log":"content 1","stream":"stdout","time":"2016-10-20T18:39:20.57606443Z"} //   {"log":"content 2","stream":"stderr","time":"2016-10-20T18:39:20.57606444Z"} func parseDockerJSONLog(log []byte, msg *logMessage) error { var l = &jsonlog.JSONLog{} l.Reset() // TODO: JSON decoding is fairly expensive, we should evaluate this. if err := json.Unmarshal(log, l); err != nil { return fmt.Errorf("failed with %v to unmarshal log %q", err, l) } msg.timestamp = l.Created msg.stream = streamType(l.Stream) msg.log = []byte(l.Log) return nil } // getParseFunc returns proper parse function based on the sample log line passed in. func getParseFunc(log []byte) (parseFunc, error) { for _, p := range parseFuncs { if err := p(log, &logMessage{}); err == nil { return p, nil } } return nil, fmt.Errorf("unsupported log format: %q", log) } // logWriter controls the writing into the stream based on the log options. type logWriter struct { stdout io.Writer stderr io.Writer opts   *logOptions remain int64 } // errMaximumWrite is returned when all bytes have been written. var errMaximumWrite = errors.New("maximum write") // errShortWrite is returned when the message is not fully written. var errShortWrite = errors.New("short write") func newLogWriter(stdout io.Writer, stderr io.Writer, opts *logOptions) *logWriter { w := &logWriter{ stdout: stdout, stderr: stderr, opts:   opts, remain: math.MaxInt64, // initialize it as infinity } if opts.bytes >= 0 { w.remain = opts.bytes } return w } </del> <ins> opts := logs.NewLogOptions(apiOpts, time.Now()) </ins> <del> // writeLogs writes logs into stdout, stderr. func (w *logWriter) write(msg *logMessage) error { if msg.timestamp.Before(w.opts.since) { // Skip the line because it's older than since return nil } line := msg.log if w.opts.timestamp { prefix := append([]byte(msg.timestamp.Format(timeFormat)), delimiter[0]) line = append(prefix, line...) } // If the line is longer than the remaining bytes, cut it. if int64(len(line)) > w.remain { line = line[:w.remain] } // Get the proper stream to write to. var stream io.Writer switch msg.stream { case stdoutType: stream = w.stdout case stderrType: stream = w.stderr default: return fmt.Errorf("unexpected stream type %q", msg.stream) } n, err := stream.Write(line) w.remain -= int64(n) if err != nil { return err } // If the line has not been fully written, return errShortWrite if n < len(line) { return errShortWrite } // If there are no more bytes left, return errMaximumWrite if w.remain <= 0 { return errMaximumWrite } return nil </del> <ins> return logs.ReadLogs(path, containerID, opts, m.runtimeService, stdout, stderr) </ins> }


<del> /* Copyright 2016 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kuberuntime import ( "bytes" "testing" "time" "github.com/stretchr/testify/assert" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestLogOptions(t *testing.T) { var ( line         = int64(8) bytes        = int64(64) timestamp    = metav1.Now() sinceseconds = int64(10) ) for c, test := range []struct { apiOpts *v1.PodLogOptions expect  *logOptions }{ { // empty options apiOpts: &v1.PodLogOptions{}, expect:  &logOptions{tail: -1, bytes: -1}, }, { // test tail lines apiOpts: &v1.PodLogOptions{TailLines: &line}, expect:  &logOptions{tail: line, bytes: -1}, }, { // test limit bytes apiOpts: &v1.PodLogOptions{LimitBytes: &bytes}, expect:  &logOptions{tail: -1, bytes: bytes}, }, { // test since timestamp apiOpts: &v1.PodLogOptions{SinceTime: &timestamp}, expect:  &logOptions{tail: -1, bytes: -1, since: timestamp.Time}, }, { // test since seconds apiOpts: &v1.PodLogOptions{SinceSeconds: &sinceseconds}, expect:  &logOptions{tail: -1, bytes: -1, since: timestamp.Add(-10 * time.Second)}, }, } { t.Logf("TestCase #%d: %+v", c, test) opts := newLogOptions(test.apiOpts, timestamp.Time) assert.Equal(t, test.expect, opts) } } func TestParseLog(t *testing.T) { timestamp, err := time.Parse(timeFormat, "2016-10-20T18:39:20.57606443Z") assert.NoError(t, err) msg := &logMessage{} for c, test := range []struct { line string msg  *logMessage err  bool }{ { // Docker log format stdout line: `{"log":"docker stdout test log","stream":"stdout","time":"2016-10-20T18:39:20.57606443Z"}` + "n", msg: &logMessage{ timestamp: timestamp, stream:    stdoutType, log:       []byte("docker stdout test log"), }, }, { // Docker log format stderr line: `{"log":"docker stderr test log","stream":"stderr","time":"2016-10-20T18:39:20.57606443Z"}` + "n", msg: &logMessage{ timestamp: timestamp, stream:    stderrType, log:       []byte("docker stderr test log"), }, }, { // CRI log format stdout line: "2016-10-20T18:39:20.57606443Z stdout cri stdout test logn", msg: &logMessage{ timestamp: timestamp, stream:    stdoutType, log:       []byte("cri stdout test logn"), }, }, { // CRI log format stderr line: "2016-10-20T18:39:20.57606443Z stderr cri stderr test logn", msg: &logMessage{ timestamp: timestamp, stream:    stderrType, log:       []byte("cri stderr test logn"), }, }, { // Unsupported Log format line: "unsupported log format test logn", msg:  &logMessage{}, err:  true, }, } { t.Logf("TestCase #%d: %+v", c, test) parse, err := getParseFunc([]byte(test.line)) if test.err { assert.Error(t, err) continue } assert.NoError(t, err) err = parse([]byte(test.line), msg) assert.NoError(t, err) assert.Equal(t, test.msg, msg) } } func TestWriteLogs(t *testing.T) { timestamp := time.Unix(1234, 4321) log := "abcdefgn" for c, test := range []struct { stream       streamType since        time.Time timestamp    bool expectStdout string expectStderr string }{ { // stderr log stream:       stderrType, expectStderr: log, }, { // stdout log stream:       stdoutType, expectStdout: log, }, { // since is after timestamp stream: stdoutType, since:  timestamp.Add(1 * time.Second), }, { // timestamp enabled stream:       stderrType, timestamp:    true, expectStderr: timestamp.Format(timeFormat) + " " + log, }, } { t.Logf("TestCase #%d: %+v", c, test) msg := &logMessage{ timestamp: timestamp, stream:    test.stream, log:       []byte(log), } stdoutBuf := bytes.NewBuffer(nil) stderrBuf := bytes.NewBuffer(nil) w := newLogWriter(stdoutBuf, stderrBuf, &logOptions{since: test.since, timestamp: test.timestamp, bytes: -1}) err := w.write(msg) assert.NoError(t, err) assert.Equal(t, test.expectStdout, stdoutBuf.String()) assert.Equal(t, test.expectStderr, stderrBuf.String()) } } func TestWriteLogsWithBytesLimit(t *testing.T) { timestamp := time.Unix(1234, 4321) timestampStr := timestamp.Format(timeFormat) log := "abcdefgn" for c, test := range []struct { stdoutLines  int stderrLines  int bytes        int timestamp    bool expectStdout string expectStderr string }{ { // limit bytes less than one line stdoutLines:  3, bytes:        3, expectStdout: "abc", }, { // limit bytes across lines stdoutLines:  3, bytes:        len(log) + 3, expectStdout: "abcdefgnabc", }, { // limit bytes more than all lines stdoutLines:  3, bytes:        3 * len(log), expectStdout: "abcdefgnabcdefgnabcdefgn", }, { // limit bytes for stderr stderrLines:  3, bytes:        len(log) + 3, expectStderr: "abcdefgnabc", }, { // limit bytes for both stdout and stderr, stdout first. stdoutLines:  1, stderrLines:  2, bytes:        len(log) + 3, expectStdout: "abcdefgn", expectStderr: "abc", }, { // limit bytes with timestamp stdoutLines:  3, timestamp:    true, bytes:        len(timestampStr) + 1 + len(log) + 2, expectStdout: timestampStr + " " + log + timestampStr[:2], }, } { t.Logf("TestCase #%d: %+v", c, test) msg := &logMessage{ timestamp: timestamp, log:       []byte(log), } stdoutBuf := bytes.NewBuffer(nil) stderrBuf := bytes.NewBuffer(nil) w := newLogWriter(stdoutBuf, stderrBuf, &logOptions{timestamp: test.timestamp, bytes: int64(test.bytes)}) for i := 0; i < test.stdoutLines; i++ { msg.stream = stdoutType if err := w.write(msg); err != nil { assert.EqualError(t, err, errMaximumWrite.Error()) } } for i := 0; i < test.stderrLines; i++ { msg.stream = stderrType if err := w.write(msg); err != nil { assert.EqualError(t, err, errMaximumWrite.Error()) } } assert.Equal(t, test.expectStdout, stdoutBuf.String()) assert.Equal(t, test.expectStderr, stderrBuf.String()) } } </del>


<ins> load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", srcs = ["logs.go"], importpath = "k8s.io/kubernetes/pkg/kubelet/kuberuntime/logs", visibility = ["//visibility:public"], deps = [ "//pkg/kubelet/apis/cri:go_default_library", "//pkg/kubelet/apis/cri/v1alpha1/runtime:go_default_library", "//pkg/util/tail:go_default_library", "//vendor/github.com/docker/docker/pkg/jsonlog:go_default_library", "//vendor/github.com/fsnotify/fsnotify:go_default_library", "//vendor/github.com/golang/glog:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", ], ) go_test( name = "go_default_test", srcs = ["logs_test.go"], importpath = "k8s.io/kubernetes/pkg/kubelet/kuberuntime/logs", library = ":go_default_library", deps = [ "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", ], ) filegroup( name = "package-srcs", srcs = glob(["**"]), tags = ["automanaged"], visibility = ["//visibility:private"], ) filegroup( name = "all-srcs", srcs = [":package-srcs"], tags = ["automanaged"], visibility = ["//visibility:public"], ) </ins>