// InstanceShutdownByProviderID returns true if the instance is in safe state to detach volumes func (az *Cloud) InstanceShutdownByProviderID(ctx context.Context, providerID string) (bool, error) { <del> return false, cloudprovider.NotImplemented </del> <ins> nodeName, err := az.vmSet.GetNodeNameByProviderID(providerID) if err != nil { return false, err } provisioningState, err := az.vmSet.GetProvisioningStateByNodeName(string(nodeName)) if err != nil { return false, err } return strings.ToLower(provisioningState) == "stopped" || strings.ToLower(provisioningState) == "deallocated", nil </ins> } // getComputeMetadata gets compute information from instance metadata.


return *machine.ID, nil } <ins> func (as *availabilitySet) GetProvisioningStateByNodeName(name string) (provisioningState string, err error) { vm, err := as.getVirtualMachine(types.NodeName(name)) if err != nil { return provisioningState, err } return *vm.ProvisioningState, nil } </ins> // GetNodeNameByProviderID gets the node name by provider ID. func (as *availabilitySet) GetNodeNameByProviderID(providerID string) (types.NodeName, error) { // NodeName is part of providerID for standard instances.


DetachDiskByName(diskName, diskURI string, nodeName types.NodeName) error // GetDataDisks gets a list of data disks attached to the node. GetDataDisks(nodeName types.NodeName) ([]compute.DataDisk, error) <ins> // GetProvisioningStateByNodeName gets the provisioning state by node name. GetProvisioningStateByNodeName(name string) (string, error) </ins> }


return ssName, instanceID, *(cachedVM.(*compute.VirtualMachineScaleSetVM)), nil } <ins> func (ss *scaleSet) GetProvisioningStateByNodeName(name string) (provisioningState string, err error) { _, _, vm, err := ss.getVmssVM(name) if err != nil { return provisioningState, err } return *vm.ProvisioningState, nil } </ins> // getCachedVirtualMachineByInstanceID gets scaleSetVMInfo from cache. // The node must belong to one of scale sets. func (ss *scaleSet) getVmssVMByInstanceID(resourceGroup, scaleSetName, instanceID string) (vm compute.VirtualMachineScaleSetVM, err error) {


type listenPortOpener struct{} // OpenLocalPort holds the given local port open. <del> func (l *listenPortOpener) OpenLocalPort(lp *utilproxy.LocalPort) (utilproxy.Closeable, error) { return openLocalPort(lp) </del> <ins> func (l *listenPortOpener) OpenLocalPort(lp *utilproxy.LocalPort, isIPv6 bool) (utilproxy.Closeable, error) { return openLocalPort(lp, isIPv6) </ins> } // Proxier implements proxy.Provider


klog.V(4).Infof("Port %s was open before and is still needed", lp.String()) replacementPortsMap[lp] = proxier.portsMap[lp] } else if svcInfo.Protocol() != v1.ProtocolSCTP { <del> socket, err := proxier.portMapper.OpenLocalPort(&lp) </del> <ins> socket, err := proxier.portMapper.OpenLocalPort(&lp, isIPv6) </ins> if err != nil { klog.Errorf("can't open %s, skipping this nodePort: %v", lp.String(), err) continue


buf.WriteByte('n') } <del> func openLocalPort(lp *utilproxy.LocalPort) (utilproxy.Closeable, error) { </del> <ins> func openLocalPort(lp *utilproxy.LocalPort, isIPv6 bool) (utilproxy.Closeable, error) { </ins> // For ports on node IPs, open the actual port and hold it, even though we // use iptables to redirect traffic. // This ensures a) that it's safe to use that port and b) that (a) stays


klog.Errorf("Failed to cast serviceInfo %q", svcName.String()) continue } <ins> isIPv6 := utilnet.IsIPv6(svcInfo.ClusterIP()) </ins> protocol := strings.ToLower(string(svcInfo.Protocol())) // Precompute svcNameString; with many services the many calls // to ServicePortName.String() show up in CPU profiles.


klog.V(4).Infof("Port %s was open before and is still needed", lp.String()) replacementPortsMap[lp] = proxier.portsMap[lp] } else { <del> socket, err := proxier.portMapper.OpenLocalPort(&lp) </del> <ins> socket, err := proxier.portMapper.OpenLocalPort(&lp, isIPv6) </ins> if err != nil { msg := fmt.Sprintf("can't open %s, skipping this externalIP: %v", lp.String(), err)


// We do not start listening on SCTP ports, according to our agreement in the // SCTP support KEP } else if svcInfo.Protocol() != v1.ProtocolSCTP { <del> socket, err := proxier.portMapper.OpenLocalPort(&lp) </del> <ins> socket, err := proxier.portMapper.OpenLocalPort(&lp, isIPv6) </ins> if err != nil { klog.Errorf("can't open %s, skipping this nodePort: %v", lp.String(), err) continue } if lp.Protocol == "udp" { <del> isIPv6 := utilnet.IsIPv6(svcInfo.ClusterIP()) </del> conntrack.ClearEntriesForPort(proxier.exec, lp.Port, isIPv6, v1.ProtocolUDP) } replacementPortsMap[lp] = socket


type listenPortOpener struct{} // OpenLocalPort holds the given local port open. <del> func (l *listenPortOpener) OpenLocalPort(lp *utilproxy.LocalPort) (utilproxy.Closeable, error) { return openLocalPort(lp) </del> <ins> func (l *listenPortOpener) OpenLocalPort(lp *utilproxy.LocalPort, isIPv6 bool) (utilproxy.Closeable, error) { return openLocalPort(lp, isIPv6) </ins> } <del> func openLocalPort(lp *utilproxy.LocalPort) (utilproxy.Closeable, error) { </del> <ins> func openLocalPort(lp *utilproxy.LocalPort, isIPv6 bool) (utilproxy.Closeable, error) { </ins> // For ports on node IPs, open the actual port and hold it, even though we // use ipvs to redirect traffic. // This ensures a) that it's safe to use that port and b) that (a) stays


var socket utilproxy.Closeable switch lp.Protocol { case "tcp": <del> listener, err := net.Listen("tcp", net.JoinHostPort(lp.IP, strconv.Itoa(lp.Port))) </del> <ins> network := "tcp4" if isIPv6 { network = "tcp6" } listener, err := net.Listen(network, net.JoinHostPort(lp.IP, strconv.Itoa(lp.Port))) </ins> if err != nil { return nil, err } socket = listener case "udp": <del> addr, err := net.ResolveUDPAddr("udp", net.JoinHostPort(lp.IP, strconv.Itoa(lp.Port))) </del> <ins> network := "udp4" if isIPv6 { network = "udp6" } addr, err := net.ResolveUDPAddr(network, net.JoinHostPort(lp.IP, strconv.Itoa(lp.Port))) </ins> if err != nil { return nil, err } <del> conn, err := net.ListenUDP("udp", addr) </del> <ins> conn, err := net.ListenUDP(network, addr) </ins> if err != nil { return nil, err }


// OpenLocalPort fakes out the listen() and bind() used by syncProxyRules // to lock a local port. <del> func (f *fakePortOpener) OpenLocalPort(lp *utilproxy.LocalPort) (utilproxy.Closeable, error) { </del> <ins> func (f *fakePortOpener) OpenLocalPort(lp *utilproxy.LocalPort, isIPv6 bool) (utilproxy.Closeable, error) { </ins> f.openPorts = append(f.openPorts, lp) return nil, nil }


// PortOpener is an interface around port opening/closing. // Abstracted out for testing. type PortOpener interface { <del> OpenLocalPort(lp *LocalPort) (Closeable, error) </del> <ins> OpenLocalPort(lp *LocalPort, isIPv6 bool) (Closeable, error) </ins> } // RevertPorts is closing ports in replacementPortsMap but not in originalPortsMap. In other words, it only


<del> 2.1 </del> <ins> 2.2 </ins>


Request Headers: {% for i, key in ipairs(keys) do %} <del> {{key}}={{headers[key]}} </del> <ins> {% local val = headers[key] %} {% if type(val) == "table" then %} {% for i = 1,#val do %} {{key}}={{val[i]}} {% end %} {% else %} {{key}}={{val}} {% end %} </ins> {% end %} Request Body:


if err != nil { return nil, err } <del> if existsPip { return &pip, nil } </del> serviceName := getServiceName(service) <del> if shouldPIPExisted { return nil, fmt.Errorf("PublicIP from annotation azure-pip-name=%s for service %s doesn't exist", pipName, serviceName) } pip.Name = to.StringPtr(pipName) pip.Location = to.StringPtr(az.Location) pip.PublicIPAddressPropertiesFormat = &network.PublicIPAddressPropertiesFormat{ PublicIPAllocationMethod: network.Static, </del> <ins> if existsPip { // return if pip exist and dns label is the same if getDomainNameLabel(&pip) == domainNameLabel { return &pip, nil } klog.V(2).Infof("ensurePublicIPExists for service(%s): pip(%s) - updating", serviceName, *pip.Name) if pip.PublicIPAddressPropertiesFormat == nil { pip.PublicIPAddressPropertiesFormat = &network.PublicIPAddressPropertiesFormat{ PublicIPAllocationMethod: network.Static, } } } else { if shouldPIPExisted { return nil, fmt.Errorf("PublicIP from annotation azure-pip-name=%s for service %s doesn't exist", pipName, serviceName) } pip.Name = to.StringPtr(pipName) pip.Location = to.StringPtr(az.Location) pip.PublicIPAddressPropertiesFormat = &network.PublicIPAddressPropertiesFormat{ PublicIPAllocationMethod: network.Static, } pip.Tags = map[string]*string{ serviceTagKey:  &serviceName, clusterNameKey: &clusterName, } if az.useStandardLoadBalancer() { pip.Sku = &network.PublicIPAddressSku{ Name: network.PublicIPAddressSkuNameStandard, } } klog.V(2).Infof("ensurePublicIPExists for service(%s): pip(%s) - creating", serviceName, *pip.Name) </ins> } <del> if len(domainNameLabel) > 0 { </del> <ins> if len(domainNameLabel) == 0 { pip.PublicIPAddressPropertiesFormat.DNSSettings = nil } else { </ins> pip.PublicIPAddressPropertiesFormat.DNSSettings = &network.PublicIPAddressDNSSettings{ DomainNameLabel: &domainNameLabel, } } <del> pip.Tags = map[string]*string{ serviceTagKey:  &serviceName, clusterNameKey: &clusterName, } if az.useStandardLoadBalancer() { pip.Sku = &network.PublicIPAddressSku{ Name: network.PublicIPAddressSkuNameStandard, } } klog.V(2).Infof("ensurePublicIPExists for service(%s): pip(%s) - creating", serviceName, *pip.Name) </del> klog.V(10).Infof("CreateOrUpdatePIP(%s, %q): start", pipResourceGroup, *pip.Name) err = az.CreateOrUpdatePIP(service, pipResourceGroup, pip) if err != nil { <del> klog.V(2).Infof("ensure(%s) abort backoff: pip(%s) - creating", serviceName, *pip.Name) </del> <ins> klog.V(2).Infof("ensure(%s) abort backoff: pip(%s)", serviceName, *pip.Name) </ins> return nil, err } klog.V(10).Infof("CreateOrUpdatePIP(%s, %q): end", pipResourceGroup, *pip.Name)


return &pip, nil } <ins> func getDomainNameLabel(pip *network.PublicIPAddress) string { if pip == nil || pip.PublicIPAddressPropertiesFormat == nil || pip.PublicIPAddressPropertiesFormat.DNSSettings == nil { return "" } return to.String(pip.PublicIPAddressPropertiesFormat.DNSSettings.DomainNameLabel) } </ins> func getIdleTimeout(s *v1.Service) (*int32, error) { const ( min = 4


existingPIPs  []network.PublicIPAddress expectedPIP   *network.PublicIPAddress expectedID    string <ins> expectedDNS   string </ins> expectedError bool }{ {


expectedID: "/subscriptions/subscription/resourceGroups/rg/providers/" + "Microsoft.Network/publicIPAddresses/pip1", }, <ins> { desc: "ensurePublicIPExists shall update existed PIP's dns label", existingPIPs: []network.PublicIPAddress{{ Name: to.StringPtr("pip1"), PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ DNSSettings: &network.PublicIPAddressDNSSettings{ DomainNameLabel: to.StringPtr("previousdns"), }, }, }}, expectedPIP: &network.PublicIPAddress{ Name: to.StringPtr("pip1"), ID: to.StringPtr("/subscriptions/subscription/resourceGroups/rg" + "/providers/Microsoft.Network/publicIPAddresses/pip1"), PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ DNSSettings: &network.PublicIPAddressDNSSettings{ DomainNameLabel: to.StringPtr("newdns"), }, }, }, expectedDNS: "newdns", }, </ins> } for i, test := range testCases {


t.Fatalf("TestCase[%d] meets unexpected error: %v", i, err) } } <del> pip, err := az.ensurePublicIPExists(&service, "pip1", "", "", false) </del> <ins> pip, err := az.ensurePublicIPExists(&service, "pip1", test.expectedDNS, "", false) </ins> if test.expectedID != "" { assert.Equal(t, test.expectedID, to.String(pip.ID), "TestCase[%d]: %s", i, test.desc) } else {


"io" "io/ioutil" "os" <ins> "regexp" </ins> "time" "github.com/Azure/azure-sdk-for-go/services/containerregistry/mgmt/2017-10-01/containerregistry"


maxReadLength      = 10 * 1 << 20 // 10MB ) <del> var containerRegistryUrls = []string{"*.azurecr.io", "*.azurecr.cn", "*.azurecr.de", "*.azurecr.us"} </del> <ins> var ( containerRegistryUrls = []string{"*.azurecr.io", "*.azurecr.cn", "*.azurecr.de", "*.azurecr.us"} acrRE                 = regexp.MustCompile(`.*.azurecr.io|.*.azurecr.cn|.*.azurecr.de|.*.azurecr.us`) ) </ins> // init registers the various means by which credentials may // be resolved on Azure.


} func (a *acrProvider) Provide(image string) credentialprovider.DockerConfig { <ins> klog.V(4).Infof("try to provide secret for image %s", image) </ins> cfg := credentialprovider.DockerConfig{} <del> ctx, cancel := getContextWithCancel() defer cancel() </del> if a.config.UseManagedIdentityExtension { <del> klog.V(4).Infof("listing registries") result, err := a.registryClient.List(ctx) if err != nil { klog.Errorf("Failed to list registries: %v", err) return cfg } for ix := range result { loginServer := getLoginServer(result[ix]) klog.V(2).Infof("loginServer: %s", loginServer) cred, err := getACRDockerEntryFromARMToken(a, loginServer) if err != nil { continue </del> <ins> if loginServer := parseACRLoginServerFromImage(image); loginServer == "" { klog.V(4).Infof("image(%s) is not from ACR, skip MSI authentication", image) } else { if cred, err := getACRDockerEntryFromARMToken(a, loginServer); err == nil { cfg[loginServer] = *cred </ins> } <del> cfg[loginServer] = *cred </del> } } else { // Add our entry for each of the supported container registry URLs


} func getACRDockerEntryFromARMToken(a *acrProvider, loginServer string) (*credentialprovider.DockerConfigEntry, error) { <ins> // Run EnsureFresh to make sure the token is valid and does not expire if err := a.servicePrincipalToken.EnsureFresh(); err != nil { klog.Errorf("Failed to ensure fresh service principal token: %v", err) return nil, err } </ins> armAccessToken := a.servicePrincipalToken.OAuthToken() klog.V(4).Infof("discovering auth redirects for: %s", loginServer)


}, nil } <ins> // parseACRLoginServerFromImage takes image as parameter and returns login server of it. // Parameter `image` is expected in following format: foo.azurecr.io/bar/imageName:version // If the provided image is not an acr image, this function will return an empty string. func parseACRLoginServerFromImage(image string) string { match := acrRE.FindAllString(image, -1) if len(match) == 1 { return match[0] } return "" } </ins> func (a *acrProvider) LazyProvide(image string) *credentialprovider.DockerConfigEntry { return nil }


} } } <ins> func TestParseACRLoginServerFromImage(t *testing.T) { tests := []struct { image    string expected string }{ { image:    "invalidImage", expected: "", }, { image:    "docker.io/library/busybox:latest", expected: "", }, { image:    "foo.azurecr.io/bar/image:version", expected: "foo.azurecr.io", }, { image:    "foo.azurecr.cn/bar/image:version", expected: "foo.azurecr.cn", }, { image:    "foo.azurecr.de/bar/image:version", expected: "foo.azurecr.de", }, { image:    "foo.azurecr.us/bar/image:version", expected: "foo.azurecr.us", }, } for _, test := range tests { if loginServer := parseACRLoginServerFromImage(test.image); loginServer != test.expected { t.Errorf("function parseACRLoginServerFromImage returns "%s" for image %s, expected "%s"", loginServer, test.image, test.expected) } } } </ins>


"github.com/golang/glog" "github.com/xanzy/go-cloudstack/cloudstack" "k8s.io/api/core/v1" <ins> cloudprovider "k8s.io/cloud-provider" </ins> ) type loadBalancer struct {


// GetLoadBalancerName retrieves the name of the LoadBalancer. func (cs *CSCloud) GetLoadBalancerName(ctx context.Context, clusterName string, service *v1.Service) string { <del> return cs.GetLoadBalancerName(ctx, clusterName, service) </del> <ins> return cloudprovider.DefaultLoadBalancerName(service) </ins> } // getLoadBalancer retrieves the IP address and ID and all the existing rules it can find.


"//pkg/api/legacyscheme:go_default_library", "//pkg/apis/core:go_default_library", "//pkg/capabilities:go_default_library", <del> "//pkg/client/chaosclient:go_default_library", </del> "//pkg/cloudprovider:go_default_library", "//pkg/cloudprovider/providers:go_default_library", "//pkg/credentialprovider:go_default_library",


"k8s.io/kubernetes/pkg/api/legacyscheme" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/capabilities" <del> "k8s.io/kubernetes/pkg/client/chaosclient" </del> "k8s.io/kubernetes/pkg/cloudprovider" "k8s.io/kubernetes/pkg/credentialprovider" "k8s.io/kubernetes/pkg/features"


clientConfig.QPS = float32(s.KubeAPIQPS) clientConfig.Burst = int(s.KubeAPIBurst) <del> addChaosToClientConfig(s, clientConfig) </del> return clientConfig, nil } <del> // addChaosToClientConfig injects random errors into client connections if configured. func addChaosToClientConfig(s *options.KubeletServer, config *restclient.Config) { if s.ChaosChance != 0.0 { config.WrapTransport = func(rt http.RoundTripper) http.RoundTripper { seed := chaosclient.NewSeed(1) // TODO: introduce a standard chaos package with more tunables - this is just a proof of concept // TODO: introduce random latency and stalls return chaosclient.NewChaosRoundTripper(rt, chaosclient.LogChaos, seed.P(s.ChaosChance, chaosclient.ErrSimulatedConnectionResetByPeer)) } } } </del> // RunKubelet is responsible for setting up and running a kubelet.  It is used in three different applications: //   1 Integration tests //   2 Kubelet binary


"//pkg/auth/authorizer/abac:all-srcs", "//pkg/auth/nodeidentifier:all-srcs", "//pkg/capabilities:all-srcs", <del> "//pkg/client/chaosclient:all-srcs", </del> "//pkg/client/clientset_generated/internalclientset:all-srcs", "//pkg/client/conditions:all-srcs", "//pkg/client/informers/informers_generated/internalversion:all-srcs",


<del> package(default_visibility = ["//visibility:public"]) load( "@io_bazel_rules_go//go:def.bzl", "go_library", "go_test", ) go_library( name = "go_default_library", srcs = ["chaosclient.go"], importpath = "k8s.io/kubernetes/pkg/client/chaosclient", deps = ["//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library"], ) go_test( name = "go_default_test", srcs = ["chaosclient_test.go"], embed = [":go_default_library"], ) filegroup( name = "package-srcs", srcs = glob(["**"]), tags = ["automanaged"], visibility = ["//visibility:private"], ) filegroup( name = "all-srcs", srcs = [":package-srcs"], tags = ["automanaged"], ) </del>


<del> reviewers: - smarterclayton - liggitt - davidopp - eparis - resouer </del>


<del> /* Copyright 2015 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ // Package chaosclient makes it easy to simulate network latency, misbehaving // servers, and random errors from servers. It is intended to stress test components // under failure conditions and expose weaknesses in the error handling logic // of the codebase. package chaosclient import ( "errors" "fmt" "log" "math/rand" "net/http" "reflect" "runtime" "k8s.io/apimachinery/pkg/util/net" ) // chaosrt provides the ability to perform simulations of HTTP client failures // under the Golang http.Transport interface. type chaosrt struct { rt     http.RoundTripper notify ChaosNotifier c      []Chaos } // Chaos intercepts requests to a remote HTTP endpoint and can inject arbitrary // failures. type Chaos interface { // Intercept should return true if the normal flow should be skipped, and the // return response and error used instead. Modifications to the request will // be ignored, but may be used to make decisions about types of failures. Intercept(req *http.Request) (bool, *http.Response, error) } // ChaosNotifier notifies another component that the ChaosRoundTripper has simulated // a failure. type ChaosNotifier interface { // OnChaos is invoked when a chaotic outcome was triggered. fn is the // source of Chaos and req was the outgoing request OnChaos(req *http.Request, c Chaos) } // ChaosFunc takes an http.Request and decides whether to alter the response. It // returns true if it wishes to mutate the response, with a http.Response or // error. type ChaosFunc func(req *http.Request) (bool, *http.Response, error) // Intercept calls the nested method `Intercept` func (fn ChaosFunc) Intercept(req *http.Request) (bool, *http.Response, error) { return fn.Intercept(req) } func (fn ChaosFunc) String() string { return runtime.FuncForPC(reflect.ValueOf(fn).Pointer()).Name() } // NewChaosRoundTripper creates an http.RoundTripper that will intercept requests // based on the provided Chaos functions. The notifier is invoked when a Chaos // Intercept is fired. func NewChaosRoundTripper(rt http.RoundTripper, notify ChaosNotifier, c ...Chaos) http.RoundTripper { return &chaosrt{rt, notify, c} } // RoundTrip gives each ChaosFunc an opportunity to intercept the request. The first // interceptor wins. func (rt *chaosrt) RoundTrip(req *http.Request) (*http.Response, error) { for _, c := range rt.c { if intercept, resp, err := c.Intercept(req); intercept { rt.notify.OnChaos(req, c) return resp, err } } return rt.rt.RoundTrip(req) } var _ = net.RoundTripperWrapper(&chaosrt{}) func (rt *chaosrt) WrappedRoundTripper() http.RoundTripper { return rt.rt } // Seed represents a consistent stream of chaos. type Seed struct { *rand.Rand } // NewSeed creates an object that assists in generating random chaotic events // based on a deterministic seed. func NewSeed(seed int64) Seed { return Seed{rand.New(rand.NewSource(seed))} } type pIntercept struct { Chaos s Seed p float64 } // P returns a ChaosFunc that fires with a probability of p (p between 0.0 // and 1.0 with 0.0 meaning never and 1.0 meaning always). func (s Seed) P(p float64, c Chaos) Chaos { return pIntercept{c, s, p} } // Intercept intercepts requests with the provided probability p. func (c pIntercept) Intercept(req *http.Request) (bool, *http.Response, error) { if c.s.Float64() < c.p { return c.Chaos.Intercept(req) } return false, nil, nil } func (c pIntercept) String() string { return fmt.Sprintf("P{%f %s}", c.p, c.Chaos) } // ErrSimulatedConnectionResetByPeer emulates the golang net error when a connection // is reset by a peer. // TODO: make this more accurate // TODO: add other error types // TODO: add a helper for returning multiple errors randomly. var ErrSimulatedConnectionResetByPeer = Error{errors.New("connection reset by peer")} // Error returns the nested error when C() is invoked. type Error struct { error } // Intercept returns the nested error func (e Error) Intercept(_ *http.Request) (bool, *http.Response, error) { return true, nil, e.error } // LogChaos is the default ChaosNotifier and writes a message to the Golang log. var LogChaos = ChaosNotifier(logChaos{}) type logChaos struct{} func (logChaos) OnChaos(req *http.Request, c Chaos) { log.Printf("Triggered chaotic behavior for %s %s: %v", req.Method, req.URL.String(), c) } </del>


<del> /* Copyright 2015 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package chaosclient import ( "net/http" "net/http/httptest" "net/url" "testing" ) type TestLogChaos struct { *testing.T } func (t TestLogChaos) OnChaos(req *http.Request, c Chaos) { t.Logf("CHAOS: chaotic behavior for %s %s: %v", req.Method, req.URL.String(), c) } func unwrapURLError(err error) error { if urlErr, ok := err.(*url.Error); ok && urlErr != nil { return urlErr.Err } return err } func TestChaos(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) defer server.Close() client := http.Client{ Transport: NewChaosRoundTripper(http.DefaultTransport, TestLogChaos{t}, ErrSimulatedConnectionResetByPeer), } resp, err := client.Get(server.URL) if unwrapURLError(err) != ErrSimulatedConnectionResetByPeer.error { t.Fatalf("expected reset by peer: %v", err) } if resp != nil { t.Fatalf("expected no response object: %#v", resp) } } func TestPartialChaos(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) defer server.Close() seed := NewSeed(1) client := http.Client{ Transport: NewChaosRoundTripper( http.DefaultTransport, TestLogChaos{t}, seed.P(0.5, ErrSimulatedConnectionResetByPeer), ), } success, fail := 0, 0 for { _, err := client.Get(server.URL) if err != nil { fail++ } else { success++ } if success > 1 && fail > 1 { break } } } </del>


k8s.io/kubernetes/pkg/apis/rbac/validation,erictune,0, k8s.io/kubernetes/pkg/apis/storage/validation,caesarxuchao,1, k8s.io/kubernetes/pkg/auth/authorizer/abac,liggitt,0, <del> k8s.io/kubernetes/pkg/client/chaosclient,deads2k,1, </del> k8s.io/kubernetes/pkg/client/legacylisters,jsafrane,1, k8s.io/kubernetes/pkg/client/listers/batch/internalversion,mqliang,0, k8s.io/kubernetes/pkg/client/listers/extensions/internalversion,eparis,1,


// must unmounted prior to detach. // If a volume with the name volumeName does not exist in the list of // attached volumes, an error is returned. <del> SetVolumeGloballyMounted(volumeName v1.UniqueVolumeName, globallyMounted bool) error </del> <ins> SetVolumeGloballyMounted(volumeName v1.UniqueVolumeName, globallyMounted bool, devicePath string) error </ins> // DeletePodFromVolume removes the given pod from the given volume in the // cache indicating the volume has been successfully unmounted from the pod.


} func (asw *actualStateOfWorld) MarkDeviceAsMounted( <del> volumeName v1.UniqueVolumeName) error { return asw.SetVolumeGloballyMounted(volumeName, true /* globallyMounted */) </del> <ins> volumeName v1.UniqueVolumeName, devicePath string) error { return asw.SetVolumeGloballyMounted(volumeName, true /* globallyMounted */, devicePath) </ins> } func (asw *actualStateOfWorld) MarkDeviceAsUnmounted( volumeName v1.UniqueVolumeName) error { <del> return asw.SetVolumeGloballyMounted(volumeName, false /* globallyMounted */) </del> <ins> return asw.SetVolumeGloballyMounted(volumeName, false /* globallyMounted */, "" /*devicePath*/) </ins> } // addVolume adds the given volume to the cache indicating the specified


} func (asw *actualStateOfWorld) SetVolumeGloballyMounted( <del> volumeName v1.UniqueVolumeName, globallyMounted bool) error { </del> <ins> volumeName v1.UniqueVolumeName, globallyMounted bool, devicePath string) error { </ins> asw.Lock() defer asw.Unlock()


} volumeObj.globallyMounted = globallyMounted <ins> volumeObj.devicePath = devicePath </ins> asw.attachedVolumes[volumeName] = volumeObj return nil }


} // Act <del> err = asw.MarkDeviceAsMounted(generatedVolumeName) </del> <ins> err = asw.MarkDeviceAsMounted(generatedVolumeName, devicePath) </ins> // Assert if err != nil {


continue } if volume.pluginIsAttachable { <del> err = rc.actualStateOfWorld.MarkDeviceAsMounted(volume.volumeName) </del> <ins> err = rc.actualStateOfWorld.MarkDeviceAsMounted(volume.volumeName, volume.devicePath) </ins> if err != nil { glog.Errorf("Could not mark device is mounted to actual state of world: %v", err) continue


MarkVolumeAsUnmounted(podName volumetypes.UniquePodName, volumeName v1.UniqueVolumeName) error // Marks the specified volume as having been globally mounted. <del> MarkDeviceAsMounted(volumeName v1.UniqueVolumeName) error </del> <ins> MarkDeviceAsMounted(volumeName v1.UniqueVolumeName, devicePath string) error </ins> // Marks the specified volume as having its global mount unmounted. MarkDeviceAsUnmounted(volumeName v1.UniqueVolumeName) error


// Update actual state of world to reflect volume is globally mounted markDeviceMountedErr := actualStateOfWorld.MarkDeviceAsMounted( <del> volumeToMount.VolumeName) </del> <ins> volumeToMount.VolumeName, devicePath) </ins> if markDeviceMountedErr != nil { // On failure, return error. Caller will log and retry. return volumeToMount.GenerateErrorDetailed("MountVolume.MarkDeviceAsMounted failed", markDeviceMountedErr)


// Update actual state of world to reflect volume is globally mounted markDeviceMappedErr := actualStateOfWorld.MarkDeviceAsMounted( <del> volumeToMount.VolumeName) </del> <ins> volumeToMount.VolumeName, devicePath) </ins> if markDeviceMappedErr != nil { // On failure, return error. Caller will log and retry. return volumeToMount.GenerateErrorDetailed("MapVolume.MarkDeviceAsMounted failed", markDeviceMappedErr)


* kube-apiserver: the `Priority` admission plugin is now enabled by default when using `--enable-admission-plugins`. If using `--admission-control` to fully specify the set of admission plugins, the `Priority` admission plugin should be added if using the `PodPriority` feature, which is enabled by default in 1.11. ([#65739](https://github.com/kubernetes/kubernetes/pull/65739), [@liggitt](https://github.com/liggitt)) * The `system-node-critical` and `system-cluster-critical` priority classes are now limited to the `kube-system` namespace by the `PodPriority` admission plugin. ([#65593](https://github.com/kubernetes/kubernetes/pull/65593), [@bsalamat](https://github.com/bsalamat)) * kubernetes-worker juju charm: Added support for setting the --enable-ssl-chain-completion option on the ingress proxy.  "action required": if your installation relies on supplying incomplete certificate chains and using OCSP to fill them in, you must set "ingress-ssl-chain-completion" to "true" in your juju configuration. ([#63845](https://github.com/kubernetes/kubernetes/pull/63845), [@paulgear](https://github.com/paulgear)) <ins> * In anticipation of CSI 1.0 in the next release, Kubernetes 1.12 calls the CSI `NodeGetInfo` RPC instead of `NodeGetId` RPC. Ensure your CSI Driver implements `NodeGetInfo(...)` before upgrading to 1.12. [@saad-ali](https://github.com/kubernetes/kubernetes/issues/68688) * Kubernetes 1.12 also enables [Kubelet device plugin registration](https://github.com/kubernetes/features/issues/595) feature by default. Before upgrading to 1.12, ensure the `driver-registrar` CSI sidecar container for your CSI driver is configured to handle plugin registration (set the `--kubelet-registration-path` parameter on `driver-registrar` to expose a new unix domain socket to handle Kubelet Plugin Registration). </ins> ### Other notable changes


set -e source "${KUBE_ROOT}/hack/lib/init.sh" <ins> kube::util::ensure-gnu-sed </ins> function usage { echo "This script starts a local kube cluster. "


#   foo: true #   bar: false for gate in $(echo ${FEATURE_GATES} | tr ',' ' '); do <del> echo $gate | sed -e 's/(.*)=(.*)/  1: 2/' </del> <ins> echo $gate | ${SED} -e 's/(.*)=(.*)/  1: 2/' </ins> done fi >>/tmp/kube-proxy.yaml


function start_kubedns { if [[ "${ENABLE_CLUSTER_DNS}" = true ]]; then cp "${KUBE_ROOT}/cluster/addons/dns/kube-dns/kube-dns.yaml.in" kube-dns.yaml <del> sed -i -e "s/{{ pillar['dns_domain'] }}/${DNS_DOMAIN}/g" kube-dns.yaml sed -i -e "s/{{ pillar['dns_server'] }}/${DNS_SERVER_IP}/g" kube-dns.yaml </del> <ins> ${SED} -i -e "s/{{ pillar['dns_domain'] }}/${DNS_DOMAIN}/g" kube-dns.yaml ${SED} -i -e "s/{{ pillar['dns_server'] }}/${DNS_SERVER_IP}/g" kube-dns.yaml </ins> # TODO update to dns role once we have one. # use kubectl to create kubedns addon ${KUBECTL} --kubeconfig="${CERT_DIR}/admin.kubeconfig" --namespace=kube-system create -f kube-dns.yaml


} if len(instances) == 0 { glog.Warningf("the instance %s does not exist anymore", providerID) <del> return true, nil </del> <ins> // returns false, because otherwise node is not deleted from cluster // false means that it will continue to check InstanceExistsByProviderID return false, nil </ins> } if len(instances) > 1 { return false, fmt.Errorf("multiple instances found for instance: %s", instanceID)


if instance.State != nil { state := aws.StringValue(instance.State.Name) // valid state for detaching volumes <del> if state == ec2.InstanceStateNameStopped || state == ec2.InstanceStateNameTerminated { </del> <ins> if state == ec2.InstanceStateNameStopped { </ins> return true, nil } }


// hoping external CCM or admin can set it. Returning // default values from here will mean, no one can // override them. <del> glog.Errorf("failed to get azure cloud in GetVolumeLimits, plugin.host: %s", plugin.host.GetHostName()) return volumeLimits, nil </del> <ins> return nil, fmt.Errorf("failed to get azure cloud in GetVolumeLimits, plugin.host: %s", plugin.host.GetHostName()) </ins> } instances, ok := az.Instances()


LogSSHResult(result) if result.Code != 0 || err != nil { <del> return nil, fmt.Errorf("failed running %q: %v (exit code %d)", cmd, err, result.Code) </del> <ins> return nil, fmt.Errorf("failed running %q: %v (exit code %d, stderr %v)", cmd, err, result.Code, result.Stderr) </ins> } return &result, nil


zero := int64(0) <ins> // Some distributions (Ubuntu 16.04 etc.) don't support the proc file. _, err := framework.IssueSSHCommandWithResult( "ls /proc/net/nf_conntrack", framework.TestContext.Provider, clientNodeInfo.node) if err != nil && strings.Contains(err.Error(), "No such file or directory") { framework.Skipf("The node %s does not support /proc/net/nf_conntrack", clientNodeInfo.name) } framework.ExpectNoError(err) </ins> clientPodSpec := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:      "e2e-net-client",


// SetCRDCondition sets the status condition. It either overwrites the existing one or creates a new one. func SetCRDCondition(crd *CustomResourceDefinition, newCondition CustomResourceDefinitionCondition) { <ins> newCondition.LastTransitionTime = metav1.NewTime(time.Now()) </ins> existingCondition := FindCRDCondition(crd, newCondition.Type) if existingCondition == nil { <del> newCondition.LastTransitionTime = metav1.NewTime(time.Now()) </del> crd.Status.Conditions = append(crd.Status.Conditions, newCondition) return } <del> if existingCondition.Status != newCondition.Status { existingCondition.Status = newCondition.Status </del> <ins> if existingCondition.Status != newCondition.Status || existingCondition.LastTransitionTime.IsZero() { </ins> existingCondition.LastTransitionTime = newCondition.LastTransitionTime } <ins> existingCondition.Status = newCondition.Status </ins> existingCondition.Reason = newCondition.Reason existingCondition.Message = newCondition.Message }


}, }, }, <ins> { name: "set new condition which doesn't have lastTransitionTime set", crdCondition: []CustomResourceDefinitionCondition{ { Type:               Established, Status:             ConditionTrue, Reason:             "Accepted", Message:            "the initial names have been accepted", LastTransitionTime: metav1.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC), }, }, newCondition: CustomResourceDefinitionCondition{ Type:    Established, Status:  ConditionFalse, Reason:  "NotAccepted", Message: "Not accepted", }, expectedcrdCondition: []CustomResourceDefinitionCondition{ { Type:               Established, Status:             ConditionFalse, Reason:             "NotAccepted", Message:            "Not accepted", LastTransitionTime: metav1.Date(2018, 1, 2, 0, 0, 0, 0, time.UTC), }, }, }, { name: "append new condition which doesn't have lastTransitionTime set", crdCondition: []CustomResourceDefinitionCondition{ { Type:               Established, Status:             ConditionTrue, Reason:             "Accepted", Message:            "the initial names have been accepted", LastTransitionTime: metav1.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC), }, }, newCondition: CustomResourceDefinitionCondition{ Type:    Terminating, Status:  ConditionFalse, Reason:  "NeverEstablished", Message: "resource was never established", }, expectedcrdCondition: []CustomResourceDefinitionCondition{ { Type:               Established, Status:             ConditionTrue, Reason:             "Accepted", Message:            "the initial names have been accepted", LastTransitionTime: metav1.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC), }, { Type:               Terminating, Status:             ConditionFalse, Reason:             "NeverEstablished", Message:            "resource was never established", LastTransitionTime: metav1.Date(2018, 2, 1, 0, 0, 0, 0, time.UTC), }, }, }, </ins> } for _, tc := range tests { crd := generateCRDwithCondition(tc.crdCondition)


if !IsCRDConditionEquivalent(&tc.expectedcrdCondition[i], &crd.Status.Conditions[i]) { t.Errorf("%v expected %v, got %v", tc.name, tc.expectedcrdCondition, crd.Status.Conditions) } <ins> if crd.Status.Conditions[i].LastTransitionTime.IsZero() { t.Errorf("%q lastTransitionTime should not be null: %v", tc.name, i, crd.Status.Conditions) } </ins> } } }


le.maybeReportTransition() desc := le.config.Lock.Describe() if err == nil { <del> glog.V(4).Infof("successfully renewed lease %v", desc) </del> <ins> glog.V(5).Infof("successfully renewed lease %v", desc) </ins> return } le.config.Lock.RecordEvent("stopped leading")


providers[name] = factory } <ins> // GetProviders returns the names of all currently registered providers. func GetProviders() []string { mutex.Lock() defer mutex.Unlock() var providerNames []string for name := range providers { providerNames = append(providerNames, name) } return providerNames } </ins> func init() { // "local" or "skeleton" can always be used. RegisterProvider("local", func() (ProviderInterface, error) {


RegisterProvider("skeleton", func() (ProviderInterface, error) { return NullProvider{}, nil }) <del> // The empty string also works, but triggers a warning. RegisterProvider("", func() (ProviderInterface, error) { Logf("The --provider flag is not set.  Treating as a conformance test.  Some tests may not be run.") return NullProvider{}, nil }) </del> <ins> // The empty string used to be accepted in the past, but is not // a valid value anymore. </ins> } // SetupProviderConfig validates the chosen provider and creates


"fmt" "io/ioutil" "os" <ins> "sort" "strings" </ins> "time" "github.com/onsi/ginkgo/config"


flag.StringVar(&TestContext.KubeVolumeDir, "volume-dir", "/var/lib/kubelet", "Path to the directory containing the kubelet volumes.") flag.StringVar(&TestContext.CertDir, "cert-dir", "", "Path to the directory containing the certs. Default is empty, which doesn't use certs.") flag.StringVar(&TestContext.RepoRoot, "repo-root", "../../", "Root directory of kubernetes repository, for finding test files.") <del> flag.StringVar(&TestContext.Provider, "provider", "", "The name of the Kubernetes provider (gce, gke, local, etc.)") </del> <ins> flag.StringVar(&TestContext.Provider, "provider", "", "The name of the Kubernetes provider (gce, gke, local, skeleton (the fallback if not set), etc.)") </ins> flag.StringVar(&TestContext.Tooling, "tooling", "", "The tooling in use (kops, gke, etc.)") flag.StringVar(&TestContext.KubectlPath, "kubectl-path", "kubectl", "The kubectl binary to use. For development, you might use 'cluster/kubectl.sh' here.") flag.StringVar(&TestContext.OutputDir, "e2e-output-dir", "/tmp", "Output directory for interesting/useful test data, like performance data, benchmarks, and other metrics.")


} // Make sure that all test runs have a valid TestContext.CloudConfig.Provider. <ins> // TODO: whether and how long this code is needed is getting discussed // in https://github.com/kubernetes/kubernetes/issues/70194. if TestContext.Provider == "" { // Some users of the e2e.test binary pass --provider=. // We need to support that, changing it would break those usages. Logf("The --provider flag is not set. Continuing as if --provider=skeleton had been used.") TestContext.Provider = "skeleton" } </ins> var err error TestContext.CloudConfig.Provider, err = SetupProviderConfig(TestContext.Provider) <del> if err == nil { return } if !os.IsNotExist(errors.Cause(err)) { Failf("Failed to setup provider config: %v", err) } // We allow unknown provider parameters for historic reasons. At least log a // warning to catch typos. // TODO (https://github.com/kubernetes/kubernetes/issues/70200): // - remove the fallback for unknown providers // - proper error message instead of Failf (which panics) klog.Warningf("Unknown provider %q, proceeding as for --provider=skeleton.", TestContext.Provider) TestContext.CloudConfig.Provider, err = SetupProviderConfig("skeleton") </del> if err != nil { <del> Failf("Failed to setup fallback skeleton provider config: %v", err) </del> <ins> if os.IsNotExist(errors.Cause(err)) { // Provide a more helpful error message when the provider is unknown. var providers []string for _, name := range GetProviders() { // The empty string is accepted, but looks odd in the output below unless we quote it. if name == "" { name = `""` } providers = append(providers, name) } sort.Strings(providers) klog.Errorf("Unknown provider %q. The following providers are known: %v", TestContext.Provider, strings.Join(providers, " ")) } else { klog.Errorf("Failed to setup provider config for %q: %v", TestContext.Provider, err) } os.Exit(1) </ins> } }


fi } <ins> create_csi_crd() { echo "create_csi_crd $1" YAML_FILE=${KUBE_ROOT}/cluster/addons/storage-crds/$1.yaml if [ -e $YAML_FILE ]; then echo "Create $1 crd" ${KUBECTL} --kubeconfig="${CERT_DIR}/admin.kubeconfig" create -f $YAML_FILE else echo "No $1 available." fi } </ins> function print_success { if [[ "${START_MODE}" != "kubeletonly" ]]; then if [[ "${ENABLE_DAEMON}" = false ]]; then


create_storage_class fi <ins> if [[ "${FEATURE_GATES:-}" == "AllAlpha=true" || "${FEATURE_GATES:-}" =~ "CSIDriverRegistry=true" ]]; then create_csi_crd "csidriver" fi if [[ "${FEATURE_GATES:-}" == "AllAlpha=true" || "${FEATURE_GATES:-}" =~ "CSINodeInfo=true" ]]; then create_csi_crd "csinodeinfo" fi </ins> print_success if [[ "${ENABLE_DAEMON}" = false ]]; then


if err != nil { return false, fmt.Errorf("got error while getting pod events: %s", err) } <del> if len(events.Items) == 0 { return false, nil // no events have occurred yet </del> <ins> for _, event := range events.Items { if strings.Contains(event.Message, msg) { return true, nil } </ins> } <del> return strings.Contains(events.Items[0].Message, msg), nil </del> <ins> return false, nil </ins> } }


defer func() { framework.DeletePodWithWait(f, f.ClientSet, pod) }() <del> err = framework.WaitForPodRunningInNamespace(f.ClientSet, pod) Expect(err).To(HaveOccurred(), "while waiting for pod to be running") </del> By("Checking for subpath error event") selector := fields.Set{


"//staging/src/k8s.io/apiserver/pkg/apis/example/v1:go_default_library", "//staging/src/k8s.io/apiserver/pkg/endpoints/request:go_default_library", "//staging/src/k8s.io/apiserver/pkg/registry/rest:go_default_library", <ins> "//staging/src/k8s.io/client-go/kubernetes/scheme:go_default_library", </ins> "//vendor/github.com/evanphx/json-patch:go_default_library", "//vendor/github.com/google/gofuzz:go_default_library", "//vendor/k8s.io/utils/trace:go_default_library",


} } <del> // transformDecodeError adds additional information when a decode fails. </del> <ins> // transformDecodeError adds additional information into a bad-request api error when a decode fails. </ins> func transformDecodeError(typer runtime.ObjectTyper, baseErr error, into runtime.Object, gvk *schema.GroupVersionKind, body []byte) error { objGVKs, _, err := typer.ObjectKinds(into) if err != nil { <del> return err </del> <ins> return errors.NewBadRequest(err.Error()) </ins> } objGVK := objGVKs[0] if gvk != nil && len(gvk.Kind) > 0 {


apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" <ins> testapigroupv1 "k8s.io/apimachinery/pkg/apis/testapigroup/v1" </ins> "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer"


examplev1 "k8s.io/apiserver/pkg/apis/example/v1" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/registry/rest" <ins> clientgoscheme "k8s.io/client-go/kubernetes/scheme" </ins> utiltrace "k8s.io/utils/trace" )


func (f mutateObjectUpdateFunc) Admit(a admission.Attributes, o admission.ObjectInterfaces) (err error) { return f(a.GetObject(), a.GetOldObject()) } <ins> func TestTransformDecodeErrorEnsuresBadRequestError(t *testing.T) { testCases := []struct { name             string typer            runtime.ObjectTyper decodedGVK       *schema.GroupVersionKind decodeIntoObject runtime.Object baseErr          error expectedErr      error }{ { name:  "decoding normal objects fails and returns a bad-request error", typer: clientgoscheme.Scheme, decodedGVK: &schema.GroupVersionKind{ Group:   testapigroupv1.GroupName, Version: "v1", Kind:    "Carp", }, decodeIntoObject: &testapigroupv1.Carp{}, // which client-go's scheme doesn't recognize baseErr:          fmt.Errorf("plain error"), }, { name:             "decoding objects with unknown GVK fails and returns a bad-request error", typer:            alwaysErrorTyper{}, decodedGVK:       nil, decodeIntoObject: &testapigroupv1.Carp{}, // which client-go's scheme doesn't recognize baseErr:          nil, }, } for _, testCase := range testCases { err := transformDecodeError(testCase.typer, testCase.baseErr, testCase.decodeIntoObject, testCase.decodedGVK, []byte(``)) if apiStatus, ok := err.(apierrors.APIStatus); !ok || apiStatus.Status().Code != http.StatusBadRequest { t.Errorf("expected bad request error but got: %v", err) } } } var _ runtime.ObjectTyper = alwaysErrorTyper{} type alwaysErrorTyper struct{} func (alwaysErrorTyper) ObjectKinds(runtime.Object) ([]schema.GroupVersionKind, bool, error) { return nil, false, fmt.Errorf("always error") } func (alwaysErrorTyper) Recognizes(gvk schema.GroupVersionKind) bool { return false } </ins>


"fmt" "io" "net" <ins> "strings" </ins> "time" csipbv1 "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc" api "k8s.io/api/core/v1" utilversion "k8s.io/apimachinery/pkg/util/version" <ins> "k8s.io/apimachinery/pkg/util/wait" </ins> utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/klog" "k8s.io/kubernetes/pkg/features"


err error, ) <ins> const ( initialDuration = 1 * time.Second factor          = 2.0 steps           = 5 ) </ins> // newV1NodeClient creates a new NodeClient with the internally used gRPC // connection set up. It also returns a closer which must to be called to close // the gRPC connection when the NodeClient is not used anymore.


accessibleTopology map[string]string, err error) { klog.V(4).Info(log("calling NodeGetInfo rpc")) <del> if c.nodeV1ClientCreator != nil { return c.nodeGetInfoV1(ctx) } else if c.nodeV0ClientCreator != nil { return c.nodeGetInfoV0(ctx) } err = fmt.Errorf("failed to call NodeGetInfo. Both nodeV1ClientCreator and nodeV0ClientCreator are nil") </del> <ins> // TODO retries should happen at a lower layer (issue #73371) backoff := wait.Backoff{Duration: initialDuration, Factor: factor, Steps: steps} err = wait.ExponentialBackoff(backoff, func() (bool, error) { var getNodeInfoError error if c.nodeV1ClientCreator != nil { nodeID, maxVolumePerNode, accessibleTopology, getNodeInfoError = c.nodeGetInfoV1(ctx) } else if c.nodeV0ClientCreator != nil { nodeID, maxVolumePerNode, accessibleTopology, getNodeInfoError = c.nodeGetInfoV0(ctx) } if nodeID != "" { return true, nil } // kubelet plugin registration service not implemented is a terminal error, no need to retry if strings.Contains(getNodeInfoError.Error(), "no handler registered for plugin type") { return false, getNodeInfoError } // Continue with exponential backoff return false, nil }) </ins> return nodeID, maxVolumePerNode, accessibleTopology, err }


return err } <del> // TODO (verult) retry with exponential backoff, possibly added in csi client library. </del> ctx, cancel := context.WithTimeout(context.Background(), csiTimeout) defer cancel()


if err != nil { klog.Error(log("registrationHandler.RegisterPlugin failed at CSI.NodeGetInfo: %v", err)) if unregErr := unregisterDriver(pluginName); unregErr != nil { <del> klog.Error(log("registrationHandler.RegisterPlugin failed to unregister plugin due to previous: %v", unregErr)) </del> <ins> klog.Error(log("registrationHandler.RegisterPlugin failed to unregister plugin due to previous error: %v", unregErr)) </ins> return unregErr } return err


return fmt.Errorf("PersistentVolumeClaims %v not all in phase %s within %v", pvcNames, phase, timeout) } <ins> // findAvailableNamespaceName random namespace name starting with baseName. func findAvailableNamespaceName(baseName string, c clientset.Interface) (string, error) { var name string err := wait.PollImmediate(Poll, 30*time.Second, func() (bool, error) { name = fmt.Sprintf("%v-%v", baseName, randomSuffix()) _, err := c.CoreV1().Namespaces().Get(name, metav1.GetOptions{}) if err == nil { // Already taken return false, nil } if apierrs.IsNotFound(err) { return true, nil } Logf("Unexpected error while getting namespace: %v", err) return false, nil }) return name, err } </ins> // CreateTestingNS should be used by every test, note that we append a common prefix to the provided test name. // Please see NewFramework instead of using this directly. func CreateTestingNS(baseName string, c clientset.Interface, labels map[string]string) (*v1.Namespace, error) {


} labels["e2e-run"] = string(RunId) <ins> // We don't use ObjectMeta.GenerateName feature, as in case of API call // failure we don't know whether the namespace was created and what is its // name. name, err := findAvailableNamespaceName(baseName, c) if err != nil { return nil, err } </ins> namespaceObj := &v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ <del> GenerateName: fmt.Sprintf("e2e-tests-%v-", baseName), Namespace:    "", Labels:       labels, </del> <ins> Name:      name, Namespace: "", Labels:    labels, </ins> }, Status: v1.NamespaceStatus{}, }


type azureToken struct { token       adal.Token <ins> environment string </ins> clientID    string tenantID    string apiserverID string


if refreshToken == "" { return nil, fmt.Errorf("no refresh token in cfg: %s", cfgRefreshToken) } <ins> environment := ts.cfg[cfgEnvironment] if environment == "" { return nil, fmt.Errorf("no environment in cfg: %s", cfgEnvironment) } </ins> clientID := ts.cfg[cfgClientID] if clientID == "" { return nil, fmt.Errorf("no client ID in cfg: %s", cfgClientID)


Resource:     fmt.Sprintf("spn:%s", apiserverID), Type:         tokenType, }, <ins> environment: environment, </ins> clientID:    clientID, tenantID:    tenantID, apiserverID: apiserverID,


newCfg := make(map[string]string) newCfg[cfgAccessToken] = token.token.AccessToken newCfg[cfgRefreshToken] = token.token.RefreshToken <ins> newCfg[cfgEnvironment] = token.environment </ins> newCfg[cfgClientID] = token.clientID newCfg[cfgTenantID] = token.tenantID newCfg[cfgApiserverID] = token.apiserverID


} func (ts *azureTokenSource) refreshToken(token *azureToken) (*azureToken, error) { <del> oauthConfig, err := adal.NewOAuthConfig(azure.PublicCloud.ActiveDirectoryEndpoint, token.tenantID) </del> <ins> env, err := azure.EnvironmentFromName(token.environment) if err != nil { return nil, err } oauthConfig, err := adal.NewOAuthConfig(env.ActiveDirectoryEndpoint, token.tenantID) </ins> if err != nil { return nil, fmt.Errorf("building the OAuth configuration for token refresh: %v", err) }


return &azureToken{ token:       spt.Token(), <ins> environment: token.environment, </ins> clientID:    token.clientID, tenantID:    token.tenantID, apiserverID: token.apiserverID,


return &azureToken{ token:       *token, <ins> environment: ts.environment.Name, </ins> clientID:    ts.clientID, tenantID:    ts.tenantID, apiserverID: ts.apiserverID,


wantCfg := token2Cfg(token) persistedCfg := persiter.Cache() <ins> wantCfgLen := len(wantCfg) persistedCfgLen := len(persistedCfg) if wantCfgLen != persistedCfgLen { t.Errorf("wantCfgLen and persistedCfgLen do not match, wantCfgLen=%v, persistedCfgLen=%v", wantCfgLen, persistedCfgLen) } </ins> for k, v := range persistedCfg { if strings.Compare(v, wantCfg[k]) != 0 { t.Errorf("Token() persisted cfg %s: got %v, want %v", k, v, wantCfg[k])


func (ts *fakeTokenSource) Token() (*azureToken, error) { return &azureToken{ token:       newFackeAzureToken(ts.accessToken, ts.expiresOn), <ins> environment: "testenv", </ins> clientID:    "fake", tenantID:    "fake", apiserverID: "fake",


cfg := make(map[string]string) cfg[cfgAccessToken] = token.token.AccessToken cfg[cfgRefreshToken] = token.token.RefreshToken <ins> cfg[cfgEnvironment] = token.environment </ins> cfg[cfgClientID] = token.clientID cfg[cfgTenantID] = token.tenantID cfg[cfgApiserverID] = token.apiserverID


nodestatus.PIDPressureCondition(kl.clock.Now, kl.evictionManager.IsUnderPIDPressure, kl.recordNodeStatusEvent), nodestatus.ReadyCondition(kl.clock.Now, kl.runtimeState.runtimeErrors, kl.runtimeState.networkErrors, validateHostFunc, kl.containerManager.Status, kl.recordNodeStatusEvent), nodestatus.VolumesInUse(kl.volumeManager.ReconcilerStatesHasBeenSynced, kl.volumeManager.GetVolumesInUse), <ins> nodestatus.RemoveOutOfDiskCondition(), </ins> // TODO(mtaufen): I decided not to move this setter for now, since all it does is send an event // and record state back to the Kubelet runtime object. In the future, I'd like to isolate // these side-effects by decoupling the decisions to send events and partial status recording


return nil } } <ins> // RemoveOutOfDiskCondition removes stale OutOfDisk condition // OutOfDisk condition has been removed from kubelet in 1.12 func RemoveOutOfDiskCondition() Setter { return func(node *v1.Node) error { var conditions []v1.NodeCondition for i := range node.Status.Conditions { if node.Status.Conditions[i].Type != v1.NodeOutOfDisk { conditions = append(conditions, node.Status.Conditions[i]) } } node.Status.Conditions = conditions return nil } } </ins>


} } <ins> func TestRemoveOutOfDiskCondition(t *testing.T) { now := time.Now() var cases = []struct { desc       string inputNode  *v1.Node expectNode *v1.Node }{ { desc: "should remove stale OutOfDiskCondition from node status", inputNode: &v1.Node{ Status: v1.NodeStatus{ Conditions: []v1.NodeCondition{ *makeMemoryPressureCondition(false, now, now), { Type:   v1.NodeOutOfDisk, Status: v1.ConditionFalse, }, *makeDiskPressureCondition(false, now, now), }, }, }, expectNode: &v1.Node{ Status: v1.NodeStatus{ Conditions: []v1.NodeCondition{ *makeMemoryPressureCondition(false, now, now), *makeDiskPressureCondition(false, now, now), }, }, }, }, } for _, tc := range cases { t.Run(tc.desc, func(t *testing.T) { // construct setter setter := RemoveOutOfDiskCondition() // call setter on node if err := setter(tc.inputNode); err != nil { t.Fatalf("unexpected error: %v", err) } // check expected node assert.True(t, apiequality.Semantic.DeepEqual(tc.expectNode, tc.inputNode), "Diff: %s", diff.ObjectDiff(tc.expectNode, tc.inputNode)) }) } } </ins> // Test Helpers: // sortableNodeAddress is a type for sorting []v1.NodeAddress


go_test( name = "go_default_test", <del> srcs = ["storage_test.go"], </del> <ins> srcs = [ "eviction_test.go", "storage_test.go", ], </ins> embed = [":go_default_library"], deps = [ "//pkg/apis/core:go_default_library", <ins> "//pkg/apis/policy:go_default_library", "//pkg/client/clientset_generated/internalclientset/fake:go_default_library", </ins> "//pkg/registry/registrytest:go_default_library", "//pkg/securitycontext:go_default_library", "//staging/src/k8s.io/api/core/v1:go_default_library",


// At this point there was either no PDB or we succeeded in decrementing // Try the delete <del> _, _, err = r.store.Delete(ctx, eviction.Name, eviction.DeleteOptions) </del> <ins> deletionOptions := eviction.DeleteOptions if deletionOptions == nil { // default to non-nil to trigger graceful deletion deletionOptions = &metav1.DeleteOptions{} } _, _, err = r.store.Delete(ctx, eviction.Name, deletionOptions) </ins> if err != nil { return nil, err }


<ins> /* Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package storage import ( "testing" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" api "k8s.io/kubernetes/pkg/apis/core" "k8s.io/kubernetes/pkg/apis/policy" "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset/fake" ) func TestEviction(t *testing.T) { testcases := []struct { name     string pdbs     []runtime.Object pod      *api.Pod eviction *policy.Eviction expectError   bool expectDeleted bool }{ { name:          "no pdbs, unscheduled pod, nil delete options, deletes immediately", pdbs:          nil, pod:           validNewPod(), eviction:      &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}}, expectDeleted: true, }, { name:          "no pdbs, scheduled pod, nil delete options, deletes gracefully", pdbs:          nil, pod:           func() *api.Pod { pod := validNewPod(); pod.Spec.NodeName = "foo"; return pod }(), eviction:      &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}}, expectDeleted: false, // not deleted immediately because of graceful deletion }, { name:          "no pdbs, scheduled pod, empty delete options, deletes gracefully", pdbs:          nil, pod:           func() *api.Pod { pod := validNewPod(); pod.Spec.NodeName = "foo"; return pod }(), eviction:      &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}, DeleteOptions: &metav1.DeleteOptions{}}, expectDeleted: false, // not deleted immediately because of graceful deletion }, { name:          "no pdbs, scheduled pod, graceless delete options, deletes immediately", pdbs:          nil, pod:           func() *api.Pod { pod := validNewPod(); pod.Spec.NodeName = "foo"; return pod }(), eviction:      &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)}, expectDeleted: true, }, { name: "matching pdbs with no disruptions allowed", pdbs: []runtime.Object{&policy.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}, Spec:       policy.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}}, Status:     policy.PodDisruptionBudgetStatus{PodDisruptionsAllowed: 0}, }}, pod: func() *api.Pod { pod := validNewPod() pod.Labels = map[string]string{"a": "true"} pod.Spec.NodeName = "foo" return pod }(), eviction:    &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)}, expectError: true, }, { name: "matching pdbs with disruptions allowed", pdbs: []runtime.Object{&policy.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}, Spec:       policy.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"a": "true"}}}, Status:     policy.PodDisruptionBudgetStatus{PodDisruptionsAllowed: 1}, }}, pod: func() *api.Pod { pod := validNewPod() pod.Labels = map[string]string{"a": "true"} pod.Spec.NodeName = "foo" return pod }(), eviction:      &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)}, expectDeleted: true, }, { name: "non-matching pdbs", pdbs: []runtime.Object{&policy.PodDisruptionBudget{ ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}, Spec:       policy.PodDisruptionBudgetSpec{Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"b": "true"}}}, Status:     policy.PodDisruptionBudgetStatus{PodDisruptionsAllowed: 0}, }}, pod: func() *api.Pod { pod := validNewPod() pod.Labels = map[string]string{"a": "true"} pod.Spec.NodeName = "foo" return pod }(), eviction:      &policy.Eviction{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"}, DeleteOptions: metav1.NewDeleteOptions(0)}, expectDeleted: true, }, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { testContext := genericapirequest.WithNamespace(genericapirequest.NewContext(), metav1.NamespaceDefault) storage, _, _, server := newStorage(t) defer server.Terminate(t) defer storage.Store.DestroyFunc() if tc.pod != nil { if _, err := storage.Create(testContext, tc.pod, nil, &metav1.CreateOptions{}); err != nil { t.Error(err) } } client := fake.NewSimpleClientset(tc.pdbs...) evictionRest := newEvictionStorage(storage.Store, client.Policy()) _, err := evictionRest.Create(testContext, tc.eviction, nil, &metav1.CreateOptions{}) if (err != nil) != tc.expectError { t.Errorf("expected error=%v, got %v", tc.expectError, err) return } if tc.expectError { return } if tc.pod != nil { existingPod, err := storage.Get(testContext, tc.pod.Name, &metav1.GetOptions{}) if tc.expectDeleted { if !apierrors.IsNotFound(err) { t.Errorf("expected to be deleted, lookup returned %#v", existingPod) } return } else if apierrors.IsNotFound(err) { t.Errorf("expected graceful deletion, got %v", err) return } if err != nil { t.Errorf("%#v", err) return } if existingPod.(*api.Pod).DeletionTimestamp == nil { t.Errorf("expected gracefully deleted pod with deletionTimestamp set, got %#v", existingPod) } } }) } } </ins>


} createNode2NormalPodEviction := func(client clientset.Interface) func() error { return func() error { <ins> zero := int64(0) </ins> return client.Policy().Evictions("ns").Evict(&policy.Eviction{ TypeMeta: metav1.TypeMeta{ APIVersion: "policy/v1beta1",


Name:      "node2normalpod", Namespace: "ns", }, <ins> DeleteOptions: &metav1.DeleteOptions{GracePeriodSeconds: &zero}, </ins> }) } } createNode2MirrorPodEviction := func(client clientset.Interface) func() error { return func() error { <ins> zero := int64(0) </ins> return client.Policy().Evictions("ns").Evict(&policy.Eviction{ TypeMeta: metav1.TypeMeta{ APIVersion: "policy/v1beta1",


Name:      "node2mirrorpod", Namespace: "ns", }, <ins> DeleteOptions: &metav1.DeleteOptions{GracePeriodSeconds: &zero}, </ins> }) } }


import ( "context" "fmt" <ins> "path" </ins> "sync" "sync/atomic" "time"


client := clientValue.Load().(*clientv3.Client) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() <del> if _, err := client.Cluster.MemberList(ctx); err != nil { return fmt.Errorf("error listing etcd members: %v", err) </del> <ins> // See https://github.com/etcd-io/etcd/blob/master/etcdctl/ctlv3/command/ep_command.go#L118 _, err := client.Get(ctx, path.Join(c.Prefix, "health")) if err == nil { return nil </ins> } <del> return nil </del> <ins> return fmt.Errorf("error getting data from etcd: %v", err) </ins> }, nil }


} } <del> // Mount : mounts source to target as NTFS with given options. </del> <ins> // Mount : mounts source to target with given options. // currently only supports cifs(smb), bind mount(for disk) </ins> func (mounter *Mounter) Mount(source string, target string, fstype string, options []string) error { target = normalizeWindowsPath(target) if source == "tmpfs" { <del> klog.V(3).Infof("azureMount: mounting source (%q), target (%q), with options (%q)", source, target, options) </del> <ins> klog.V(3).Infof("mounting source (%q), target (%q), with options (%q)", source, target, options) </ins> return os.MkdirAll(target, 0755) }


return err } <del> klog.V(4).Infof("azureMount: mount options(%q) source:%q, target:%q, fstype:%q, begin to mount", </del> <ins> klog.V(4).Infof("mount options(%q) source:%q, target:%q, fstype:%q, begin to mount", </ins> options, source, target, fstype) <del> bindSource := "" </del> <ins> bindSource := source </ins> // tell it's going to mount azure disk or azure file according to options if bind, _, _ := isBind(options); bind {


bindSource = normalizeWindowsPath(source) } else { if len(options) < 2 { <del> klog.Warningf("azureMount: mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting", </del> <ins> klog.Warningf("mount options(%q) command number(%d) less than 2, source:%q, target:%q, skip mounting", </ins> options, len(options), source, target) return nil } // currently only cifs mount is supported if strings.ToLower(fstype) != "cifs" { <del> return fmt.Errorf("azureMount: only cifs mount is supported now, fstype: %q, mounting source (%q), target (%q), with options (%q)", fstype, source, target, options) </del> <ins> return fmt.Errorf("only cifs mount is supported now, fstype: %q, mounting source (%q), target (%q), with options (%q)", fstype, source, target, options) </ins> } <del> bindSource = source // use PowerShell Environment Variables to store user input string to prevent command line injection // https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_environment_variables?view=powershell-5.1 cmdLine := fmt.Sprintf(`$PWord = ConvertTo-SecureString -String $Env:smbpassword -AsPlainText -Force` + `;$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $Env:smbuser, $PWord` + `;New-SmbGlobalMapping -RemotePath $Env:smbremotepath -Credential $Credential`) cmd := exec.Command("powershell", "/c", cmdLine) cmd.Env = append(os.Environ(), fmt.Sprintf("smbuser=%s", options[0]), fmt.Sprintf("smbpassword=%s", options[1]), fmt.Sprintf("smbremotepath=%s", source)) if output, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("azureMount: SmbGlobalMapping failed: %v, only SMB mount is supported now, output: %q", err, string(output)) </del> <ins> if output, err := newSMBMapping(options[0], options[1], source); err != nil { if isSMBMappingExist(source) { klog.V(2).Infof("SMB Mapping(%s) already exists, now begin to remove and remount", source) if output, err := removeSMBMapping(source); err != nil { return fmt.Errorf("Remove-SmbGlobalMapping failed: %v, output: %q", err, output) } if output, err := newSMBMapping(options[0], options[1], source); err != nil { return fmt.Errorf("New-SmbGlobalMapping remount failed: %v, output: %q", err, output) } } else { return fmt.Errorf("New-SmbGlobalMapping failed: %v, output: %q", err, output) } </ins> } }


return nil } <ins> // do the SMB mount with username, password, remotepath // return (output, error) func newSMBMapping(username, password, remotepath string) (string, error) { if username == "" || password == "" || remotepath == "" { return "", fmt.Errorf("invalid parameter(username: %s, password: %s, remoteapth: %s)", username, password, remotepath) } // use PowerShell Environment Variables to store user input string to prevent command line injection // https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_environment_variables?view=powershell-5.1 cmdLine := `$PWord = ConvertTo-SecureString -String $Env:smbpassword -AsPlainText -Force` + `;$Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList $Env:smbuser, $PWord` + `;New-SmbGlobalMapping -RemotePath $Env:smbremotepath -Credential $Credential` cmd := exec.Command("powershell", "/c", cmdLine) cmd.Env = append(os.Environ(), fmt.Sprintf("smbuser=%s", username), fmt.Sprintf("smbpassword=%s", password), fmt.Sprintf("smbremotepath=%s", remotepath)) output, err := cmd.CombinedOutput() return string(output), err } // check whether remotepath is already mounted func isSMBMappingExist(remotepath string) bool { cmd := exec.Command("powershell", "/c", `Get-SmbGlobalMapping -RemotePath $Env:smbremotepath`) cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath)) _, err := cmd.CombinedOutput() return err == nil } // remove SMB mapping func removeSMBMapping(remotepath string) (string, error) { cmd := exec.Command("powershell", "/c", `Remove-SmbGlobalMapping -RemotePath $Env:smbremotepath -Force`) cmd.Env = append(os.Environ(), fmt.Sprintf("smbremotepath=%s", remotepath)) output, err := cmd.CombinedOutput() return string(output), err } </ins> // Unmount unmounts the target. func (mounter *Mounter) Unmount(target string) error { klog.V(4).Infof("azureMount: Unmount target (%q)", target)


} } } <ins> func TestNewSMBMapping(t *testing.T) { tests := []struct { username    string password    string remotepath  string expectError bool }{ { "", "password", `remotepath`, true, }, { "username", "", `remotepath`, true, }, { "username", "password", "", true, }, } for _, test := range tests { _, err := newSMBMapping(test.username, test.password, test.remotepath) if test.expectError { assert.NotNil(t, err, "Expect error during newSMBMapping(%s, %s, %s, %v)", test.username, test.password, test.remotepath) } else { assert.Nil(t, err, "Expect error is nil during newSMBMapping(%s, %s, %s, %v)", test.username, test.password, test.remotepath) } } } </ins>


name = "go_default_library", srcs = [ "density.go", <ins> "dns.go", </ins> "framework.go", "gmsa.go", "hybrid_network.go",


<ins> /* Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package windows import ( "regexp" "strings" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" imageutils "k8s.io/kubernetes/test/utils/image" "github.com/onsi/ginkgo" ) const dnsTestPodHostName = "dns-querier-1" const dnsTestServiceName = "dns-test-service" var _ = SIGDescribe("DNS", func() { ginkgo.BeforeEach(func() { framework.SkipUnlessNodeOSDistroIs("windows") }) f := framework.NewDefaultFramework("dns") ginkgo.It("should support configurable pod DNS servers", func() { ginkgo.By("Preparing a test DNS service with injected DNS names...") testInjectedIP := "1.1.1.1" testSearchPath := "resolv.conf.local" ginkgo.By("Creating a pod with dnsPolicy=None and customized dnsConfig...") testUtilsPod := generateDNSUtilsPod() testUtilsPod.Spec.DNSPolicy = v1.DNSNone testUtilsPod.Spec.DNSConfig = &v1.PodDNSConfig{ Nameservers: []string{testInjectedIP}, Searches:    []string{testSearchPath}, } testUtilsPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(testUtilsPod) framework.ExpectNoError(err) e2elog.Logf("Created pod %v", testUtilsPod) defer func() { e2elog.Logf("Deleting pod %s...", testUtilsPod.Name) if err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(testUtilsPod.Name, metav1.NewDeleteOptions(0)); err != nil { e2elog.Failf("Failed to delete pod %s: %v", testUtilsPod.Name, err) } }() framework.ExpectNoError(f.WaitForPodRunning(testUtilsPod.Name), "failed to wait for pod %s to be running", testUtilsPod.Name) ginkgo.By("Verifying customized DNS option is configured on pod...") cmd := []string{"ipconfig", "/all"} stdout, _, err := f.ExecWithOptions(framework.ExecOptions{ Command:       cmd, Namespace:     f.Namespace.Name, PodName:       testUtilsPod.Name, ContainerName: "util", CaptureStdout: true, CaptureStderr: true, }) framework.ExpectNoError(err) e2elog.Logf("ipconfig /all:n%s", stdout) dnsRegex, err := regexp.Compile(`DNS Servers[s*.]*:(s*[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3})+`) if dnsRegex.MatchString(stdout) { match := dnsRegex.FindString(stdout) if !strings.Contains(match, testInjectedIP) { e2elog.Failf("customized DNS options not found in ipconfig /all, got: %s", match) } } else { e2elog.Failf("cannot find DNS server info in ipconfig /all output: n%s", stdout) } // TODO: Add more test cases for other DNSPolicies. }) }) func generateDNSUtilsPod() *v1.Pod { return &v1.Pod{ TypeMeta: metav1.TypeMeta{ Kind: "Pod", }, ObjectMeta: metav1.ObjectMeta{ GenerateName: "e2e-dns-utils-", }, Spec: v1.PodSpec{ Containers: []v1.Container{ { Name:    "util", Image:   imageutils.GetE2EImage(imageutils.Dnsutils), Command: []string{"sleep", "10000"}, }, }, }, } } </ins>


if spec.IsKubeletExpandable() { // for kubelet expandable volumes, return a noop plugin that // returns success for expand on the controller <del> klog.Warningf("FindExpandablePluginBySpec(%s) -> returning noopExpandableVolumePluginInstance", spec.Name()) </del> <ins> klog.V(4).Infof("FindExpandablePluginBySpec(%s) -> returning noopExpandableVolumePluginInstance", spec.Name()) </ins> return &noopExpandableVolumePluginInstance{spec}, nil } <del> klog.Warningf("FindExpandablePluginBySpec(%s) -> err:%v", spec.Name(), err) </del> <ins> klog.V(4).Infof("FindExpandablePluginBySpec(%s) -> err:%v", spec.Name(), err) </ins> return nil, err }


optional APIServiceStatus status = 3; } <ins> // APIServiceCondition describes the state of an APIService at a particular point </ins> message APIServiceCondition { // Type is the type of the condition. optional string type = 1;


# Admission Controllers to invoke prior to persisting objects in cluster ENABLE_ADMISSION_PLUGINS="LimitRanger,ResourceQuota" <del> DISABLE_ADMISSION_PLUGINS="ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook" </del> <ins> DISABLE_ADMISSION_PLUGINS="ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,StorageObjectInUseProtection" </ins> # Include RBAC (to exercise bootstrapping), and AlwaysAllow to allow all actions AUTHORIZATION_MODE="RBAC,AlwaysAllow"


ETCDCTL=$(which etcdctl) KUBECTL="${KUBE_OUTPUT_HOSTBIN}/kubectl" UPDATE_ETCD_OBJECTS_SCRIPT="${KUBE_ROOT}/cluster/update-storage-objects.sh" <del> DISABLE_ADMISSION_PLUGINS="ServiceAccount,NamespaceLifecycle,LimitRanger,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,PersistentVolumeLabel,DefaultStorageClass" </del> <ins> DISABLE_ADMISSION_PLUGINS="ServiceAccount,NamespaceLifecycle,LimitRanger,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,PersistentVolumeLabel,DefaultStorageClass,StorageObjectInUseProtection" </ins> function startApiServer() { local storage_versions=${1:-""}


// DefaultOffAdmissionPlugins get admission plugins off by default for kube-apiserver. func DefaultOffAdmissionPlugins() sets.String { defaultOnPlugins := sets.NewString( <del> lifecycle.PluginName,                //NamespaceLifecycle limitranger.PluginName,              //LimitRanger serviceaccount.PluginName,           //ServiceAccount setdefault.PluginName,               //DefaultStorageClass resize.PluginName,                   //PersistentVolumeClaimResize defaulttolerationseconds.PluginName, //DefaultTolerationSeconds mutatingwebhook.PluginName,          //MutatingAdmissionWebhook validatingwebhook.PluginName,        //ValidatingAdmissionWebhook resourcequota.PluginName,            //ResourceQuota </del> <ins> lifecycle.PluginName,                    //NamespaceLifecycle limitranger.PluginName,                  //LimitRanger serviceaccount.PluginName,               //ServiceAccount setdefault.PluginName,                   //DefaultStorageClass resize.PluginName,                       //PersistentVolumeClaimResize defaulttolerationseconds.PluginName,     //DefaultTolerationSeconds mutatingwebhook.PluginName,              //MutatingAdmissionWebhook validatingwebhook.PluginName,            //ValidatingAdmissionWebhook resourcequota.PluginName,                //ResourceQuota storageobjectinuseprotection.PluginName, //StorageObjectInUseProtection </ins> ) if utilfeature.DefaultFeatureGate.Enabled(features.PodPriority) {


var hns HostNetworkService hns = hnsV1{} supportedFeatures := hcn.GetSupportedFeatures() <del> if supportedFeatures.Api.V2 { </del> <ins> if supportedFeatures.RemoteSubnet { </ins> hns = hnsV2{} }


} By("creating the pod with failed condition") <del> pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod) Expect(err).ToNot(HaveOccurred(), "while creating pod") err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout) Expect(err).To(HaveOccurred(), "while waiting for pod to be running") </del> var podClient *framework.PodClient podClient = f.PodClient() <ins> pod = podClient.Create(pod) err := framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout) Expect(err).To(HaveOccurred(), "while waiting for pod to be running") </ins> By("updating the pod") podClient.Update(podName, func(pod *v1.Pod) {


} By("creating the pod") <del> pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod) </del> <ins> var podClient *framework.PodClient podClient = f.PodClient() pod = podClient.Create(pod) </ins> By("waiting for pod running") <del> err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout) </del> <ins> err := framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout) </ins> Expect(err).NotTo(HaveOccurred(), "while waiting for pod to be running") By("creating a file in subpath")


} By("updating the annotation value") <del> var podClient *framework.PodClient podClient = f.PodClient() </del> podClient.Update(podName, func(pod *v1.Pod) { pod.ObjectMeta.Annotations["mysubpath"] = "mynewpath" })


// Start pod By(fmt.Sprintf("Creating pod %s", pod.Name)) <del> pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod) Expect(err).ToNot(HaveOccurred(), "while creating pod") </del> <ins> var podClient *framework.PodClient podClient = f.PodClient() pod = podClient.Create(pod) </ins> defer func() { framework.DeletePodWithWait(f, f.ClientSet, pod) }() <del> err = framework.WaitForPodRunningInNamespace(f.ClientSet, pod) </del> <ins> err := framework.WaitForPodRunningInNamespace(f.ClientSet, pod) </ins> Expect(err).ToNot(HaveOccurred(), "while waiting for pod to be running") <del> var podClient *framework.PodClient podClient = f.PodClient() </del> By("updating the pod") podClient.Update(podName, func(pod *v1.Pod) { pod.ObjectMeta.Annotations = map[string]string{"mysubpath": "newsubpath"}


func testPodFailSubpath(f *framework.Framework, pod *v1.Pod) { <del> pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(pod) Expect(err).ToNot(HaveOccurred(), "while creating pod") </del> <ins> var podClient *framework.PodClient podClient = f.PodClient() pod = podClient.Create(pod) </ins> defer func() { framework.DeletePodWithWait(f, f.ClientSet, pod) }() <del> err = framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout) </del> <ins> err := framework.WaitTimeoutForPodRunningInNamespace(f.ClientSet, pod.Name, pod.Namespace, framework.PodStartShortTimeout) </ins> Expect(err).To(HaveOccurred(), "while waiting for pod to be running") }


var _ = utils.SIGDescribe("CSI mock volume", func() { type testParameters struct { <del> attachable      bool </del> <ins> disableAttach   bool </ins> attachLimit     int registerDriver  bool podInfoVersion  *string


csics := f.CSIClientSet var err error <del> m.driver = drivers.InitMockCSIDriver(tp.registerDriver, tp.attachable, tp.podInfoVersion, tp.attachLimit) </del> <ins> m.driver = drivers.InitMockCSIDriver(tp.registerDriver, !tp.disableAttach, tp.podInfoVersion, tp.attachLimit) </ins> config, testCleanup := m.driver.PrepareTest(f) m.testCleanups = append(m.testCleanups, testCleanup) m.config = config


// The CSIDriverRegistry feature gate is needed for this test in Kubernetes 1.12. Context("CSI attach test using mock driver [Feature:CSIDriverRegistry]", func() { tests := []struct { <del> name             string driverAttachable bool deployDriverCRD  bool </del> <ins> name            string disableAttach   bool deployDriverCRD bool </ins> }{ { <del> name:             "should not require VolumeAttach for drivers without attachment", driverAttachable: false, deployDriverCRD:  true, </del> <ins> name:            "should not require VolumeAttach for drivers without attachment", disableAttach:   true, deployDriverCRD: true, </ins> }, { <del> name:             "should require VolumeAttach for drivers with attachment", driverAttachable: true, deployDriverCRD:  true, </del> <ins> name:            "should require VolumeAttach for drivers with attachment", deployDriverCRD: true, </ins> }, { <del> name:             "should preserve attachment policy when no CSIDriver present", driverAttachable: true, deployDriverCRD:  false, </del> <ins> name:            "should preserve attachment policy when no CSIDriver present", deployDriverCRD: false, </ins> }, } for _, t := range tests { test := t It(t.name, func() { var err error <del> init(testParameters{registerDriver: test.deployDriverCRD, attachable: test.driverAttachable}) </del> <ins> init(testParameters{registerDriver: test.deployDriverCRD, disableAttach: test.disableAttach}) </ins> defer cleanup() _, claim, pod := createPod()


_, err = m.cs.StorageV1beta1().VolumeAttachments().Get(attachmentName, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { <del> if test.driverAttachable { </del> <ins> if !test.disableAttach { </ins> framework.ExpectNoError(err, "Expected VolumeAttachment but none was found") } } else { framework.ExpectNoError(err, "Failed to find VolumeAttachment") } } <del> if !test.driverAttachable { </del> <ins> if test.disableAttach { </ins> Expect(err).To(HaveOccurred(), "Unexpected VolumeAttachment found") } })


// define volume limit to be 2 for this test var err error <del> init(testParameters{attachable: true, nodeSelectorKey: "node-attach-limit-csi", attachLimit: 2}) </del> <ins> init(testParameters{nodeSelectorKey: "node-attach-limit-csi", attachLimit: 2}) </ins> defer cleanup() nodeName := m.config.ClientNodeName attachKey := v1.ResourceName(volumeutil.GetCSIAttachLimitKey(m.provisioner))


func TestGetZone(t *testing.T) { cloud := &Cloud{ Config: Config{ <del> Location: "eastus", </del> <ins> Location:            "eastus", UseInstanceMetadata: true, </ins> }, } testcases := []struct {


import ( "context" "fmt" <ins> "os" </ins> "strconv" "strings"


// GetZone returns the Zone containing the current availability zone and locality region that the program is running in. // If the node is not running with availability zones, then it will fall back to fault domain. func (az *Cloud) GetZone(ctx context.Context) (cloudprovider.Zone, error) { <del> metadata, err := az.metadata.GetMetadata() if err != nil { return cloudprovider.Zone{}, err } </del> <ins> if az.UseInstanceMetadata { metadata, err := az.metadata.GetMetadata() if err != nil { return cloudprovider.Zone{}, err } </ins> <del> if metadata.Compute == nil { return cloudprovider.Zone{}, fmt.Errorf("failure of getting compute information from instance metadata") } </del> <ins> if metadata.Compute == nil { return cloudprovider.Zone{}, fmt.Errorf("failure of getting compute information from instance metadata") } </ins> <del> zone := "" if metadata.Compute.Zone != "" { zoneID, err := strconv.Atoi(metadata.Compute.Zone) if err != nil { return cloudprovider.Zone{}, fmt.Errorf("failed to parse zone ID %q: %v", metadata.Compute.Zone, err) </del> <ins> zone := "" if metadata.Compute.Zone != "" { zoneID, err := strconv.Atoi(metadata.Compute.Zone) if err != nil { return cloudprovider.Zone{}, fmt.Errorf("failed to parse zone ID %q: %v", metadata.Compute.Zone, err) } zone = az.makeZone(zoneID) } else { klog.V(3).Infof("Availability zone is not enabled for the node, falling back to fault domain") zone = metadata.Compute.FaultDomain </ins> } <del> zone = az.makeZone(zoneID) } else { klog.V(3).Infof("Availability zone is not enabled for the node, falling back to fault domain") zone = metadata.Compute.FaultDomain } return cloudprovider.Zone{ FailureDomain: zone, Region:        az.Location, }, nil </del> <ins> return cloudprovider.Zone{ FailureDomain: zone, Region:        az.Location, }, nil } // if UseInstanceMetadata is false, get Zone name by calling ARM hostname, err := os.Hostname() if err != nil { return cloudprovider.Zone{}, fmt.Errorf("failure getting hostname from kernel") } return az.vmSet.GetZoneByNodeName(strings.ToLower(hostname)) </ins> } // GetZoneByProviderID implements Zones.GetZoneByProviderID


"//pkg/kubelet/apis/resourcemetrics/v1alpha1:go_default_library", "//pkg/kubelet/container:go_default_library", "//pkg/kubelet/prober:go_default_library", <ins> "//pkg/kubelet/server/metrics:go_default_library", </ins> "//pkg/kubelet/server/portforward:go_default_library", "//pkg/kubelet/server/remotecommand:go_default_library", "//pkg/kubelet/server/stats:go_default_library",


name = "all-srcs", srcs = [ ":package-srcs", <ins> "//pkg/kubelet/server/metrics:all-srcs", </ins> "//pkg/kubelet/server/portforward:all-srcs", "//pkg/kubelet/server/remotecommand:all-srcs", "//pkg/kubelet/server/stats:all-srcs",


<ins> package(default_visibility = ["//visibility:public"]) load( "@io_bazel_rules_go//go:def.bzl", "go_library", ) go_library( name = "go_default_library", srcs = ["metrics.go"], importpath = "k8s.io/kubernetes/pkg/kubelet/server/metrics", deps = [ "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", ], ) filegroup( name = "package-srcs", srcs = glob(["**"]), tags = ["automanaged"], visibility = ["//visibility:private"], ) filegroup( name = "all-srcs", srcs = [":package-srcs"], tags = ["automanaged"], ) </ins>


<ins> /* Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package metrics import ( "sync" "time" "github.com/prometheus/client_golang/prometheus" ) const ( kubeletSubsystem = "kubelet" ) var ( // HTTPRequests tracks the number of the http requests received since the server started. HTTPRequests = prometheus.NewCounterVec( prometheus.CounterOpts{ Subsystem: kubeletSubsystem, Name:      "http_requests_total", Help:      "Number of the http requests received since the server started", }, // server_type aims to differentiate the readonly server and the readwrite server. // long_running marks whether the request is long-running or not. // Currently, long-running requests include exec/attach/portforward/debug. []string{"method", "path", "host", "server_type", "long_running"}, ) // HTTPRequestsDuration tracks the duration in seconds to serve http requests. HTTPRequestsDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Subsystem: kubeletSubsystem, Name:      "http_requests_duration_seconds", Help:      "Duration in seconds to serve http requests", // Use DefBuckets for now, will customize the buckets if necessary. Buckets: prometheus.DefBuckets, }, []string{"method", "path", "host", "server_type", "long_running"}, ) // HTTPInflightRequests tracks the number of the inflight http requests. HTTPInflightRequests = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Subsystem: kubeletSubsystem, Name:      "http_inflight_requests", Help:      "Number of the inflight http requests", }, []string{"method", "path", "host", "server_type", "long_running"}, ) ) var registerMetrics sync.Once // Register all metrics. func Register() { registerMetrics.Do(func() { prometheus.MustRegister(HTTPRequests) prometheus.MustRegister(HTTPRequestsDuration) prometheus.MustRegister(HTTPInflightRequests) }) } // SinceInSeconds gets the time since the specified start in seconds. func SinceInSeconds(start time.Time) float64 { return time.Since(start).Seconds() } </ins>


"k8s.io/kubernetes/pkg/kubelet/apis/resourcemetrics/v1alpha1" kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/prober" <ins> servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics" </ins> "k8s.io/kubernetes/pkg/kubelet/server/portforward" remotecommandserver "k8s.io/kubernetes/pkg/kubelet/server/remotecommand" "k8s.io/kubernetes/pkg/kubelet/server/stats"


proxyStream(response.ResponseWriter, request.Request, url) } <ins> // trimURLPath trims a URL path. // For paths in the format of "/metrics/xxx", "metrics/xxx" is returned; // For all other paths, the first part of the path is returned. func trimURLPath(path string) string { parts := strings.SplitN(strings.TrimPrefix(path, "/"), "/", 3) if len(parts) == 0 { return path } if parts[0] == "metrics" && len(parts) > 1 { return fmt.Sprintf("%s/%s", parts[0], parts[1]) } return parts[0] } // isLongRunningRequest determines whether the request is long-running or not. func isLongRunningRequest(path string) bool { longRunningRequestPaths := []string{"exec", "attach", "portforward", "debug"} for _, p := range longRunningRequestPaths { if p == path { return true } } return false } </ins> // ServeHTTP responds to HTTP requests on the Kubelet. func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) { defer httplog.NewLogged(req, &w).StacktraceWhen(


http.StatusSwitchingProtocols, ), ).Log() <ins> // monitor http requests var serverType string if s.auth == nil { serverType = "readonly" } else { serverType = "readwrite" } method, path, host := req.Method, trimURLPath(req.URL.Path), req.URL.Host longRunning := strconv.FormatBool(isLongRunningRequest(path)) servermetrics.HTTPRequests.WithLabelValues(method, path, host, serverType, longRunning).Inc() servermetrics.HTTPInflightRequests.WithLabelValues(method, path, host, serverType, longRunning).Inc() defer servermetrics.HTTPInflightRequests.WithLabelValues(method, path, host, serverType, longRunning).Dec() startTime := time.Now() defer servermetrics.HTTPRequestsDuration.WithLabelValues(method, path, host, serverType, longRunning).Observe(servermetrics.SinceInSeconds(startTime)) </ins> s.restfulCont.ServeHTTP(w, req) }


assert.Equal(t, http.StatusOK, resp.StatusCode) } <ins> func TestTrimURLPath(t *testing.T) { tests := []struct { path, expected string }{ {"", ""}, {"//", ""}, {"/pods", "pods"}, {"pods", "pods"}, {"pods/", "pods"}, {"good/", "good"}, {"pods/probes", "pods"}, {"metrics", "metrics"}, {"metrics/resource", "metrics/resource"}, {"metrics/hello", "metrics/hello"}, } for _, test := range tests { assert.Equal(t, test.expected, trimURLPath(test.path), fmt.Sprintf("path is: %s", test.path)) } } </ins>


return &container, nil } } <ins> for _, container := range pod.Spec.InitContainers { if container.Name == containerName { return &container, nil } } </ins> return nil, fmt.Errorf("container %s not found", containerName) }


pod:           getPod("foo", "", "", "10Mi", "100Mi"), expectedValue: "104857600", }, <ins> { fs: &v1.ResourceFieldSelector{ Resource: "limits.cpu", }, cName:         "init-foo", pod:           getPod("foo", "", "9", "", ""), expectedValue: "9", }, { fs: &v1.ResourceFieldSelector{ Resource: "requests.cpu", }, cName:         "init-foo", pod:           getPod("foo", "", "", "", ""), expectedValue: "0", }, { fs: &v1.ResourceFieldSelector{ Resource: "requests.cpu", }, cName:         "init-foo", pod:           getPod("foo", "8", "", "", ""), expectedValue: "8", }, { fs: &v1.ResourceFieldSelector{ Resource: "requests.cpu", }, cName:         "init-foo", pod:           getPod("foo", "100m", "", "", ""), expectedValue: "1", }, { fs: &v1.ResourceFieldSelector{ Resource: "requests.cpu", Divisor:  resource.MustParse("100m"), }, cName:         "init-foo", pod:           getPod("foo", "1200m", "", "", ""), expectedValue: "12", }, { fs: &v1.ResourceFieldSelector{ Resource: "requests.memory", }, cName:         "init-foo", pod:           getPod("foo", "", "", "100Mi", ""), expectedValue: "104857600", }, { fs: &v1.ResourceFieldSelector{ Resource: "requests.memory", Divisor:  resource.MustParse("1Mi"), }, cName:         "init-foo", pod:           getPod("foo", "", "", "100Mi", "1Gi"), expectedValue: "100", }, { fs: &v1.ResourceFieldSelector{ Resource: "limits.memory", }, cName:         "init-foo", pod:           getPod("foo", "", "", "10Mi", "100Mi"), expectedValue: "104857600", }, </ins> } as := assert.New(t) for idx, tc := range cases {


Resources: resources, }, }, <ins> InitContainers: []v1.Container{ { Name:      "init-" + cname, Resources: resources, }, }, </ins> }, } }


// BusyBoxImage is the image URI of BusyBox. BusyBoxImage = imageutils.GetE2EImage(imageutils.BusyBox) <ins> // AgnHostImage is the image URI of AgnHost AgnHostImage = imageutils.GetE2EImage(imageutils.Agnhost) </ins> // For parsing Kubectl version for version-skewed testing. gitVersionRegexp = regexp.MustCompile("GitVersion:"(v.+?)"")


Containers: []v1.Container{ { Name:    "detector", <del> Image:   imageutils.GetE2EImage(imageutils.Agnhost), Command: []string{"/bin/sleep", "3600"}, </del> <ins> Image:   AgnHostImage, Command: []string{"pause"}, </ins> }, }, },


} } <del> // PingCommand is the type to hold ping command. type PingCommand string const ( // IPv4PingCommand is a ping command for IPv4. IPv4PingCommand PingCommand = "ping" // IPv6PingCommand is a ping command for IPv6. IPv6PingCommand PingCommand = "ping6" ) </del> // CheckConnectivityToHost launches a pod to test connectivity to the specified // host. An error will be returned if the host is not reachable from the pod. // // An empty nodeName will use the schedule to choose where the pod is executed. <del> func CheckConnectivityToHost(f *Framework, nodeName, podName, host string, pingCmd PingCommand, timeout int) error { </del> <ins> func CheckConnectivityToHost(f *Framework, nodeName, podName, host string, port, timeout int) error { </ins> contName := fmt.Sprintf("%s-container", podName) command := []string{ <del> string(pingCmd), "-c", "3", // send 3 pings "-W", "2", // wait at most 2 seconds for a reply </del> <ins> "nc", "-vz", </ins> "-w", strconv.Itoa(timeout), host, <ins> strconv.Itoa(port), </ins> } pod := &v1.Pod{


Containers: []v1.Container{ { Name:    contName, <del> Image:   BusyBoxImage, </del> <ins> Image:   AgnHostImage, </ins> Command: command, }, },


Containers: []v1.Container{ { Name:  "agnhost", <del> Image: imageutils.GetE2EImage(imageutils.Agnhost), </del> <ins> Image: AgnHostImage, </ins> Args:  args, }, },


}) ginkgo.It("should provide Internet connection for containers [Feature:Networking-IPv4]", func() { <del> ginkgo.By("Running container which tries to ping 8.8.8.8") </del> <ins> ginkgo.By("Running container which tries to connect to 8.8.8.8") </ins> framework.ExpectNoError( <del> framework.CheckConnectivityToHost(f, "", "ping-test", "8.8.8.8", framework.IPv4PingCommand, 30)) </del> <ins> framework.CheckConnectivityToHost(f, "", "connectivity-test", "8.8.8.8", 53, 30)) </ins> }) ginkgo.It("should provide Internet connection for containers [Feature:Networking-IPv6][Experimental]", func() { <del> ginkgo.By("Running container which tries to ping 2001:4860:4860::8888") </del> <ins> ginkgo.By("Running container which tries to connect to 2001:4860:4860::8888") </ins> framework.ExpectNoError( <del> framework.CheckConnectivityToHost(f, "", "ping-test", "2001:4860:4860::8888", framework.IPv6PingCommand, 30)) </del> <ins> framework.CheckConnectivityToHost(f, "", "connectivity-test", "2001:4860:4860::8888", 53, 30)) </ins> }) // First test because it has no dependencies on variables created later on.


fs.BoolVar(&o.CleanupAndExit, "cleanup-iptables", o.CleanupAndExit, "If true cleanup iptables and ipvs rules and exit.") fs.MarkDeprecated("cleanup-iptables", "This flag is replaced by --cleanup.") fs.BoolVar(&o.CleanupAndExit, "cleanup", o.CleanupAndExit, "If true cleanup iptables and ipvs rules and exit.") <del> fs.BoolVar(&o.CleanupIPVS, "cleanup-ipvs", o.CleanupIPVS, "If true make kube-proxy cleanup ipvs rules before running.  Default is true") </del> <ins> fs.BoolVar(&o.CleanupIPVS, "cleanup-ipvs", o.CleanupIPVS, "If true and --cleanup is specified, kube-proxy will also flush IPVS rules, in addition to normal cleanup.") </ins> // All flags below here are deprecated and will eventually be removed.


// NewProxyServer returns a new ProxyServer. func NewProxyServer(o *Options) (*ProxyServer, error) { <del> return newProxyServer(o.config, o.CleanupAndExit, o.CleanupIPVS, o.scheme, o.master) </del> <ins> return newProxyServer(o.config, o.CleanupAndExit, o.scheme, o.master) </ins> } func newProxyServer( config *proxyconfigapi.KubeProxyConfiguration, cleanupAndExit bool, <del> cleanupIPVS bool, </del> scheme *runtime.Scheme, master string) (*ProxyServer, error) {


proxier = proxierIPTables serviceEventHandler = proxierIPTables endpointsEventHandler = proxierIPTables <del> // No turning back. Remove artifacts that might still exist from the userspace Proxier. klog.V(0).Info("Tearing down inactive rules.") // TODO this has side effects that should only happen when Run() is invoked. userspace.CleanupLeftovers(iptInterface) // IPVS Proxier will generate some iptables rules, need to clean them before switching to other proxy mode. // Besides, ipvs proxier will create some ipvs rules as well.  Because there is no way to tell if a given // ipvs rule is created by IPVS proxier or not.  Users should explicitly specify `--clean-ipvs=true` to flush // all ipvs rules when kube-proxy start up.  Users do this operation should be with caution. if canUseIPVS { ipvs.CleanupLeftovers(ipvsInterface, iptInterface, ipsetInterface, cleanupIPVS) } </del> } else if proxyMode == proxyModeIPVS { klog.V(0).Info("Using ipvs Proxier.") proxierIPVS, err := ipvs.NewProxier(


proxier = proxierIPVS serviceEventHandler = proxierIPVS endpointsEventHandler = proxierIPVS <del> klog.V(0).Info("Tearing down inactive rules.") // TODO this has side effects that should only happen when Run() is invoked. userspace.CleanupLeftovers(iptInterface) iptables.CleanupLeftovers(iptInterface) </del> } else { klog.V(0).Info("Using userspace Proxier.") // This is a proxy.LoadBalancer which NewProxier needs but has methods we don't need for