// +build !providerless

/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package aws

import (
	"context"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"reflect"
	"sort"
	"strings"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/elb"
	"github.com/aws/aws-sdk-go/service/elbv2"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/mock"
	"github.com/stretchr/testify/require"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes/fake"
	cloudvolume "k8s.io/cloud-provider/volume"
)

const TestClusterID = "clusterid.test"
const TestClusterName = "testCluster"

type MockedFakeEC2 struct {
	*FakeEC2Impl
	mock.Mock
}

func (m *MockedFakeEC2) expectDescribeSecurityGroups(clusterID, groupName string) {
	tags := []*ec2.Tag{
		{Key: aws.String(TagNameKubernetesClusterLegacy), Value: aws.String(clusterID)},
		{Key: aws.String(fmt.Sprintf("%s%s", TagNameKubernetesClusterPrefix, clusterID)), Value: aws.String(ResourceLifecycleOwned)},
	}

	m.On("DescribeSecurityGroups", &ec2.DescribeSecurityGroupsInput{Filters: []*ec2.Filter{
		newEc2Filter("group-name", groupName),
		newEc2Filter("vpc-id", ""),
	}}).Return([]*ec2.SecurityGroup{{Tags: tags}})
}

func (m *MockedFakeEC2) DescribeVolumes(request *ec2.DescribeVolumesInput) ([]*ec2.Volume, error) {
	args := m.Called(request)
	return args.Get(0).([]*ec2.Volume), nil
}

func (m *MockedFakeEC2) DeleteVolume(request *ec2.DeleteVolumeInput) (*ec2.DeleteVolumeOutput, error) {
	args := m.Called(request)
	return args.Get(0).(*ec2.DeleteVolumeOutput), nil
}

func (m *MockedFakeEC2) DescribeSecurityGroups(request *ec2.DescribeSecurityGroupsInput) ([]*ec2.SecurityGroup, error) {
	args := m.Called(request)
	return args.Get(0).([]*ec2.SecurityGroup), nil
}

func (m *MockedFakeEC2) CreateVolume(request *ec2.CreateVolumeInput) (*ec2.Volume, error) {
	// mock requires stable input, and in CreateDisk we invoke buildTags which uses
	// a map to create tags, which then get converted into an array. This leads to
	// unstable sorting order which confuses mock. Sorted tags are not needed in
	// regular code, but are a must in tests here:
	for i := 0; i < len(request.TagSpecifications); i++ {
		if request.TagSpecifications[i] == nil {
			continue
		}
		tags := request.TagSpecifications[i].Tags
		sort.Slice(tags, func(i, j int) bool {
			if tags[i] == nil && tags[j] != nil {
				return false
			}
			if tags[i] != nil && tags[j] == nil {
				return true
			}
			return *tags[i].Key < *tags[j].Key
		})
	}
	args := m.Called(request)
	return args.Get(0).(*ec2.Volume), nil
}

type MockedFakeELB struct {
	*FakeELB
	mock.Mock
}

func (m *MockedFakeELB) DescribeLoadBalancers(input *elb.DescribeLoadBalancersInput) (*elb.DescribeLoadBalancersOutput, error) {
	args := m.Called(input)
	return args.Get(0).(*elb.DescribeLoadBalancersOutput), nil
}

func (m *MockedFakeELB) expectDescribeLoadBalancers(loadBalancerName string) {
	m.On("DescribeLoadBalancers", &elb.DescribeLoadBalancersInput{LoadBalancerNames: []*string{aws.String(loadBalancerName)}}).Return(&elb.DescribeLoadBalancersOutput{
		LoadBalancerDescriptions: []*elb.LoadBalancerDescription{{}},
	})
}

func (m *MockedFakeELB) AddTags(input *elb.AddTagsInput) (*elb.AddTagsOutput, error) {
	args := m.Called(input)
	return args.Get(0).(*elb.AddTagsOutput), nil
}

func (m *MockedFakeELB) ConfigureHealthCheck(input *elb.ConfigureHealthCheckInput) (*elb.ConfigureHealthCheckOutput, error) {
	args := m.Called(input)
	if args.Get(0) == nil {
		return nil, args.Error(1)
	}
	return args.Get(0).(*elb.ConfigureHealthCheckOutput), args.Error(1)
}

func (m *MockedFakeELB) expectConfigureHealthCheck(loadBalancerName *string, expectedHC *elb.HealthCheck, returnErr error) {
	expected := &elb.ConfigureHealthCheckInput{HealthCheck: expectedHC, LoadBalancerName: loadBalancerName}
	call := m.On("ConfigureHealthCheck", expected)
	if returnErr != nil {
		call.Return(nil, returnErr)
	} else {
		call.Return(&elb.ConfigureHealthCheckOutput{}, nil)
	}
}

func TestReadAWSCloudConfig(t *testing.T) {
	tests := []struct {
		name string

		reader io.Reader
		aws    Services

		expectError bool
		zone        string
	}{
		{
			"No config reader",
			nil, nil,
			true, "",
		},
		{
			"Empty config, no metadata",
			strings.NewReader(""), nil,
			true, "",
		},
		{
			"No zone in config, no metadata",
			strings.NewReader("[global]\n"), nil,
			true, "",
		},
		{
			"Zone in config, no metadata",
			strings.NewReader("[global]\nzone = eu-west-1a"), nil,
			false, "eu-west-1a",
		},
		{
			"No zone in config, metadata does not have zone",
			strings.NewReader("[global]\n"), newMockedFakeAWSServices(TestClusterID).WithAz(""),
			true, "",
		},
		{
			"No zone in config, metadata has zone",
			strings.NewReader("[global]\n"), newMockedFakeAWSServices(TestClusterID),
			false, "us-east-1a",
		},
		{
			"Zone in config should take precedence over metadata",
			strings.NewReader("[global]\nzone = eu-west-1a"), newMockedFakeAWSServices(TestClusterID),
			false, "eu-west-1a",
		},
	}

	for _, test := range tests {
		t.Logf("Running test case %s", test.name)
		var metadata EC2Metadata
		if test.aws != nil {
			metadata, _ = test.aws.Metadata()
		}
		cfg, err := readAWSCloudConfig(test.reader)
		if err == nil {
			err = updateConfigZone(cfg, metadata)
		}
		if test.expectError {
			if err == nil {
				t.Errorf("Should error for case %s (cfg=%v)", test.name, cfg)
			}
		} else {
			if err != nil {
				t.Errorf("Should succeed for case: %s", test.name)
			}
			if cfg.Global.Zone != test.zone {
				t.Errorf("Incorrect zone value (%s vs %s) for case: %s",
					cfg.Global.Zone, test.zone, test.name)
			}
		}
	}
}

type ServiceDescriptor struct {
	name                         string
	region                       string
	signingRegion, signingMethod string
	signingName                  string
}

func TestOverridesActiveConfig(t *testing.T) {
	tests := []struct {
		name string

		reader io.Reader
		aws    Services

		expectError        bool
		active             bool
		servicesOverridden []ServiceDescriptor
	}{
		{
			"No overrides",
			strings.NewReader(`
				[global]
				`),
			nil,
			false, false,
			[]ServiceDescriptor{},
		},
		{
			"Missing Service Name",
			strings.NewReader(`
                [global]

                [ServiceOverride "1"]
                 Region=sregion
                 URL=https://s3.foo.bar
                 SigningRegion=sregion
                 SigningMethod = sign
                `),
			nil,
			true, false,
			[]ServiceDescriptor{},
		},
		{
			"Missing Service Region",
			strings.NewReader(`
                [global]

                [ServiceOverride "1"]
                 Service=s3
                 URL=https://s3.foo.bar
                 SigningRegion=sregion
                 SigningMethod = sign
                 `),
			nil,
			true, false,
			[]ServiceDescriptor{},
		},
		{
			"Missing URL",
			strings.NewReader(`
                  [global]

                  [ServiceOverride "1"]
                   Service="s3"
                   Region=sregion
                   SigningRegion=sregion
                   SigningMethod = sign
                  `),
			nil,
			true, false,
			[]ServiceDescriptor{},
		},
		{
			"Missing Signing Region",
			strings.NewReader(`
                [global]

                [ServiceOverride "1"]
                 Service=s3
                 Region=sregion
                 URL=https://s3.foo.bar
                 SigningMethod = sign
                 `),
			nil,
			true, false,
			[]ServiceDescriptor{},
		},
		{
			"Active Overrides",
			strings.NewReader(`
                [Global]

               [ServiceOverride "1"]
                Service = "s3      "
                Region = sregion
                URL = https://s3.foo.bar
                SigningRegion = sregion
                SigningMethod = v4
                `),
			nil,
			false, true,
			[]ServiceDescriptor{{name: "s3", region: "sregion", signingRegion: "sregion", signingMethod: "v4"}},
		},
		{
			"Multiple Overridden Services",
			strings.NewReader(`
                [Global]
                 vpc = vpc-abc1234567

				[ServiceOverride "1"]
                  Service=s3
                  Region=sregion1
                  URL=https://s3.foo.bar
                  SigningRegion=sregion1
                  SigningMethod = v4

				[ServiceOverride "2"]
                  Service=ec2
                  Region=sregion2
                  URL=https://ec2.foo.bar
                  SigningRegion=sregion2
                  SigningMethod = v4`),
			nil,
			false, true,
			[]ServiceDescriptor{{name: "s3", region: "sregion1", signingRegion: "sregion1", signingMethod: "v4"},
				{name: "ec2", region: "sregion2", signingRegion: "sregion2", signingMethod: "v4"}},
		},
		{
			"Duplicate Services",
			strings.NewReader(`
                [Global]
                 vpc = vpc-abc1234567

				[ServiceOverride "1"]
                  Service=s3
                  Region=sregion1
                  URL=https://s3.foo.bar
                  SigningRegion=sregion
                  SigningMethod = sign

				[ServiceOverride "2"]
                  Service=s3
                  Region=sregion1
                  URL=https://s3.foo.bar
                  SigningRegion=sregion
                  SigningMethod = sign`),
			nil,
			true, false,
			[]ServiceDescriptor{},
		},
		{
			"Multiple Overridden Services in Multiple regions",
			strings.NewReader(`
                 [global]

				[ServiceOverride "1"]
                 Service=s3
                 Region=region1
                 URL=https://s3.foo.bar
                 SigningRegion=sregion1

				[ServiceOverride "2"]
                 Service=ec2
                 Region=region2
                 URL=https://ec2.foo.bar
                 SigningRegion=sregion
                 SigningMethod = v4
                 `),
			nil,
			false, true,
			[]ServiceDescriptor{{name: "s3", region: "region1", signingRegion: "sregion1", signingMethod: ""},
				{name: "ec2", region: "region2", signingRegion: "sregion", signingMethod: "v4"}},
		},
		{
			"Multiple regions, Same Service",
			strings.NewReader(`
                 [global]

				[ServiceOverride "1"]
                Service=s3
                Region=region1
                URL=https://s3.foo.bar
                SigningRegion=sregion1
                SigningMethod = v3

				[ServiceOverride "2"]
                 Service=s3
                 Region=region2
                 URL=https://s3.foo.bar
                 SigningRegion=sregion1
				 SigningMethod = v4
                 SigningName = "name"
                 `),
			nil,
			false, true,
			[]ServiceDescriptor{{name: "s3", region: "region1", signingRegion: "sregion1", signingMethod: "v3"},
				{name: "s3", region: "region2", signingRegion: "sregion1", signingMethod: "v4", signingName: "name"}},
		},
	}

	for _, test := range tests {
		t.Logf("Running test case %s", test.name)
		cfg, err := readAWSCloudConfig(test.reader)
		if err == nil {
			err = cfg.validateOverrides()
		}
		if test.expectError {
			if err == nil {
				t.Errorf("Should error for case %s (cfg=%v)", test.name, cfg)
			}
		} else {
			if err != nil {
				t.Errorf("Should succeed for case: %s, got %v", test.name, err)
			}

			if len(cfg.ServiceOverride) != len(test.servicesOverridden) {
				t.Errorf("Expected %d overridden services, received %d for case %s",
					len(test.servicesOverridden), len(cfg.ServiceOverride), test.name)
			} else {
				for _, sd := range test.servicesOverridden {
					var found *struct {
						Service       string
						Region        string
						URL           string
						SigningRegion string
						SigningMethod string
						SigningName   string
					}
					for _, v := range cfg.ServiceOverride {
						if v.Service == sd.name && v.Region == sd.region {
							found = v
							break
						}
					}
					if found == nil {
						t.Errorf("Missing override for service %s in case %s",
							sd.name, test.name)
					} else {
						if found.SigningRegion != sd.signingRegion {
							t.Errorf("Expected signing region '%s', received '%s' for case %s",
								sd.signingRegion, found.SigningRegion, test.name)
						}
						if found.SigningMethod != sd.signingMethod {
							t.Errorf("Expected signing method '%s', received '%s' for case %s",
								sd.signingMethod, found.SigningRegion, test.name)
						}
						targetName := fmt.Sprintf("https://%s.foo.bar", sd.name)
						if found.URL != targetName {
							t.Errorf("Expected Endpoint '%s', received '%s' for case %s",
								targetName, found.URL, test.name)
						}
						if found.SigningName != sd.signingName {
							t.Errorf("Expected signing name '%s', received '%s' for case %s",
								sd.signingName, found.SigningName, test.name)
						}

						fn := cfg.getResolver()
						ep1, e := fn(sd.name, sd.region, nil)
						if e != nil {
							t.Errorf("Expected a valid endpoint for %s in case %s",
								sd.name, test.name)
						} else {
							targetName := fmt.Sprintf("https://%s.foo.bar", sd.name)
							if ep1.URL != targetName {
								t.Errorf("Expected endpoint url: %s, received %s in case %s",
									targetName, ep1.URL, test.name)
							}
							if ep1.SigningRegion != sd.signingRegion {
								t.Errorf("Expected signing region '%s', received '%s' in case %s",
									sd.signingRegion, ep1.SigningRegion, test.name)
							}
							if ep1.SigningMethod != sd.signingMethod {
								t.Errorf("Expected signing method '%s', received '%s' in case %s",
									sd.signingMethod, ep1.SigningRegion, test.name)
							}
						}
					}
				}
			}
		}
	}
}

func TestNewAWSCloud(t *testing.T) {
	tests := []struct {
		name string

		reader      io.Reader
		awsServices Services

		expectError bool
		region      string
	}{
		{
			"No config reader",
			nil, newMockedFakeAWSServices(TestClusterID).WithAz(""),
			true, "",
		},
		{
			"Config specifies valid zone",
			strings.NewReader("[global]\nzone = eu-west-1a"), newMockedFakeAWSServices(TestClusterID),
			false, "eu-west-1",
		},
		{
			"Gets zone from metadata when not in config",
			strings.NewReader("[global]\n"),
			newMockedFakeAWSServices(TestClusterID),
			false, "us-east-1",
		},
		{
			"No zone in config or metadata",
			strings.NewReader("[global]\n"),
			newMockedFakeAWSServices(TestClusterID).WithAz(""),
			true, "",
		},
	}

	for _, test := range tests {
		t.Logf("Running test case %s", test.name)
		cfg, err := readAWSCloudConfig(test.reader)
		var c *Cloud
		if err == nil {
			c, err = newAWSCloud(*cfg, test.awsServices)
		}
		if test.expectError {
			if err == nil {
				t.Errorf("Should error for case %s", test.name)
			}
		} else {
			if err != nil {
				t.Errorf("Should succeed for case: %s, got %v", test.name, err)
			} else if c.region != test.region {
				t.Errorf("Incorrect region value (%s vs %s) for case: %s",
					c.region, test.region, test.name)
			}
		}
	}
}

func mockInstancesResp(selfInstance *ec2.Instance, instances []*ec2.Instance) (*Cloud, *FakeAWSServices) {
	awsServices := newMockedFakeAWSServices(TestClusterID)
	awsServices.instances = instances
	awsServices.selfInstance = selfInstance
	awsCloud, err := newAWSCloud(CloudConfig{}, awsServices)
	if err != nil {
		panic(err)
	}
	awsCloud.kubeClient = fake.NewSimpleClientset()
	return awsCloud, awsServices
}

func mockAvailabilityZone(availabilityZone string) *Cloud {
	awsServices := newMockedFakeAWSServices(TestClusterID).WithAz(availabilityZone)
	awsCloud, err := newAWSCloud(CloudConfig{}, awsServices)
	if err != nil {
		panic(err)
	}
	awsCloud.kubeClient = fake.NewSimpleClientset()
	return awsCloud
}

func testHasNodeAddress(t *testing.T, addrs []v1.NodeAddress, addressType v1.NodeAddressType, address string) {
	for _, addr := range addrs {
		if addr.Type == addressType && addr.Address == address {
			return
		}
	}
	t.Errorf("Did not find expected address: %s:%s in %v", addressType, address, addrs)
}

func makeInstance(num int, privateIP, publicIP, privateDNSName, publicDNSName string, setNetInterface bool) ec2.Instance {
	var tag ec2.Tag
	tag.Key = aws.String(TagNameKubernetesClusterLegacy)
	tag.Value = aws.String(TestClusterID)
	tags := []*ec2.Tag{&tag}

	instance := ec2.Instance{
		InstanceId:       aws.String(fmt.Sprintf("i-%d", num)),
		PrivateDnsName:   aws.String(privateDNSName),
		PrivateIpAddress: aws.String(privateIP),
		PublicDnsName:    aws.String(publicDNSName),
		PublicIpAddress:  aws.String(publicIP),
		InstanceType:     aws.String("c3.large"),
		Tags:             tags,
		Placement:        &ec2.Placement{AvailabilityZone: aws.String("us-east-1a")},
		State: &ec2.InstanceState{
			Name: aws.String("running"),
		},
	}
	if setNetInterface == true {
		instance.NetworkInterfaces = []*ec2.InstanceNetworkInterface{
			{
				Status: aws.String(ec2.NetworkInterfaceStatusInUse),
				PrivateIpAddresses: []*ec2.InstancePrivateIpAddress{
					{
						PrivateIpAddress: aws.String(privateIP),
					},
				},
			},
		}
	}
	return instance
}

func TestNodeAddresses(t *testing.T) {
	// Note instance0 and instance1 have the same name
	// (we test that this produces an error)
	instance0 := makeInstance(0, "192.168.0.1", "1.2.3.4", "instance-same.ec2.internal", "instance-same.ec2.external", true)
	instance1 := makeInstance(1, "192.168.0.2", "", "instance-same.ec2.internal", "", false)
	instance2 := makeInstance(2, "192.168.0.1", "1.2.3.4", "instance-other.ec2.internal", "", false)
	instances := []*ec2.Instance{&instance0, &instance1, &instance2}

	aws1, _ := mockInstancesResp(&instance0, []*ec2.Instance{&instance0})
	_, err1 := aws1.NodeAddresses(context.TODO(), "instance-mismatch.ec2.internal")
	if err1 == nil {
		t.Errorf("Should error when no instance found")
	}

	aws2, _ := mockInstancesResp(&instance2, instances)
	_, err2 := aws2.NodeAddresses(context.TODO(), "instance-same.ec2.internal")
	if err2 == nil {
		t.Errorf("Should error when multiple instances found")
	}

	aws3, _ := mockInstancesResp(&instance0, instances[0:1])
	// change node name so it uses the instance instead of metadata
	aws3.selfAWSInstance.nodeName = "foo"
	addrs3, err3 := aws3.NodeAddresses(context.TODO(), "instance-same.ec2.internal")
	if err3 != nil {
		t.Errorf("Should not error when instance found")
	}
	if len(addrs3) != 5 {
		t.Errorf("Should return exactly 5 NodeAddresses")
	}
	testHasNodeAddress(t, addrs3, v1.NodeInternalIP, "192.168.0.1")
	testHasNodeAddress(t, addrs3, v1.NodeExternalIP, "1.2.3.4")
	testHasNodeAddress(t, addrs3, v1.NodeExternalDNS, "instance-same.ec2.external")
	testHasNodeAddress(t, addrs3, v1.NodeInternalDNS, "instance-same.ec2.internal")
	testHasNodeAddress(t, addrs3, v1.NodeHostName, "instance-same.ec2.internal")
}

func TestNodeAddressesWithMetadata(t *testing.T) {
	instance := makeInstance(0, "", "2.3.4.5", "instance.ec2.internal", "", false)
	instances := []*ec2.Instance{&instance}
	awsCloud, awsServices := mockInstancesResp(&instance, instances)

	awsServices.networkInterfacesMacs = []string{"0a:77:89:f3:9c:f6", "0a:26:64:c4:6a:48"}
	awsServices.networkInterfacesPrivateIPs = [][]string{{"192.168.0.1"}, {"192.168.0.2"}}
	addrs, err := awsCloud.NodeAddresses(context.TODO(), "")
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	testHasNodeAddress(t, addrs, v1.NodeInternalIP, "192.168.0.1")
	testHasNodeAddress(t, addrs, v1.NodeInternalIP, "192.168.0.2")
	testHasNodeAddress(t, addrs, v1.NodeExternalIP, "2.3.4.5")
	var index1, index2 int
	for i, addr := range addrs {
		if addr.Type == v1.NodeInternalIP && addr.Address == "192.168.0.1" {
			index1 = i
		} else if addr.Type == v1.NodeInternalIP && addr.Address == "192.168.0.2" {
			index2 = i
		}
	}
	if index1 > index2 {
		t.Errorf("Addresses in incorrect order: %v", addrs)
	}
}

func TestParseMetadataLocalHostname(t *testing.T) {
	tests := []struct {
		name        string
		metadata    string
		hostname    string
		internalDNS []string
	}{
		{
			"single hostname",
			"ip-172-31-16-168.us-west-2.compute.internal",
			"ip-172-31-16-168.us-west-2.compute.internal",
			[]string{"ip-172-31-16-168.us-west-2.compute.internal"},
		},
		{
			"dhcp options set with three additional domain names",
			"ip-172-31-16-168.us-west-2.compute.internal example.com example.ca example.org",
			"ip-172-31-16-168.us-west-2.compute.internal",
			[]string{"ip-172-31-16-168.us-west-2.compute.internal", "ip-172-31-16-168.example.com", "ip-172-31-16-168.example.ca", "ip-172-31-16-168.example.org"},
		},
	}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			hostname, internalDNS := parseMetadataLocalHostname(test.metadata)
			if hostname != test.hostname {
				t.Errorf("got hostname %v, expected %v", hostname, test.hostname)
			}
			for i, v := range internalDNS {
				if v != test.internalDNS[i] {
					t.Errorf("got an internalDNS %v, expected %v", v, test.internalDNS[i])
				}
			}
		})
	}
}

func TestGetRegion(t *testing.T) {
	aws := mockAvailabilityZone("us-west-2e")
	zones, ok := aws.Zones()
	if !ok {
		t.Fatalf("Unexpected missing zones impl")
	}
	zone, err := zones.GetZone(context.TODO())
	if err != nil {
		t.Fatalf("unexpected error %v", err)
	}
	if zone.Region != "us-west-2" {
		t.Errorf("Unexpected region: %s", zone.Region)
	}
	if zone.FailureDomain != "us-west-2e" {
		t.Errorf("Unexpected FailureDomain: %s", zone.FailureDomain)
	}
}

func TestFindVPCID(t *testing.T) {
	awsServices := newMockedFakeAWSServices(TestClusterID)
	c, err := newAWSCloud(CloudConfig{}, awsServices)
	if err != nil {
		t.Errorf("Error building aws cloud: %v", err)
		return
	}
	vpcID, err := c.findVPCID()
	if err != nil {
		t.Errorf("Unexpected error: %v", err)
	}
	if vpcID != "vpc-mac0" {
		t.Errorf("Unexpected vpcID: %s", vpcID)
	}
}

func constructSubnets(subnetsIn map[int]map[string]string) (subnetsOut []*ec2.Subnet) {
	for i := range subnetsIn {
		subnetsOut = append(
			subnetsOut,
			constructSubnet(
				subnetsIn[i]["id"],
				subnetsIn[i]["az"],
			),
		)
	}
	return
}

func constructSubnet(id string, az string) *ec2.Subnet {
	return &ec2.Subnet{
		SubnetId:         &id,
		AvailabilityZone: &az,
	}
}

func constructRouteTables(routeTablesIn map[string]bool) (routeTablesOut []*ec2.RouteTable) {
	routeTablesOut = append(routeTablesOut,
		&ec2.RouteTable{
			Associations: []*ec2.RouteTableAssociation{{Main: aws.Bool(true)}},
			Routes: []*ec2.Route{{
				DestinationCidrBlock: aws.String("0.0.0.0/0"),
				GatewayId:            aws.String("igw-main"),
			}},
		})

	for subnetID := range routeTablesIn {
		routeTablesOut = append(
			routeTablesOut,
			constructRouteTable(
				subnetID,
				routeTablesIn[subnetID],
			),
		)
	}
	return
}

func constructRouteTable(subnetID string, public bool) *ec2.RouteTable {
	var gatewayID string
	if public {
		gatewayID = "igw-" + subnetID[len(subnetID)-8:8]
	} else {
		gatewayID = "vgw-" + subnetID[len(subnetID)-8:8]
	}
	return &ec2.RouteTable{
		Associations: []*ec2.RouteTableAssociation{{SubnetId: aws.String(subnetID)}},
		Routes: []*ec2.Route{{
			DestinationCidrBlock: aws.String("0.0.0.0/0"),
			GatewayId:            aws.String(gatewayID),
		}},
	}
}

func Test_findELBSubnets(t *testing.T) {
	awsServices := newMockedFakeAWSServices(TestClusterID)
	c, err := newAWSCloud(CloudConfig{}, awsServices)
	if err != nil {
		t.Errorf("Error building aws cloud: %v", err)
		return
	}
	subnetA0000001 := &ec2.Subnet{
		AvailabilityZone: aws.String("us-west-2a"),
		SubnetId:         aws.String("subnet-a0000001"),
		Tags: []*ec2.Tag{
			{
				Key:   aws.String(TagNameSubnetPublicELB),
				Value: aws.String("1"),
			},
		},
	}
	subnetA0000002 := &ec2.Subnet{
		AvailabilityZone: aws.String("us-west-2a"),
		SubnetId:         aws.String("subnet-a0000002"),
		Tags: []*ec2.Tag{
			{
				Key:   aws.String(TagNameSubnetPublicELB),
				Value: aws.String("1"),
			},
		},
	}
	subnetA0000003 := &ec2.Subnet{
		AvailabilityZone: aws.String("us-west-2a"),
		SubnetId:         aws.String("subnet-a0000003"),
		Tags: []*ec2.Tag{
			{
				Key:   aws.String(c.tagging.clusterTagKey()),
				Value: aws.String("owned"),
			},
			{
				Key:   aws.String(TagNameSubnetInternalELB),
				Value: aws.String("1"),
			},
		},
	}
	subnetB0000001 := &ec2.Subnet{
		AvailabilityZone: aws.String("us-west-2b"),
		SubnetId:         aws.String("subnet-b0000001"),
		Tags: []*ec2.Tag{
			{
				Key:   aws.String(c.tagging.clusterTagKey()),
				Value: aws.String("owned"),
			},
			{
				Key:   aws.String(TagNameSubnetPublicELB),
				Value: aws.String("1"),
			},
		},
	}
	subnetB0000002 := &ec2.Subnet{
		AvailabilityZone: aws.String("us-west-2b"),
		SubnetId:         aws.String("subnet-b0000002"),
		Tags: []*ec2.Tag{
			{
				Key:   aws.String(c.tagging.clusterTagKey()),
				Value: aws.String("owned"),
			},
			{
				Key:   aws.String(TagNameSubnetInternalELB),
				Value: aws.String("1"),
			},
		},
	}
	subnetC0000001 := &ec2.Subnet{
		AvailabilityZone: aws.String("us-west-2c"),
		SubnetId:         aws.String("subnet-c0000001"),
		Tags: []*ec2.Tag{
			{
				Key:   aws.String(c.tagging.clusterTagKey()),
				Value: aws.String("owned"),
			},
			{
				Key:   aws.String(TagNameSubnetInternalELB),
				Value: aws.String("1"),
			},
		},
	}
	subnetOther := &ec2.Subnet{
		AvailabilityZone: aws.String("us-west-2c"),
		SubnetId:         aws.String("subnet-other"),
		Tags: []*ec2.Tag{
			{
				Key:   aws.String(TagNameKubernetesClusterPrefix + "clusterid.other"),
				Value: aws.String("owned"),
			},
			{
				Key:   aws.String(TagNameSubnetInternalELB),
				Value: aws.String("1"),
			},
		},
	}
	subnetNoTag := &ec2.Subnet{
		AvailabilityZone: aws.String("us-west-2c"),
		SubnetId:         aws.String("subnet-notag"),
	}

	tests := []struct {
		name        string
		subnets     []*ec2.Subnet
		routeTables map[string]bool
		internal    bool
		want        []string
	}{
		{
			name: "no subnets",
		},
		{
			name: "single tagged subnet",
			subnets: []*ec2.Subnet{
				subnetA0000001,
			},
			routeTables: map[string]bool{
				"subnet-a0000001": true,
			},
			internal: false,
			want:     []string{"subnet-a0000001"},
		},
		{
			name: "no matching public subnet",
			subnets: []*ec2.Subnet{
				subnetA0000002,
			},
			routeTables: map[string]bool{
				"subnet-a0000002": false,
			},
			want: nil,
		},
		{
			name: "prefer role over cluster tag",
			subnets: []*ec2.Subnet{
				subnetA0000001,
				subnetA0000003,
			},
			routeTables: map[string]bool{
				"subnet-a0000001": true,
				"subnet-a0000003": true,
			},
			want: []string{"subnet-a0000001"},
		},
		{
			name: "prefer cluster tag",
			subnets: []*ec2.Subnet{
				subnetC0000001,
				subnetNoTag,
			},
			want: []string{"subnet-c0000001"},
		},
		{
			name: "include untagged",
			subnets: []*ec2.Subnet{
				subnetA0000001,
				subnetNoTag,
			},
			routeTables: map[string]bool{
				"subnet-a0000001": true,
				"subnet-notag":    true,
			},
			want: []string{"subnet-a0000001", "subnet-notag"},
		},
		{
			name: "ignore some other cluster owned subnet",
			subnets: []*ec2.Subnet{
				subnetB0000001,
				subnetOther,
			},
			routeTables: map[string]bool{
				"subnet-b0000001": true,
				"subnet-other":    true,
			},
			want: []string{"subnet-b0000001"},
		},
		{
			name: "prefer matching role",
			subnets: []*ec2.Subnet{
				subnetB0000001,
				subnetB0000002,
			},
			routeTables: map[string]bool{
				"subnet-b0000001": false,
				"subnet-b0000002": false,
			},
			want:     []string{"subnet-b0000002"},
			internal: true,
		},
		{
			name: "choose lexicographic order",
			subnets: []*ec2.Subnet{
				subnetA0000001,
				subnetA0000002,
			},
			routeTables: map[string]bool{
				"subnet-a0000001": true,
				"subnet-a0000002": true,
			},
			want: []string{"subnet-a0000001"},
		},
		{
			name: "everything",
			subnets: []*ec2.Subnet{
				subnetA0000001,
				subnetA0000002,
				subnetB0000001,
				subnetB0000002,
				subnetC0000001,
				subnetNoTag,
				subnetOther,
			},
			routeTables: map[string]bool{
				"subnet-a0000001": true,
				"subnet-a0000002": true,
				"subnet-b0000001": true,
				"subnet-b0000002": true,
				"subnet-c0000001": true,
				"subnet-notag":    true,
				"subnet-other":    true,
			},
			want: []string{"subnet-a0000001", "subnet-b0000001", "subnet-c0000001"},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			awsServices.ec2.RemoveSubnets()
			awsServices.ec2.RemoveRouteTables()
			for _, subnet := range tt.subnets {
				awsServices.ec2.CreateSubnet(subnet)
			}
			routeTables := constructRouteTables(tt.routeTables)
			for _, rt := range routeTables {
				awsServices.ec2.CreateRouteTable(rt)
			}
			got, _ := c.findELBSubnets(tt.internal)
			sort.Strings(tt.want)
			sort.Strings(got)
			assert.Equal(t, tt.want, got)
		})
	}
}

func Test_getLoadBalancerSubnets(t *testing.T) {
	awsServices := newMockedFakeAWSServices(TestClusterID)
	c, err := newAWSCloud(CloudConfig{}, awsServices)
	if err != nil {
		t.Errorf("Error building aws cloud: %v", err)
		return
	}
	tests := []struct {
		name        string
		service     *v1.Service
		subnets     []*ec2.Subnet
		internalELB bool
		want        []string
		wantErr     error
	}{
		{
			name:    "no annotation",
			service: &v1.Service{},
		},
		{
			name: "annotation with no subnets",
			service: &v1.Service{
				ObjectMeta: metav1.ObjectMeta{
					Annotations: map[string]string{
						"service.beta.kubernetes.io/aws-load-balancer-subnets": "\t",
					},
				},
			},
			wantErr: errors.New("unable to resolve empty subnet slice"),
		},
		{
			name: "subnet ids",
			subnets: []*ec2.Subnet{
				{
					AvailabilityZone: aws.String("us-west-2c"),
					SubnetId:         aws.String("subnet-a000001"),
				},
				{
					AvailabilityZone: aws.String("us-west-2b"),
					SubnetId:         aws.String("subnet-a000002"),
				},
			},
			service: &v1.Service{
				ObjectMeta: metav1.ObjectMeta{
					Annotations: map[string]string{
						"service.beta.kubernetes.io/aws-load-balancer-subnets": "subnet-a000001, subnet-a000002",
					},
				},
			},
			want: []string{"subnet-a000001", "subnet-a000002"},
		},
		{
			name: "subnet names",
			subnets: []*ec2.Subnet{
				{
					AvailabilityZone: aws.String("us-west-2c"),
					SubnetId:         aws.String("subnet-a000001"),
				},
				{
					AvailabilityZone: aws.String("us-west-2b"),
					SubnetId:         aws.String("subnet-a000002"),
				},
			},
			service: &v1.Service{
				ObjectMeta: metav1.ObjectMeta{
					Annotations: map[string]string{
						"service.beta.kubernetes.io/aws-load-balancer-subnets": "My Subnet 1, My Subnet 2 ",
					},
				},
			},
			want: []string{"subnet-a000001", "subnet-a000002"},
		},
		{
			name: "unable to find all subnets",
			subnets: []*ec2.Subnet{
				{
					AvailabilityZone: aws.String("us-west-2c"),
					SubnetId:         aws.String("subnet-a000001"),
				},
			},
			service: &v1.Service{
				ObjectMeta: metav1.ObjectMeta{
					Annotations: map[string]string{
						"service.beta.kubernetes.io/aws-load-balancer-subnets": "My Subnet 1, My Subnet 2, Test Subnet ",
					},
				},
			},
			wantErr: errors.New("expected to find 3, but found 1 subnets"),
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			awsServices.ec2.RemoveSubnets()
			for _, subnet := range tt.subnets {
				awsServices.ec2.CreateSubnet(subnet)
			}
			got, err := c.getLoadBalancerSubnets(tt.service, tt.internalELB)
			if tt.wantErr != nil {
				assert.EqualError(t, err, tt.wantErr.Error())
			} else {
				assert.Equal(t, tt.want, got)
			}
		})
	}
}

func TestSubnetIDsinVPC(t *testing.T) {
	awsServices := newMockedFakeAWSServices(TestClusterID)
	c, err := newAWSCloud(CloudConfig{}, awsServices)
	if err != nil {
		t.Errorf("Error building aws cloud: %v", err)
		return
	}

	// test with 3 subnets from 3 different AZs
	subnets := make(map[int]map[string]string)
	subnets[0] = make(map[string]string)
	subnets[0]["id"] = "subnet-a0000001"
	subnets[0]["az"] = "af-south-1a"
	subnets[1] = make(map[string]string)
	subnets[1]["id"] = "subnet-b0000001"
	subnets[1]["az"] = "af-south-1b"
	subnets[2] = make(map[string]string)
	subnets[2]["id"] = "subnet-c0000001"
	subnets[2]["az"] = "af-south-1c"
	constructedSubnets := constructSubnets(subnets)
	awsServices.ec2.RemoveSubnets()
	for _, subnet := range constructedSubnets {
		awsServices.ec2.CreateSubnet(subnet)
	}

	routeTables := map[string]bool{
		"subnet-a0000001": true,
		"subnet-b0000001": true,
		"subnet-c0000001": true,
	}
	constructedRouteTables := constructRouteTables(routeTables)
	awsServices.ec2.RemoveRouteTables()
	for _, rt := range constructedRouteTables {
		awsServices.ec2.CreateRouteTable(rt)
	}

	result, err := c.findELBSubnets(false)
	if err != nil {
		t.Errorf("Error listing subnets: %v", err)
		return
	}

	if len(result) != 3 {
		t.Errorf("Expected 3 subnets but got %d", len(result))
		return
	}

	resultSet := make(map[string]bool)
	for _, v := range result {
		resultSet[v] = true
	}

	for i := range subnets {
		if !resultSet[subnets[i]["id"]] {
			t.Errorf("Expected subnet%d '%s' in result: %v", i, subnets[i]["id"], result)
			return
		}
	}

	// test implicit routing table - when subnets are not explicitly linked to a table they should use main
	constructedRouteTables = constructRouteTables(map[string]bool{})
	awsServices.ec2.RemoveRouteTables()
	for _, rt := range constructedRouteTables {
		awsServices.ec2.CreateRouteTable(rt)
	}

	result, err = c.findELBSubnets(false)
	if err != nil {
		t.Errorf("Error listing subnets: %v", err)
		return
	}

	if len(result) != 3 {
		t.Errorf("Expected 3 subnets but got %d", len(result))
		return
	}

	resultSet = make(map[string]bool)
	for _, v := range result {
		resultSet[v] = true
	}

	for i := range subnets {
		if !resultSet[subnets[i]["id"]] {
			t.Errorf("Expected subnet%d '%s' in result: %v", i, subnets[i]["id"], result)
			return
		}
	}

	// Test with 5 subnets from 3 different AZs.
	// Add 2 duplicate AZ subnets lexicographically chosen one is the middle element in array to
	// check that we both choose the correct entry when it comes after and before another element
	// in the same AZ.
	subnets[3] = make(map[string]string)
	subnets[3]["id"] = "subnet-c0000000"
	subnets[3]["az"] = "af-south-1c"
	subnets[4] = make(map[string]string)
	subnets[4]["id"] = "subnet-c0000002"
	subnets[4]["az"] = "af-south-1c"
	constructedSubnets = constructSubnets(subnets)
	awsServices.ec2.RemoveSubnets()
	for _, subnet := range constructedSubnets {
		awsServices.ec2.CreateSubnet(subnet)
	}
	routeTables["subnet-c0000000"] = true
	routeTables["subnet-c0000002"] = true
	constructedRouteTables = constructRouteTables(routeTables)
	awsServices.ec2.RemoveRouteTables()
	for _, rt := range constructedRouteTables {
		awsServices.ec2.CreateRouteTable(rt)
	}

	result, err = c.findELBSubnets(false)
	if err != nil {
		t.Errorf("Error listing subnets: %v", err)
		return
	}

	if len(result) != 3 {
		t.Errorf("Expected 3 subnets but got %d", len(result))
		return
	}

	expected := []*string{aws.String("subnet-a0000001"), aws.String("subnet-b0000001"), aws.String("subnet-c0000000")}
	for _, s := range result {
		if !contains(expected, s) {
			t.Errorf("Unexpected subnet '%s' found", s)
			return
		}
	}

	delete(routeTables, "subnet-c0000002")

	// test with 6 subnets from 3 different AZs
	// with 3 private subnets
	subnets[4] = make(map[string]string)
	subnets[4]["id"] = "subnet-d0000001"
	subnets[4]["az"] = "af-south-1a"
	subnets[5] = make(map[string]string)
	subnets[5]["id"] = "subnet-d0000002"
	subnets[5]["az"] = "af-south-1b"

	constructedSubnets = constructSubnets(subnets)
	awsServices.ec2.RemoveSubnets()
	for _, subnet := range constructedSubnets {
		awsServices.ec2.CreateSubnet(subnet)
	}

	routeTables["subnet-a0000001"] = false
	routeTables["subnet-b0000001"] = false
	routeTables["subnet-c0000001"] = false
	routeTables["subnet-c0000000"] = true
	routeTables["subnet-d0000001"] = true
	routeTables["subnet-d0000002"] = true
	constructedRouteTables = constructRouteTables(routeTables)
	awsServices.ec2.RemoveRouteTables()
	for _, rt := range constructedRouteTables {
		awsServices.ec2.CreateRouteTable(rt)
	}
	result, err = c.findELBSubnets(false)
	if err != nil {
		t.Errorf("Error listing subnets: %v", err)
		return
	}

	if len(result) != 3 {
		t.Errorf("Expected 3 subnets but got %d", len(result))
		return
	}

	expected = []*string{aws.String("subnet-c0000000"), aws.String("subnet-d0000001"), aws.String("subnet-d0000002")}
	for _, s := range result {
		if !contains(expected, s) {
			t.Errorf("Unexpected subnet '%s' found", s)
			return
		}
	}
}

func TestIpPermissionExistsHandlesMultipleGroupIds(t *testing.T) {
	oldIPPermission := ec2.IpPermission{
		UserIdGroupPairs: []*ec2.UserIdGroupPair{
			{GroupId: aws.String("firstGroupId")},
			{GroupId: aws.String("secondGroupId")},
			{GroupId: aws.String("thirdGroupId")},
		},
	}

	existingIPPermission := ec2.IpPermission{
		UserIdGroupPairs: []*ec2.UserIdGroupPair{
			{GroupId: aws.String("secondGroupId")},
		},
	}

	newIPPermission := ec2.IpPermission{
		UserIdGroupPairs: []*ec2.UserIdGroupPair{
			{GroupId: aws.String("fourthGroupId")},
		},
	}

	equals := ipPermissionExists(&existingIPPermission, &oldIPPermission, false)
	if !equals {
		t.Errorf("Should have been considered equal since first is in the second array of groups")
	}

	equals = ipPermissionExists(&newIPPermission, &oldIPPermission, false)
	if equals {
		t.Errorf("Should have not been considered equal since first is not in the second array of groups")
	}

	// The first pair matches, but the second does not
	newIPPermission2 := ec2.IpPermission{
		UserIdGroupPairs: []*ec2.UserIdGroupPair{
			{GroupId: aws.String("firstGroupId")},
			{GroupId: aws.String("fourthGroupId")},
		},
	}
	equals = ipPermissionExists(&newIPPermission2, &oldIPPermission, false)
	if equals {
		t.Errorf("Should have not been considered equal since first is not in the second array of groups")
	}
}

func TestIpPermissionExistsHandlesRangeSubsets(t *testing.T) {
	// Two existing scenarios we'll test against
	emptyIPPermission := ec2.IpPermission{}

	oldIPPermission := ec2.IpPermission{
		IpRanges: []*ec2.IpRange{
			{CidrIp: aws.String("10.0.0.0/8")},
			{CidrIp: aws.String("192.168.1.0/24")},
		},
	}

	// Two already existing ranges and a new one
	existingIPPermission := ec2.IpPermission{
		IpRanges: []*ec2.IpRange{
			{CidrIp: aws.String("10.0.0.0/8")},
		},
	}
	existingIPPermission2 := ec2.IpPermission{
		IpRanges: []*ec2.IpRange{
			{CidrIp: aws.String("192.168.1.0/24")},
		},
	}

	newIPPermission := ec2.IpPermission{
		IpRanges: []*ec2.IpRange{
			{CidrIp: aws.String("172.16.0.0/16")},
		},
	}

	exists := ipPermissionExists(&emptyIPPermission, &emptyIPPermission, false)
	if !exists {
		t.Errorf("Should have been considered existing since we're comparing a range array against itself")
	}
	exists = ipPermissionExists(&oldIPPermission, &oldIPPermission, false)
	if !exists {
		t.Errorf("Should have been considered existing since we're comparing a range array against itself")
	}

	exists = ipPermissionExists(&existingIPPermission, &oldIPPermission, false)
	if !exists {
		t.Errorf("Should have been considered existing since 10.* is in oldIPPermission's array of ranges")
	}
	exists = ipPermissionExists(&existingIPPermission2, &oldIPPermission, false)
	if !exists {
		t.Errorf("Should have been considered existing since 192.* is in oldIpPermission2's array of ranges")
	}

	exists = ipPermissionExists(&newIPPermission, &emptyIPPermission, false)
	if exists {
		t.Errorf("Should have not been considered existing since we compared against a missing array of ranges")
	}
	exists = ipPermissionExists(&newIPPermission, &oldIPPermission, false)
	if exists {
		t.Errorf("Should have not been considered existing since 172.* is not in oldIPPermission's array of ranges")
	}
}

func TestIpPermissionExistsHandlesMultipleGroupIdsWithUserIds(t *testing.T) {
	oldIPPermission := ec2.IpPermission{
		UserIdGroupPairs: []*ec2.UserIdGroupPair{
			{GroupId: aws.String("firstGroupId"), UserId: aws.String("firstUserId")},
			{GroupId: aws.String("secondGroupId"), UserId: aws.String("secondUserId")},
			{GroupId: aws.String("thirdGroupId"), UserId: aws.String("thirdUserId")},
		},
	}

	existingIPPermission := ec2.IpPermission{
		UserIdGroupPairs: []*ec2.UserIdGroupPair{
			{GroupId: aws.String("secondGroupId"), UserId: aws.String("secondUserId")},
		},
	}

	newIPPermission := ec2.IpPermission{
		UserIdGroupPairs: []*ec2.UserIdGroupPair{
			{GroupId: aws.String("secondGroupId"), UserId: aws.String("anotherUserId")},
		},
	}

	equals := ipPermissionExists(&existingIPPermission, &oldIPPermission, true)
	if !equals {
		t.Errorf("Should have been considered equal since first is in the second array of groups")
	}

	equals = ipPermissionExists(&newIPPermission, &oldIPPermission, true)
	if equals {
		t.Errorf("Should have not been considered equal since first is not in the second array of groups")
	}
}

func TestFindInstanceByNodeNameExcludesTerminatedInstances(t *testing.T) {
	awsStates := []struct {
		id       int64
		state    string
		expected bool
	}{
		{0, ec2.InstanceStateNamePending, true},
		{16, ec2.InstanceStateNameRunning, true},
		{32, ec2.InstanceStateNameShuttingDown, true},
		{48, ec2.InstanceStateNameTerminated, false},
		{64, ec2.InstanceStateNameStopping, true},
		{80, ec2.InstanceStateNameStopped, true},
	}
	awsServices := newMockedFakeAWSServices(TestClusterID)

	nodeName := types.NodeName("my-dns.internal")

	var tag ec2.Tag
	tag.Key = aws.String(TagNameKubernetesClusterLegacy)
	tag.Value = aws.String(TestClusterID)
	tags := []*ec2.Tag{&tag}

	var testInstance ec2.Instance
	testInstance.PrivateDnsName = aws.String(string(nodeName))
	testInstance.Tags = tags

	awsDefaultInstances := awsServices.instances
	for _, awsState := range awsStates {
		id := "i-" + awsState.state
		testInstance.InstanceId = aws.String(id)
		testInstance.State = &ec2.InstanceState{Code: aws.Int64(awsState.id), Name: aws.String(awsState.state)}

		awsServices.instances = append(awsDefaultInstances, &testInstance)

		c, err := newAWSCloud(CloudConfig{}, awsServices)
		if err != nil {
			t.Errorf("Error building aws cloud: %v", err)
			return
		}

		resultInstance, err := c.findInstanceByNodeName(nodeName)

		if awsState.expected {
			if err != nil || resultInstance == nil {
				t.Errorf("Expected to find instance %v", *testInstance.InstanceId)
				return
			}
			if *resultInstance.InstanceId != *testInstance.InstanceId {
				t.Errorf("Wrong instance returned by findInstanceByNodeName() expected: %v, actual: %v", *testInstance.InstanceId, *resultInstance.InstanceId)
				return
			}
		} else {
			if err == nil && resultInstance != nil {
				t.Errorf("Did not expect to find instance %v", *resultInstance.InstanceId)
				return
			}
		}
	}
}

func TestGetInstanceByNodeNameBatching(t *testing.T) {
	awsServices := newMockedFakeAWSServices(TestClusterID)
	c, err := newAWSCloud(CloudConfig{}, awsServices)
	assert.Nil(t, err, "Error building aws cloud: %v", err)
	var tag ec2.Tag
	tag.Key = aws.String(TagNameKubernetesClusterPrefix + TestClusterID)
	tag.Value = aws.String("")
	tags := []*ec2.Tag{&tag}
	nodeNames := []string{}
	for i := 0; i < 200; i++ {
		nodeName := fmt.Sprintf("ip-171-20-42-%d.ec2.internal", i)
		nodeNames = append(nodeNames, nodeName)
		ec2Instance := &ec2.Instance{}
		instanceID := fmt.Sprintf("i-abcedf%d", i)
		ec2Instance.InstanceId = aws.String(instanceID)
		ec2Instance.PrivateDnsName = aws.String(nodeName)
		ec2Instance.State = &ec2.InstanceState{Code: aws.Int64(48), Name: aws.String("running")}
		ec2Instance.Tags = tags
		awsServices.instances = append(awsServices.instances, ec2Instance)

	}

	instances, err := c.getInstancesByNodeNames(nodeNames)
	assert.Nil(t, err, "Error getting instances by nodeNames %v: %v", nodeNames, err)
	assert.NotEmpty(t, instances)
	assert.Equal(t, 200, len(instances), "Expected 200 but got less")
}

func TestGetVolumeLabels(t *testing.T) {
	awsServices := newMockedFakeAWSServices(TestClusterID)
	c, err := newAWSCloud(CloudConfig{}, awsServices)
	assert.Nil(t, err, "Error building aws cloud: %v", err)
	volumeID := EBSVolumeID("vol-VolumeId")
	expectedVolumeRequest := &ec2.DescribeVolumesInput{VolumeIds: []*string{volumeID.awsString()}}
	awsServices.ec2.(*MockedFakeEC2).On("DescribeVolumes", expectedVolumeRequest).Return([]*ec2.Volume{
		{
			VolumeId:         volumeID.awsString(),
			AvailabilityZone: aws.String("us-east-1a"),
		},
	})

	labels, err := c.GetVolumeLabels(KubernetesVolumeID("aws:///" + string(volumeID)))

	assert.Nil(t, err, "Error creating Volume %v", err)
	assert.Equal(t, map[string]string{
		v1.LabelTopologyZone:   "us-east-1a",
		v1.LabelTopologyRegion: "us-east-1"}, labels)
	awsServices.ec2.(*MockedFakeEC2).AssertExpectations(t)
}

func TestGetLabelsForVolume(t *testing.T) {
	defaultVolume := EBSVolumeID("vol-VolumeId").awsString()
	tests := []struct {
		name               string
		pv                 *v1.PersistentVolume
		expectedVolumeID   *string
		expectedEC2Volumes []*ec2.Volume
		expectedLabels     map[string]string
		expectedError      error
	}{
		{
			"not an EBS volume",
			&v1.PersistentVolume{
				Spec: v1.PersistentVolumeSpec{},
			},
			nil,
			nil,
			nil,
			nil,
		},
		{
			"volume which is being provisioned",
			&v1.PersistentVolume{
				Spec: v1.PersistentVolumeSpec{
					PersistentVolumeSource: v1.PersistentVolumeSource{
						AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
							VolumeID: cloudvolume.ProvisionedVolumeName,
						},
					},
				},
			},
			nil,
			nil,
			nil,
			nil,
		},
		{
			"no volumes found",
			&v1.PersistentVolume{
				Spec: v1.PersistentVolumeSpec{
					PersistentVolumeSource: v1.PersistentVolumeSource{
						AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
							VolumeID: "vol-VolumeId",
						},
					},
				},
			},
			defaultVolume,
			nil,
			nil,
			fmt.Errorf("no volumes found"),
		},
		{
			"correct labels for volume",
			&v1.PersistentVolume{
				Spec: v1.PersistentVolumeSpec{
					PersistentVolumeSource: v1.PersistentVolumeSource{
						AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
							VolumeID: "vol-VolumeId",
						},
					},
				},
			},
			defaultVolume,
			[]*ec2.Volume{{
				VolumeId:         defaultVolume,
				AvailabilityZone: aws.String("us-east-1a"),
			}},
			map[string]string{
				v1.LabelTopologyZone:   "us-east-1a",
				v1.LabelTopologyRegion: "us-east-1",
			},
			nil,
		},
	}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			awsServices := newMockedFakeAWSServices(TestClusterID)
			expectedVolumeRequest := &ec2.DescribeVolumesInput{VolumeIds: []*string{test.expectedVolumeID}}
			awsServices.ec2.(*MockedFakeEC2).On("DescribeVolumes", expectedVolumeRequest).Return(test.expectedEC2Volumes)

			c, err := newAWSCloud(CloudConfig{}, awsServices)
			assert.Nil(t, err, "Error building aws cloud: %v", err)

			l, err := c.GetLabelsForVolume(context.TODO(), test.pv)
			assert.Equal(t, test.expectedLabels, l)
			assert.Equal(t, test.expectedError, err)
		})

	}
}

func TestDescribeLoadBalancerOnDelete(t *testing.T) {
	awsServices := newMockedFakeAWSServices(TestClusterID)
	c, _ := newAWSCloud(CloudConfig{}, awsServices)
	awsServices.elb.(*MockedFakeELB).expectDescribeLoadBalancers("aid")

	c.EnsureLoadBalancerDeleted(context.TODO(), TestClusterName, &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "myservice", UID: "id"}})
}

func TestDescribeLoadBalancerOnUpdate(t *testing.T) {
	awsServices := newMockedFakeAWSServices(TestClusterID)
	c, _ := newAWSCloud(CloudConfig{}, awsServices)
	awsServices.elb.(*MockedFakeELB).expectDescribeLoadBalancers("aid")

	c.UpdateLoadBalancer(context.TODO(), TestClusterName, &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "myservice", UID: "id"}}, []*v1.Node{})
}

func TestDescribeLoadBalancerOnGet(t *testing.T) {
	awsServices := newMockedFakeAWSServices(TestClusterID)
	c, _ := newAWSCloud(CloudConfig{}, awsServices)
	awsServices.elb.(*MockedFakeELB).expectDescribeLoadBalancers("aid")

	c.GetLoadBalancer(context.TODO(), TestClusterName, &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "myservice", UID: "id"}})
}

func TestDescribeLoadBalancerOnEnsure(t *testing.T) {
	awsServices := newMockedFakeAWSServices(TestClusterID)
	c, _ := newAWSCloud(CloudConfig{}, awsServices)
	awsServices.elb.(*MockedFakeELB).expectDescribeLoadBalancers("aid")

	c.EnsureLoadBalancer(context.TODO(), TestClusterName, &v1.Service{ObjectMeta: metav1.ObjectMeta{Name: "myservice", UID: "id"}}, []*v1.Node{})
}

func TestCheckProtocol(t *testing.T) {
	tests := []struct {
		name        string
		annotations map[string]string
		port        v1.ServicePort
		wantErr     error
	}{
		{
			name:        "TCP with ELB",
			annotations: make(map[string]string),
			port:        v1.ServicePort{Protocol: v1.ProtocolTCP, Port: int32(8080)},
			wantErr:     nil,
		},
		{
			name:        "TCP with NLB",
			annotations: map[string]string{ServiceAnnotationLoadBalancerType: "nlb"},
			port:        v1.ServicePort{Protocol: v1.ProtocolTCP, Port: int32(8080)},
			wantErr:     nil,
		},
		{
			name:        "UDP with ELB",
			annotations: make(map[string]string),
			port:        v1.ServicePort{Protocol: v1.ProtocolUDP, Port: int32(8080)},
			wantErr:     fmt.Errorf("Protocol UDP not supported by load balancer"),
		},
		{
			name:        "UDP with NLB",
			annotations: map[string]string{ServiceAnnotationLoadBalancerType: "nlb"},
			port:        v1.ServicePort{Protocol: v1.ProtocolUDP, Port: int32(8080)},
			wantErr:     nil,
		},
	}
	for _, test := range tests {
		tt := test
		t.Run(tt.name, func(t *testing.T) {
			t.Parallel()
			err := checkProtocol(tt.port, tt.annotations)
			if tt.wantErr != nil && err == nil {
				t.Errorf("Expected error: want=%s got =%s", tt.wantErr, err)
			}
			if tt.wantErr == nil && err != nil {
				t.Errorf("Unexpected error: want=%s got =%s", tt.wantErr, err)
			}
		})
	}
}

func TestBuildListener(t *testing.T) {
	tests := []struct {
		name string

		lbPort                    int64
		portName                  string
		instancePort              int64
		backendProtocolAnnotation string
		certAnnotation            string
		sslPortAnnotation         string

		expectError      bool
		lbProtocol       string
		instanceProtocol string
		certID           string
	}{
		{
			"No cert or BE protocol annotation, passthrough",
			80, "", 7999, "", "", "",
			false, "tcp", "tcp", "",
		},
		{
			"Cert annotation without BE protocol specified, SSL->TCP",
			80, "", 8000, "", "cert", "",
			false, "ssl", "tcp", "cert",
		},
		{
			"BE protocol without cert annotation, passthrough",
			443, "", 8001, "https", "", "",
			false, "tcp", "tcp", "",
		},
		{
			"Invalid cert annotation, bogus backend protocol",
			443, "", 8002, "bacon", "foo", "",
			true, "tcp", "tcp", "",
		},
		{
			"Invalid cert annotation, protocol followed by equal sign",
			443, "", 8003, "http=", "=", "",
			true, "tcp", "tcp", "",
		},
		{
			"HTTPS->HTTPS",
			443, "", 8004, "https", "cert", "",
			false, "https", "https", "cert",
		},
		{
			"HTTPS->HTTP",
			443, "", 8005, "http", "cert", "",
			false, "https", "http", "cert",
		},
		{
			"SSL->SSL",
			443, "", 8006, "ssl", "cert", "",
			false, "ssl", "ssl", "cert",
		},
		{
			"SSL->TCP",
			443, "", 8007, "tcp", "cert", "",
			false, "ssl", "tcp", "cert",
		},
		{
			"Port in whitelist",
			1234, "", 8008, "tcp", "cert", "1234,5678",
			false, "ssl", "tcp", "cert",
		},
		{
			"Port not in whitelist, passthrough",
			443, "", 8009, "tcp", "cert", "1234,5678",
			false, "tcp", "tcp", "",
		},
		{
			"Named port in whitelist",
			1234, "bar", 8010, "tcp", "cert", "foo,bar",
			false, "ssl", "tcp", "cert",
		},
		{
			"Named port not in whitelist, passthrough",
			443, "", 8011, "tcp", "cert", "foo,bar",
			false, "tcp", "tcp", "",
		},
		{
			"HTTP->HTTP",
			80, "", 8012, "http", "", "",
			false, "http", "http", "",
		},
	}

	for _, test := range tests {
		t.Logf("Running test case %s", test.name)
		annotations := make(map[string]string)
		if test.backendProtocolAnnotation != "" {
			annotations[ServiceAnnotationLoadBalancerBEProtocol] = test.backendProtocolAnnotation
		}
		if test.certAnnotation != "" {
			annotations[ServiceAnnotationLoadBalancerCertificate] = test.certAnnotation
		}
		ports := getPortSets(test.sslPortAnnotation)
		l, err := buildListener(v1.ServicePort{
			NodePort: int32(test.instancePort),
			Port:     int32(test.lbPort),
			Name:     test.portName,
			Protocol: v1.Protocol("tcp"),
		}, annotations, ports)
		if test.expectError {
			if err == nil {
				t.Errorf("Should error for case %s", test.name)
			}
		} else {
			if err != nil {
				t.Errorf("Should succeed for case: %s, got %v", test.name, err)
			} else {
				var cert *string
				if test.certID != "" {
					cert = &test.certID
				}
				expected := &elb.Listener{
					InstancePort:     &test.instancePort,
					InstanceProtocol: &test.instanceProtocol,
					LoadBalancerPort: &test.lbPort,
					Protocol:         &test.lbProtocol,
					SSLCertificateId: cert,
				}
				if !reflect.DeepEqual(l, expected) {
					t.Errorf("Incorrect listener (%v vs expected %v) for case: %s",
						l, expected, test.name)
				}
			}
		}
	}
}

func TestProxyProtocolEnabled(t *testing.T) {
	policies := sets.NewString(ProxyProtocolPolicyName, "FooBarFoo")
	fakeBackend := &elb.BackendServerDescription{
		InstancePort: aws.Int64(80),
		PolicyNames:  stringSetToPointers(policies),
	}
	result := proxyProtocolEnabled(fakeBackend)
	assert.True(t, result, "expected to find %s in %s", ProxyProtocolPolicyName, policies)

	policies = sets.NewString("FooBarFoo")
	fakeBackend = &elb.BackendServerDescription{
		InstancePort: aws.Int64(80),
		PolicyNames: []*string{
			aws.String("FooBarFoo"),
		},
	}
	result = proxyProtocolEnabled(fakeBackend)
	assert.False(t, result, "did not expect to find %s in %s", ProxyProtocolPolicyName, policies)

	policies = sets.NewString()
	fakeBackend = &elb.BackendServerDescription{
		InstancePort: aws.Int64(80),
	}
	result = proxyProtocolEnabled(fakeBackend)
	assert.False(t, result, "did not expect to find %s in %s", ProxyProtocolPolicyName, policies)
}

func TestGetKeyValuePropertiesFromAnnotation(t *testing.T) {
	tagTests := []struct {
		Annotations map[string]string
		Tags        map[string]string
	}{
		{
			Annotations: map[string]string{
				ServiceAnnotationLoadBalancerAdditionalTags: "Key=Val",
			},
			Tags: map[string]string{
				"Key": "Val",
			},
		},
		{
			Annotations: map[string]string{
				ServiceAnnotationLoadBalancerAdditionalTags: "Key1=Val1, Key2=Val2",
			},
			Tags: map[string]string{
				"Key1": "Val1",
				"Key2": "Val2",
			},
		},
		{
			Annotations: map[string]string{
				ServiceAnnotationLoadBalancerAdditionalTags: "Key1=, Key2=Val2",
				"anotherKey": "anotherValue",
			},
			Tags: map[string]string{
				"Key1": "",
				"Key2": "Val2",
			},
		},
		{
			Annotations: map[string]string{
				"Nothing": "Key1=, Key2=Val2, Key3",
			},
			Tags: map[string]string{},
		},
		{
			Annotations: map[string]string{
				ServiceAnnotationLoadBalancerAdditionalTags: "K=V K1=V2,Key1========, =====, ======Val, =Val, , 234,",
			},
			Tags: map[string]string{
				"K":    "V K1",
				"Key1": "",
				"234":  "",
			},
		},
	}

	for _, tagTest := range tagTests {
		result := getKeyValuePropertiesFromAnnotation(tagTest.Annotations, ServiceAnnotationLoadBalancerAdditionalTags)
		for k, v := range result {
			if len(result) != len(tagTest.Tags) {
				t.Errorf("incorrect expected length: %v != %v", result, tagTest.Tags)
				continue
			}
			if tagTest.Tags[k] != v {
				t.Errorf("%s != %s", tagTest.Tags[k], v)
				continue
			}
		}
	}
}

func TestLBExtraSecurityGroupsAnnotation(t *testing.T) {
	awsServices := newMockedFakeAWSServices(TestClusterID)
	c, _ := newAWSCloud(CloudConfig{}, awsServices)

	sg1 := map[string]string{ServiceAnnotationLoadBalancerExtraSecurityGroups: "sg-000001"}
	sg2 := map[string]string{ServiceAnnotationLoadBalancerExtraSecurityGroups: "sg-000002"}
	sg3 := map[string]string{ServiceAnnotationLoadBalancerExtraSecurityGroups: "sg-000001, sg-000002"}

	tests := []struct {
		name string

		annotations map[string]string
		expectedSGs []string
	}{
		{"No extra SG annotation", map[string]string{}, []string{}},
		{"Empty extra SGs specified", map[string]string{ServiceAnnotationLoadBalancerExtraSecurityGroups: ", ,,"}, []string{}},
		{"SG specified", sg1, []string{sg1[ServiceAnnotationLoadBalancerExtraSecurityGroups]}},
		{"Multiple SGs specified", sg3, []string{sg1[ServiceAnnotationLoadBalancerExtraSecurityGroups], sg2[ServiceAnnotationLoadBalancerExtraSecurityGroups]}},
	}

	awsServices.ec2.(*MockedFakeEC2).expectDescribeSecurityGroups(TestClusterID, "k8s-elb-aid")

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			serviceName := types.NamespacedName{Namespace: "default", Name: "myservice"}

			sgList, setupSg, err := c.buildELBSecurityGroupList(serviceName, "aid", test.annotations)
			assert.NoError(t, err, "buildELBSecurityGroupList failed")
			extraSGs := sgList[1:]
			assert.True(t, sets.NewString(test.expectedSGs...).Equal(sets.NewString(extraSGs...)),
				"Security Groups expected=%q , returned=%q", test.expectedSGs, extraSGs)
			assert.True(t, setupSg, "Security Groups Setup Permissions Flag expected=%t , returned=%t", true, setupSg)
		})
	}
}

func TestLBSecurityGroupsAnnotation(t *testing.T) {
	awsServices := newMockedFakeAWSServices(TestClusterID)
	c, _ := newAWSCloud(CloudConfig{}, awsServices)

	sg1 := map[string]string{ServiceAnnotationLoadBalancerSecurityGroups: "sg-000001"}
	sg2 := map[string]string{ServiceAnnotationLoadBalancerSecurityGroups: "sg-000002"}
	sg3 := map[string]string{ServiceAnnotationLoadBalancerSecurityGroups: "sg-000001, sg-000002"}

	tests := []struct {
		name string

		annotations map[string]string
		expectedSGs []string
	}{
		{"SG specified", sg1, []string{sg1[ServiceAnnotationLoadBalancerSecurityGroups]}},
		{"Multiple SGs specified", sg3, []string{sg1[ServiceAnnotationLoadBalancerSecurityGroups], sg2[ServiceAnnotationLoadBalancerSecurityGroups]}},
	}

	awsServices.ec2.(*MockedFakeEC2).expectDescribeSecurityGroups(TestClusterID, "k8s-elb-aid")

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			serviceName := types.NamespacedName{Namespace: "default", Name: "myservice"}

			sgList, setupSg, err := c.buildELBSecurityGroupList(serviceName, "aid", test.annotations)
			assert.NoError(t, err, "buildELBSecurityGroupList failed")
			assert.True(t, sets.NewString(test.expectedSGs...).Equal(sets.NewString(sgList...)),
				"Security Groups expected=%q , returned=%q", test.expectedSGs, sgList)
			assert.False(t, setupSg, "Security Groups Setup Permissions Flag expected=%t , returned=%t", false, setupSg)
		})
	}
}

// Test that we can add a load balancer tag
func TestAddLoadBalancerTags(t *testing.T) {
	loadBalancerName := "test-elb"
	awsServices := newMockedFakeAWSServices(TestClusterID)
	c, _ := newAWSCloud(CloudConfig{}, awsServices)

	want := make(map[string]string)
	want["tag1"] = "val1"

	expectedAddTagsRequest := &elb.AddTagsInput{
		LoadBalancerNames: []*string{&loadBalancerName},
		Tags: []*elb.Tag{
			{
				Key:   aws.String("tag1"),
				Value: aws.String("val1"),
			},
		},
	}
	awsServices.elb.(*MockedFakeELB).On("AddTags", expectedAddTagsRequest).Return(&elb.AddTagsOutput{})

	err := c.addLoadBalancerTags(loadBalancerName, want)
	assert.Nil(t, err, "Error adding load balancer tags: %v", err)
	awsServices.elb.(*MockedFakeELB).AssertExpectations(t)
}

func TestEnsureLoadBalancerHealthCheck(t *testing.T) {
	tests := []struct {
		name        string
		annotations map[string]string
		want        elb.HealthCheck
	}{
		{
			name:        "falls back to HC defaults",
			annotations: map[string]string{},
			want: elb.HealthCheck{
				HealthyThreshold:   aws.Int64(2),
				UnhealthyThreshold: aws.Int64(6),
				Timeout:            aws.Int64(5),
				Interval:           aws.Int64(10),
				Target:             aws.String("TCP:8080"),
			},
		},
		{
			name:        "healthy threshold override",
			annotations: map[string]string{ServiceAnnotationLoadBalancerHCHealthyThreshold: "7"},
			want: elb.HealthCheck{
				HealthyThreshold:   aws.Int64(7),
				UnhealthyThreshold: aws.Int64(6),
				Timeout:            aws.Int64(5),
				Interval:           aws.Int64(10),
				Target:             aws.String("TCP:8080"),
			},
		},
		{
			name:        "unhealthy threshold override",
			annotations: map[string]string{ServiceAnnotationLoadBalancerHCUnhealthyThreshold: "7"},
			want: elb.HealthCheck{
				HealthyThreshold:   aws.Int64(2),
				UnhealthyThreshold: aws.Int64(7),
				Timeout:            aws.Int64(5),
				Interval:           aws.Int64(10),
				Target:             aws.String("TCP:8080"),
			},
		},
		{
			name:        "timeout override",
			annotations: map[string]string{ServiceAnnotationLoadBalancerHCTimeout: "7"},
			want: elb.HealthCheck{
				HealthyThreshold:   aws.Int64(2),
				UnhealthyThreshold: aws.Int64(6),
				Timeout:            aws.Int64(7),
				Interval:           aws.Int64(10),
				Target:             aws.String("TCP:8080"),
			},
		},
		{
			name:        "interval override",
			annotations: map[string]string{ServiceAnnotationLoadBalancerHCInterval: "7"},
			want: elb.HealthCheck{
				HealthyThreshold:   aws.Int64(2),
				UnhealthyThreshold: aws.Int64(6),
				Timeout:            aws.Int64(5),
				Interval:           aws.Int64(7),
				Target:             aws.String("TCP:8080"),
			},
		},
		{
			name: "healthcheck port override",
			annotations: map[string]string{
				ServiceAnnotationLoadBalancerHealthCheckPort: "2122",
			},
			want: elb.HealthCheck{
				HealthyThreshold:   aws.Int64(2),
				UnhealthyThreshold: aws.Int64(6),
				Timeout:            aws.Int64(5),
				Interval:           aws.Int64(10),
				Target:             aws.String("TCP:2122"),
			},
		},
		{
			name: "healthcheck protocol override",
			annotations: map[string]string{
				ServiceAnnotationLoadBalancerHealthCheckProtocol: "HTTP",
			},
			want: elb.HealthCheck{
				HealthyThreshold:   aws.Int64(2),
				UnhealthyThreshold: aws.Int64(6),
				Timeout:            aws.Int64(5),
				Interval:           aws.Int64(10),
				Target:             aws.String("HTTP:8080/"),
			},
		},
		{
			name: "healthcheck protocol, port, path override",
			annotations: map[string]string{
				ServiceAnnotationLoadBalancerHealthCheckProtocol: "HTTPS",
				ServiceAnnotationLoadBalancerHealthCheckPath:     "/healthz",
				ServiceAnnotationLoadBalancerHealthCheckPort:     "31224",
			},
			want: elb.HealthCheck{
				HealthyThreshold:   aws.Int64(2),
				UnhealthyThreshold: aws.Int64(6),
				Timeout:            aws.Int64(5),
				Interval:           aws.Int64(10),
				Target:             aws.String("HTTPS:31224/healthz"),
			},
		},
		{
			name: "healthcheck protocol SSL",
			annotations: map[string]string{
				ServiceAnnotationLoadBalancerHealthCheckProtocol: "SSL",
				ServiceAnnotationLoadBalancerHealthCheckPath:     "/healthz",
				ServiceAnnotationLoadBalancerHealthCheckPort:     "3124",
			},
			want: elb.HealthCheck{
				HealthyThreshold:   aws.Int64(2),
				UnhealthyThreshold: aws.Int64(6),
				Timeout:            aws.Int64(5),
				Interval:           aws.Int64(10),
				Target:             aws.String("SSL:3124"),
			},
		},
		{
			name: "healthcheck port annotation traffic-port",
			annotations: map[string]string{
				ServiceAnnotationLoadBalancerHealthCheckProtocol: "TCP",
				ServiceAnnotationLoadBalancerHealthCheckPort:     "traffic-port",
			},
			want: elb.HealthCheck{
				HealthyThreshold:   aws.Int64(2),
				UnhealthyThreshold: aws.Int64(6),
				Timeout:            aws.Int64(5),
				Interval:           aws.Int64(10),
				Target:             aws.String("TCP:8080"),
			},
		},
	}
	lbName := "myLB"
	// this HC will always differ from the expected HC and thus it is expected an
	// API call will be made to update it
	currentHC := &elb.HealthCheck{}
	elbDesc := &elb.LoadBalancerDescription{LoadBalancerName: &lbName, HealthCheck: currentHC}
	defaultHealthyThreshold := int64(2)
	defaultUnhealthyThreshold := int64(6)
	defaultTimeout := int64(5)
	defaultInterval := int64(10)
	protocol, path, port := "TCP", "", int32(8080)
	target := "TCP:8080"
	defaultHC := &elb.HealthCheck{
		HealthyThreshold:   &defaultHealthyThreshold,
		UnhealthyThreshold: &defaultUnhealthyThreshold,
		Timeout:            &defaultTimeout,
		Interval:           &defaultInterval,
		Target:             &target,
	}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			awsServices := newMockedFakeAWSServices(TestClusterID)
			c, err := newAWSCloud(CloudConfig{}, awsServices)
			assert.Nil(t, err, "Error building aws cloud: %v", err)
			expectedHC := test.want
			awsServices.elb.(*MockedFakeELB).expectConfigureHealthCheck(&lbName, &expectedHC, nil)

			err = c.ensureLoadBalancerHealthCheck(elbDesc, protocol, port, path, test.annotations)

			require.NoError(t, err)
			awsServices.elb.(*MockedFakeELB).AssertExpectations(t)
		})
	}

	t.Run("does not make an API call if the current health check is the same", func(t *testing.T) {
		awsServices := newMockedFakeAWSServices(TestClusterID)
		c, err := newAWSCloud(CloudConfig{}, awsServices)
		assert.Nil(t, err, "Error building aws cloud: %v", err)
		expectedHC := *defaultHC
		timeout := int64(3)
		expectedHC.Timeout = &timeout
		annotations := map[string]string{ServiceAnnotationLoadBalancerHCTimeout: "3"}
		var currentHC elb.HealthCheck
		currentHC = expectedHC

		// NOTE no call expectations are set on the ELB mock
		// test default HC
		elbDesc := &elb.LoadBalancerDescription{LoadBalancerName: &lbName, HealthCheck: defaultHC}
		err = c.ensureLoadBalancerHealthCheck(elbDesc, protocol, port, path, map[string]string{})
		assert.NoError(t, err)
		// test HC with override
		elbDesc = &elb.LoadBalancerDescription{LoadBalancerName: &lbName, HealthCheck: &currentHC}
		err = c.ensureLoadBalancerHealthCheck(elbDesc, protocol, port, path, annotations)
		assert.NoError(t, err)
	})

	t.Run("validates resulting expected health check before making an API call", func(t *testing.T) {
		awsServices := newMockedFakeAWSServices(TestClusterID)
		c, err := newAWSCloud(CloudConfig{}, awsServices)
		assert.Nil(t, err, "Error building aws cloud: %v", err)
		expectedHC := *defaultHC
		invalidThreshold := int64(1)
		expectedHC.HealthyThreshold = &invalidThreshold
		require.Error(t, expectedHC.Validate()) // confirm test precondition
		annotations := map[string]string{ServiceAnnotationLoadBalancerHCTimeout: "1"}

		// NOTE no call expectations are set on the ELB mock
		err = c.ensureLoadBalancerHealthCheck(elbDesc, protocol, port, path, annotations)

		require.Error(t, err)
	})

	t.Run("handles invalid override values", func(t *testing.T) {
		awsServices := newMockedFakeAWSServices(TestClusterID)
		c, err := newAWSCloud(CloudConfig{}, awsServices)
		assert.Nil(t, err, "Error building aws cloud: %v", err)
		annotations := map[string]string{ServiceAnnotationLoadBalancerHCTimeout: "3.3"}

		// NOTE no call expectations are set on the ELB mock
		err = c.ensureLoadBalancerHealthCheck(elbDesc, protocol, port, path, annotations)

		require.Error(t, err)
	})

	t.Run("returns error when updating the health check fails", func(t *testing.T) {
		awsServices := newMockedFakeAWSServices(TestClusterID)
		c, err := newAWSCloud(CloudConfig{}, awsServices)
		assert.Nil(t, err, "Error building aws cloud: %v", err)
		returnErr := fmt.Errorf("throttling error")
		awsServices.elb.(*MockedFakeELB).expectConfigureHealthCheck(&lbName, defaultHC, returnErr)

		err = c.ensureLoadBalancerHealthCheck(elbDesc, protocol, port, path, map[string]string{})

		require.Error(t, err)
		awsServices.elb.(*MockedFakeELB).AssertExpectations(t)
	})
}

func TestFindSecurityGroupForInstance(t *testing.T) {
	groups := map[string]*ec2.SecurityGroup{"sg123": {GroupId: aws.String("sg123")}}
	id, err := findSecurityGroupForInstance(&ec2.Instance{SecurityGroups: []*ec2.GroupIdentifier{{GroupId: aws.String("sg123"), GroupName: aws.String("my_group")}}}, groups)
	if err != nil {
		t.Error()
	}
	assert.Equal(t, *id.GroupId, "sg123")
	assert.Equal(t, *id.GroupName, "my_group")
}

func TestFindSecurityGroupForInstanceMultipleTagged(t *testing.T) {
	groups := map[string]*ec2.SecurityGroup{"sg123": {GroupId: aws.String("sg123")}}
	_, err := findSecurityGroupForInstance(&ec2.Instance{
		SecurityGroups: []*ec2.GroupIdentifier{
			{GroupId: aws.String("sg123"), GroupName: aws.String("my_group")},
			{GroupId: aws.String("sg123"), GroupName: aws.String("another_group")},
		},
	}, groups)
	require.Error(t, err)
	assert.Contains(t, err.Error(), "sg123(my_group)")
	assert.Contains(t, err.Error(), "sg123(another_group)")
}

func TestCreateDisk(t *testing.T) {
	awsServices := newMockedFakeAWSServices(TestClusterID)
	c, _ := newAWSCloud(CloudConfig{}, awsServices)

	volumeOptions := &VolumeOptions{
		AvailabilityZone: "us-east-1a",
		CapacityGB:       10,
	}
	request := &ec2.CreateVolumeInput{
		AvailabilityZone: aws.String("us-east-1a"),
		Encrypted:        aws.Bool(false),
		VolumeType:       aws.String(DefaultVolumeType),
		Size:             aws.Int64(10),
		TagSpecifications: []*ec2.TagSpecification{
			{ResourceType: aws.String(ec2.ResourceTypeVolume), Tags: []*ec2.Tag{
				// CreateVolume from MockedFakeEC2 expects sorted tags, so we need to
				// always have these tags sorted:
				{Key: aws.String(TagNameKubernetesClusterLegacy), Value: aws.String(TestClusterID)},
				{Key: aws.String(fmt.Sprintf("%s%s", TagNameKubernetesClusterPrefix, TestClusterID)), Value: aws.String(ResourceLifecycleOwned)},
			}},
		},
	}

	volume := &ec2.Volume{
		AvailabilityZone: aws.String("us-east-1a"),
		VolumeId:         aws.String("vol-volumeId0"),
		State:            aws.String("available"),
	}
	awsServices.ec2.(*MockedFakeEC2).On("CreateVolume", request).Return(volume, nil)

	describeVolumesRequest := &ec2.DescribeVolumesInput{
		VolumeIds: []*string{aws.String("vol-volumeId0")},
	}
	awsServices.ec2.(*MockedFakeEC2).On("DescribeVolumes", describeVolumesRequest).Return([]*ec2.Volume{volume}, nil)

	volumeID, err := c.CreateDisk(volumeOptions)
	assert.Nil(t, err, "Error creating disk: %v", err)
	assert.Equal(t, volumeID, KubernetesVolumeID("aws://us-east-1a/vol-volumeId0"))
	awsServices.ec2.(*MockedFakeEC2).AssertExpectations(t)
}

func TestCreateDiskFailDescribeVolume(t *testing.T) {
	awsServices := newMockedFakeAWSServices(TestClusterID)
	c, _ := newAWSCloud(CloudConfig{}, awsServices)

	volumeOptions := &VolumeOptions{
		AvailabilityZone: "us-east-1a",
		CapacityGB:       10,
	}
	request := &ec2.CreateVolumeInput{
		AvailabilityZone: aws.String("us-east-1a"),
		Encrypted:        aws.Bool(false),
		VolumeType:       aws.String(DefaultVolumeType),
		Size:             aws.Int64(10),
		TagSpecifications: []*ec2.TagSpecification{
			{ResourceType: aws.String(ec2.ResourceTypeVolume), Tags: []*ec2.Tag{
				// CreateVolume from MockedFakeEC2 expects sorted tags, so we need to
				// always have these tags sorted:
				{Key: aws.String(TagNameKubernetesClusterLegacy), Value: aws.String(TestClusterID)},
				{Key: aws.String(fmt.Sprintf("%s%s", TagNameKubernetesClusterPrefix, TestClusterID)), Value: aws.String(ResourceLifecycleOwned)},
			}},
		},
	}

	volume := &ec2.Volume{
		AvailabilityZone: aws.String("us-east-1a"),
		VolumeId:         aws.String("vol-volumeId0"),
		State:            aws.String("creating"),
	}
	awsServices.ec2.(*MockedFakeEC2).On("CreateVolume", request).Return(volume, nil)

	describeVolumesRequest := &ec2.DescribeVolumesInput{
		VolumeIds: []*string{aws.String("vol-volumeId0")},
	}
	deleteVolumeRequest := &ec2.DeleteVolumeInput{
		VolumeId: aws.String("vol-volumeId0"),
	}
	awsServices.ec2.(*MockedFakeEC2).On("DescribeVolumes", describeVolumesRequest).Return([]*ec2.Volume{volume}, nil)
	awsServices.ec2.(*MockedFakeEC2).On("DeleteVolume", deleteVolumeRequest).Return(&ec2.DeleteVolumeOutput{}, nil)

	volumeID, err := c.CreateDisk(volumeOptions)
	assert.Error(t, err)
	assert.Equal(t, volumeID, KubernetesVolumeID(""))
	awsServices.ec2.(*MockedFakeEC2).AssertExpectations(t)
}

func TestRegionIsValid(t *testing.T) {
	fake := newMockedFakeAWSServices("fakeCluster")
	fake.selfInstance.Placement = &ec2.Placement{
		AvailabilityZone: aws.String("pl-fake-999a"),
	}

	// This is the legacy list that was removed, using this to ensure we avoid
	// region regressions if something goes wrong in the SDK
	regions := []string{
		"ap-northeast-1",
		"ap-northeast-2",
		"ap-northeast-3",
		"ap-south-1",
		"ap-southeast-1",
		"ap-southeast-2",
		"ca-central-1",
		"eu-central-1",
		"eu-west-1",
		"eu-west-2",
		"eu-west-3",
		"sa-east-1",
		"us-east-1",
		"us-east-2",
		"us-west-1",
		"us-west-2",
		"cn-north-1",
		"cn-northwest-1",
		"us-gov-west-1",
		"ap-northeast-3",

		// Ensures that we always trust what the metadata service returns
		"pl-fake-999",
	}

	for _, region := range regions {
		assert.True(t, isRegionValid(region, fake.metadata), "expected region '%s' to be valid but it was not", region)
	}

	assert.False(t, isRegionValid("pl-fake-991a", fake.metadata), "expected region 'pl-fake-991' to be invalid but it was not")
}

func TestNodeNameToProviderID(t *testing.T) {
	testNodeName := types.NodeName("ip-10-0-0-1.ec2.internal")
	testProviderID := "aws:///us-east-1c/i-02bce90670bb0c7cd"
	fakeAWS := newMockedFakeAWSServices(TestClusterID)
	c, err := newAWSCloud(CloudConfig{}, fakeAWS)
	assert.NoError(t, err)

	fakeClient := &fake.Clientset{}
	fakeInformerFactory := informers.NewSharedInformerFactory(fakeClient, 0)
	c.SetInformers(fakeInformerFactory)

	// no node name
	_, err = c.nodeNameToProviderID("")
	assert.Error(t, err)

	// informer has not synced
	c.nodeInformerHasSynced = informerNotSynced
	_, err = c.nodeNameToProviderID(testNodeName)
	assert.Error(t, err)

	// informer has synced but node not found
	c.nodeInformerHasSynced = informerSynced
	_, err = c.nodeNameToProviderID(testNodeName)
	assert.Error(t, err)

	// we are able to find the node in cache
	err = c.nodeInformer.Informer().GetStore().Add(&v1.Node{
		ObjectMeta: metav1.ObjectMeta{
			Name: string(testNodeName),
		},
		Spec: v1.NodeSpec{
			ProviderID: testProviderID,
		},
	})
	assert.NoError(t, err)
	_, err = c.nodeNameToProviderID(testNodeName)
	assert.NoError(t, err)
}

func informerSynced() bool {
	return true
}

func informerNotSynced() bool {
	return false
}

type MockedFakeELBV2 struct {
	LoadBalancers []*elbv2.LoadBalancer
	TargetGroups  []*elbv2.TargetGroup
	Listeners     []*elbv2.Listener

	// keys on all of these maps are ARNs
	LoadBalancerAttributes map[string]map[string]string
	Tags                   map[string][]elbv2.Tag
	RegisteredInstances    map[string][]string // value is list of instance IDs
}

func (m *MockedFakeELBV2) AddTags(request *elbv2.AddTagsInput) (*elbv2.AddTagsOutput, error) {
	for _, arn := range request.ResourceArns {
		for _, tag := range request.Tags {
			m.Tags[aws.StringValue(arn)] = append(m.Tags[aws.StringValue(arn)], *tag)
		}
	}

	return &elbv2.AddTagsOutput{}, nil
}

func (m *MockedFakeELBV2) CreateLoadBalancer(request *elbv2.CreateLoadBalancerInput) (*elbv2.CreateLoadBalancerOutput, error) {
	accountID := 123456789
	arn := fmt.Sprintf("arn:aws:elasticloadbalancing:us-east-1:%d:loadbalancer/net/%x/%x",
		accountID,
		rand.Uint64(),
		rand.Uint32())

	newLB := &elbv2.LoadBalancer{
		LoadBalancerArn:  aws.String(arn),
		LoadBalancerName: request.Name,
		Type:             aws.String(elbv2.LoadBalancerTypeEnumNetwork),
		VpcId:            aws.String("vpc-abc123def456abc78"),
	}
	m.LoadBalancers = append(m.LoadBalancers, newLB)

	return &elbv2.CreateLoadBalancerOutput{
		LoadBalancers: []*elbv2.LoadBalancer{newLB},
	}, nil
}

func (m *MockedFakeELBV2) DescribeLoadBalancers(request *elbv2.DescribeLoadBalancersInput) (*elbv2.DescribeLoadBalancersOutput, error) {
	findMeNames := make(map[string]bool)
	for _, name := range request.Names {
		findMeNames[aws.StringValue(name)] = true
	}

	findMeARNs := make(map[string]bool)
	for _, arn := range request.LoadBalancerArns {
		findMeARNs[aws.StringValue(arn)] = true
	}

	result := []*elbv2.LoadBalancer{}

	for _, lb := range m.LoadBalancers {
		if _, present := findMeNames[aws.StringValue(lb.LoadBalancerName)]; present {
			result = append(result, lb)
			delete(findMeNames, aws.StringValue(lb.LoadBalancerName))
		} else if _, present := findMeARNs[aws.StringValue(lb.LoadBalancerArn)]; present {
			result = append(result, lb)
			delete(findMeARNs, aws.StringValue(lb.LoadBalancerArn))
		}
	}

	if len(findMeNames) > 0 || len(findMeARNs) > 0 {
		return nil, awserr.New(elbv2.ErrCodeLoadBalancerNotFoundException, "not found", nil)
	}

	return &elbv2.DescribeLoadBalancersOutput{
		LoadBalancers: result,
	}, nil
}

func (m *MockedFakeELBV2) DeleteLoadBalancer(*elbv2.DeleteLoadBalancerInput) (*elbv2.DeleteLoadBalancerOutput, error) {
	panic("Not implemented")
}

func (m *MockedFakeELBV2) ModifyLoadBalancerAttributes(request *elbv2.ModifyLoadBalancerAttributesInput) (*elbv2.ModifyLoadBalancerAttributesOutput, error) {
	attrMap, present := m.LoadBalancerAttributes[aws.StringValue(request.LoadBalancerArn)]

	if !present {
		attrMap = make(map[string]string)
		m.LoadBalancerAttributes[aws.StringValue(request.LoadBalancerArn)] = attrMap
	}

	for _, attr := range request.Attributes {
		attrMap[aws.StringValue(attr.Key)] = aws.StringValue(attr.Value)
	}

	return &elbv2.ModifyLoadBalancerAttributesOutput{
		Attributes: request.Attributes,
	}, nil
}

func (m *MockedFakeELBV2) DescribeLoadBalancerAttributes(request *elbv2.DescribeLoadBalancerAttributesInput) (*elbv2.DescribeLoadBalancerAttributesOutput, error) {
	attrs := []*elbv2.LoadBalancerAttribute{}

	if lbAttrs, present := m.LoadBalancerAttributes[aws.StringValue(request.LoadBalancerArn)]; present {
		for key, value := range lbAttrs {
			attrs = append(attrs, &elbv2.LoadBalancerAttribute{
				Key:   aws.String(key),
				Value: aws.String(value),
			})
		}
	}

	return &elbv2.DescribeLoadBalancerAttributesOutput{
		Attributes: attrs,
	}, nil
}

func (m *MockedFakeELBV2) CreateTargetGroup(request *elbv2.CreateTargetGroupInput) (*elbv2.CreateTargetGroupOutput, error) {
	accountID := 123456789
	arn := fmt.Sprintf("arn:aws:elasticloadbalancing:us-east-1:%d:targetgroup/%x/%x",
		accountID,
		rand.Uint64(),
		rand.Uint32())

	newTG := &elbv2.TargetGroup{
		TargetGroupArn:             aws.String(arn),
		TargetGroupName:            request.Name,
		Port:                       request.Port,
		Protocol:                   request.Protocol,
		HealthCheckProtocol:        request.HealthCheckProtocol,
		HealthCheckPath:            request.HealthCheckPath,
		HealthCheckPort:            request.HealthCheckPort,
		HealthCheckTimeoutSeconds:  request.HealthCheckTimeoutSeconds,
		HealthCheckIntervalSeconds: request.HealthCheckIntervalSeconds,
		HealthyThresholdCount:      request.HealthyThresholdCount,
		UnhealthyThresholdCount:    request.UnhealthyThresholdCount,
	}

	m.TargetGroups = append(m.TargetGroups, newTG)

	return &elbv2.CreateTargetGroupOutput{
		TargetGroups: []*elbv2.TargetGroup{newTG},
	}, nil
}

func (m *MockedFakeELBV2) DescribeTargetGroups(request *elbv2.DescribeTargetGroupsInput) (*elbv2.DescribeTargetGroupsOutput, error) {
	var targetGroups []*elbv2.TargetGroup

	if request.LoadBalancerArn != nil {
		targetGroups = []*elbv2.TargetGroup{}

		for _, tg := range m.TargetGroups {
			for _, lbArn := range tg.LoadBalancerArns {
				if aws.StringValue(lbArn) == aws.StringValue(request.LoadBalancerArn) {
					targetGroups = append(targetGroups, tg)
					break
				}
			}
		}
	} else if len(request.Names) != 0 {
		targetGroups = []*elbv2.TargetGroup{}

		for _, tg := range m.TargetGroups {
			for _, name := range request.Names {
				if aws.StringValue(tg.TargetGroupName) == aws.StringValue(name) {
					targetGroups = append(targetGroups, tg)
					break
				}
			}
		}
	} else if len(request.TargetGroupArns) != 0 {
		targetGroups = []*elbv2.TargetGroup{}

		for _, tg := range m.TargetGroups {
			for _, arn := range request.TargetGroupArns {
				if aws.StringValue(tg.TargetGroupArn) == aws.StringValue(arn) {
					targetGroups = append(targetGroups, tg)
					break
				}
			}
		}
	} else {
		targetGroups = m.TargetGroups
	}

	return &elbv2.DescribeTargetGroupsOutput{
		TargetGroups: targetGroups,
	}, nil
}

func (m *MockedFakeELBV2) ModifyTargetGroup(request *elbv2.ModifyTargetGroupInput) (*elbv2.ModifyTargetGroupOutput, error) {
	var matchingTargetGroup *elbv2.TargetGroup
	dirtyGroups := []*elbv2.TargetGroup{}

	for _, tg := range m.TargetGroups {
		if aws.StringValue(tg.TargetGroupArn) == aws.StringValue(request.TargetGroupArn) {
			matchingTargetGroup = tg
			break
		}
	}

	if matchingTargetGroup != nil {
		dirtyGroups = append(dirtyGroups, matchingTargetGroup)

		if request.HealthCheckEnabled != nil {
			matchingTargetGroup.HealthCheckEnabled = request.HealthCheckEnabled
		}
		if request.HealthCheckIntervalSeconds != nil {
			matchingTargetGroup.HealthCheckIntervalSeconds = request.HealthCheckIntervalSeconds
		}
		if request.HealthCheckPath != nil {
			matchingTargetGroup.HealthCheckPath = request.HealthCheckPath
		}
		if request.HealthCheckPort != nil {
			matchingTargetGroup.HealthCheckPort = request.HealthCheckPort
		}
		if request.HealthCheckProtocol != nil {
			matchingTargetGroup.HealthCheckProtocol = request.HealthCheckProtocol
		}
		if request.HealthCheckTimeoutSeconds != nil {
			matchingTargetGroup.HealthCheckTimeoutSeconds = request.HealthCheckTimeoutSeconds
		}
		if request.HealthyThresholdCount != nil {
			matchingTargetGroup.HealthyThresholdCount = request.HealthyThresholdCount
		}
		if request.Matcher != nil {
			matchingTargetGroup.Matcher = request.Matcher
		}
		if request.UnhealthyThresholdCount != nil {
			matchingTargetGroup.UnhealthyThresholdCount = request.UnhealthyThresholdCount
		}
	}

	return &elbv2.ModifyTargetGroupOutput{
		TargetGroups: dirtyGroups,
	}, nil
}

func (m *MockedFakeELBV2) DeleteTargetGroup(request *elbv2.DeleteTargetGroupInput) (*elbv2.DeleteTargetGroupOutput, error) {
	newTargetGroups := []*elbv2.TargetGroup{}

	for _, tg := range m.TargetGroups {
		if aws.StringValue(tg.TargetGroupArn) != aws.StringValue(request.TargetGroupArn) {
			newTargetGroups = append(newTargetGroups, tg)
		}
	}

	m.TargetGroups = newTargetGroups

	delete(m.RegisteredInstances, aws.StringValue(request.TargetGroupArn))

	return &elbv2.DeleteTargetGroupOutput{}, nil
}

func (m *MockedFakeELBV2) DescribeTargetHealth(request *elbv2.DescribeTargetHealthInput) (*elbv2.DescribeTargetHealthOutput, error) {
	healthDescriptions := []*elbv2.TargetHealthDescription{}

	var matchingTargetGroup *elbv2.TargetGroup

	for _, tg := range m.TargetGroups {
		if aws.StringValue(tg.TargetGroupArn) == aws.StringValue(request.TargetGroupArn) {
			matchingTargetGroup = tg
			break
		}
	}

	if registeredTargets, present := m.RegisteredInstances[aws.StringValue(request.TargetGroupArn)]; present {
		for _, target := range registeredTargets {
			healthDescriptions = append(healthDescriptions, &elbv2.TargetHealthDescription{
				HealthCheckPort: matchingTargetGroup.HealthCheckPort,
				Target: &elbv2.TargetDescription{
					Id:   aws.String(target),
					Port: matchingTargetGroup.Port,
				},
				TargetHealth: &elbv2.TargetHealth{
					State: aws.String("healthy"),
				},
			})
		}
	}

	return &elbv2.DescribeTargetHealthOutput{
		TargetHealthDescriptions: healthDescriptions,
	}, nil
}

func (m *MockedFakeELBV2) DescribeTargetGroupAttributes(*elbv2.DescribeTargetGroupAttributesInput) (*elbv2.DescribeTargetGroupAttributesOutput, error) {
	panic("Not implemented")
}

func (m *MockedFakeELBV2) ModifyTargetGroupAttributes(*elbv2.ModifyTargetGroupAttributesInput) (*elbv2.ModifyTargetGroupAttributesOutput, error) {
	panic("Not implemented")
}

func (m *MockedFakeELBV2) RegisterTargets(request *elbv2.RegisterTargetsInput) (*elbv2.RegisterTargetsOutput, error) {
	arn := aws.StringValue(request.TargetGroupArn)
	alreadyExists := make(map[string]bool)
	for _, targetID := range m.RegisteredInstances[arn] {
		alreadyExists[targetID] = true
	}
	for _, target := range request.Targets {
		if !alreadyExists[aws.StringValue(target.Id)] {
			m.RegisteredInstances[arn] = append(m.RegisteredInstances[arn], aws.StringValue(target.Id))
		}
	}
	return &elbv2.RegisterTargetsOutput{}, nil
}

func (m *MockedFakeELBV2) DeregisterTargets(request *elbv2.DeregisterTargetsInput) (*elbv2.DeregisterTargetsOutput, error) {
	removeMe := make(map[string]bool)

	for _, target := range request.Targets {
		removeMe[aws.StringValue(target.Id)] = true
	}
	newRegisteredInstancesForArn := []string{}
	for _, targetID := range m.RegisteredInstances[aws.StringValue(request.TargetGroupArn)] {
		if !removeMe[targetID] {
			newRegisteredInstancesForArn = append(newRegisteredInstancesForArn, targetID)
		}
	}
	m.RegisteredInstances[aws.StringValue(request.TargetGroupArn)] = newRegisteredInstancesForArn

	return &elbv2.DeregisterTargetsOutput{}, nil
}

func (m *MockedFakeELBV2) CreateListener(request *elbv2.CreateListenerInput) (*elbv2.CreateListenerOutput, error) {
	accountID := 123456789
	arn := fmt.Sprintf("arn:aws:elasticloadbalancing:us-east-1:%d:listener/net/%x/%x/%x",
		accountID,
		rand.Uint64(),
		rand.Uint32(),
		rand.Uint32())

	newListener := &elbv2.Listener{
		ListenerArn:     aws.String(arn),
		Port:            request.Port,
		Protocol:        request.Protocol,
		DefaultActions:  request.DefaultActions,
		LoadBalancerArn: request.LoadBalancerArn,
	}

	m.Listeners = append(m.Listeners, newListener)

	for _, tg := range m.TargetGroups {
		for _, action := range request.DefaultActions {
			if aws.StringValue(action.TargetGroupArn) == aws.StringValue(tg.TargetGroupArn) {
				tg.LoadBalancerArns = append(tg.LoadBalancerArns, request.LoadBalancerArn)
				break
			}
		}
	}

	return &elbv2.CreateListenerOutput{
		Listeners: []*elbv2.Listener{newListener},
	}, nil
}

func (m *MockedFakeELBV2) DescribeListeners(request *elbv2.DescribeListenersInput) (*elbv2.DescribeListenersOutput, error) {
	if len(request.ListenerArns) == 0 && request.LoadBalancerArn == nil {
		return &elbv2.DescribeListenersOutput{
			Listeners: m.Listeners,
		}, nil
	} else if len(request.ListenerArns) == 0 {
		listeners := []*elbv2.Listener{}

		for _, lb := range m.Listeners {
			if aws.StringValue(lb.LoadBalancerArn) == aws.StringValue(request.LoadBalancerArn) {
				listeners = append(listeners, lb)
			}
		}

		return &elbv2.DescribeListenersOutput{
			Listeners: listeners,
		}, nil
	}
	panic("Not implemented")
}

func (m *MockedFakeELBV2) DeleteListener(*elbv2.DeleteListenerInput) (*elbv2.DeleteListenerOutput, error) {
	panic("Not implemented")
}

func (m *MockedFakeELBV2) ModifyListener(request *elbv2.ModifyListenerInput) (*elbv2.ModifyListenerOutput, error) {
	modifiedListeners := []*elbv2.Listener{}

	for _, listener := range m.Listeners {
		if aws.StringValue(listener.ListenerArn) == aws.StringValue(request.ListenerArn) {
			if request.DefaultActions != nil {
				// for each old action, find the corresponding target group, and remove the listener's LB ARN from its list
				for _, action := range listener.DefaultActions {
					var targetGroupForAction *elbv2.TargetGroup

					for _, tg := range m.TargetGroups {
						if aws.StringValue(action.TargetGroupArn) == aws.StringValue(tg.TargetGroupArn) {
							targetGroupForAction = tg
							break
						}
					}

					if targetGroupForAction != nil {
						newLoadBalancerARNs := []*string{}
						for _, lbArn := range targetGroupForAction.LoadBalancerArns {
							if aws.StringValue(lbArn) != aws.StringValue(listener.LoadBalancerArn) {
								newLoadBalancerARNs = append(newLoadBalancerARNs, lbArn)
							}
						}

						targetGroupForAction.LoadBalancerArns = newLoadBalancerARNs
					}
				}

				listener.DefaultActions = request.DefaultActions

				// for each new action, add the listener's LB ARN to that action's target groups' lists
				for _, action := range request.DefaultActions {
					var targetGroupForAction *elbv2.TargetGroup

					for _, tg := range m.TargetGroups {
						if aws.StringValue(action.TargetGroupArn) == aws.StringValue(tg.TargetGroupArn) {
							targetGroupForAction = tg
							break
						}
					}

					if targetGroupForAction != nil {
						targetGroupForAction.LoadBalancerArns = append(targetGroupForAction.LoadBalancerArns, listener.LoadBalancerArn)
					}
				}
			}
			if request.Port != nil {
				listener.Port = request.Port
			}
			if request.Protocol != nil {
				listener.Protocol = request.Protocol
			}

			modifiedListeners = append(modifiedListeners, listener)
		}
	}

	return &elbv2.ModifyListenerOutput{
		Listeners: modifiedListeners,
	}, nil
}

func (m *MockedFakeELBV2) WaitUntilLoadBalancersDeleted(*elbv2.DescribeLoadBalancersInput) error {
	panic("Not implemented")
}

func (m *MockedFakeEC2) maybeExpectDescribeSecurityGroups(clusterID, groupName string) {
	tags := []*ec2.Tag{
		{Key: aws.String(TagNameKubernetesClusterLegacy), Value: aws.String(clusterID)},
		{Key: aws.String(fmt.Sprintf("%s%s", TagNameKubernetesClusterPrefix, clusterID)), Value: aws.String(ResourceLifecycleOwned)},
	}

	m.On("DescribeSecurityGroups", &ec2.DescribeSecurityGroupsInput{Filters: []*ec2.Filter{
		newEc2Filter("group-name", groupName),
		newEc2Filter("vpc-id", ""),
	}}).Maybe().Return([]*ec2.SecurityGroup{{Tags: tags}})

	m.On("DescribeSecurityGroups", &ec2.DescribeSecurityGroupsInput{}).Maybe().Return([]*ec2.SecurityGroup{{Tags: tags}})
}

func TestNLBNodeRegistration(t *testing.T) {
	awsServices := newMockedFakeAWSServices(TestClusterID)
	awsServices.elbv2 = &MockedFakeELBV2{Tags: make(map[string][]elbv2.Tag), RegisteredInstances: make(map[string][]string), LoadBalancerAttributes: make(map[string]map[string]string)}
	c, _ := newAWSCloud(CloudConfig{}, awsServices)

	awsServices.ec2.(*MockedFakeEC2).Subnets = []*ec2.Subnet{
		{
			AvailabilityZone: aws.String("us-west-2a"),
			SubnetId:         aws.String("subnet-abc123de"),
			Tags: []*ec2.Tag{
				{
					Key:   aws.String(c.tagging.clusterTagKey()),
					Value: aws.String("owned"),
				},
			},
		},
	}

	awsServices.ec2.(*MockedFakeEC2).RouteTables = []*ec2.RouteTable{
		{
			Associations: []*ec2.RouteTableAssociation{
				{
					Main:                    aws.Bool(true),
					RouteTableAssociationId: aws.String("rtbassoc-abc123def456abc78"),
					RouteTableId:            aws.String("rtb-abc123def456abc78"),
					SubnetId:                aws.String("subnet-abc123de"),
				},
			},
			RouteTableId: aws.String("rtb-abc123def456abc78"),
			Routes: []*ec2.Route{
				{
					DestinationCidrBlock: aws.String("0.0.0.0/0"),
					GatewayId:            aws.String("igw-abc123def456abc78"),
					State:                aws.String("active"),
				},
			},
		},
	}
	awsServices.ec2.(*MockedFakeEC2).maybeExpectDescribeSecurityGroups(TestClusterID, "k8s-elb-aid")

	nodes := []*v1.Node{makeNamedNode(awsServices, 0, "a"), makeNamedNode(awsServices, 1, "b"), makeNamedNode(awsServices, 2, "c")}

	fauxService := &v1.Service{
		ObjectMeta: metav1.ObjectMeta{
			Name: "myservice",
			UID:  "id",
			Annotations: map[string]string{
				"service.beta.kubernetes.io/aws-load-balancer-type": "nlb",
			},
		},
		Spec: v1.ServiceSpec{
			Ports: []v1.ServicePort{
				{
					Name:       "http",
					Port:       8080,
					NodePort:   31173,
					TargetPort: intstr.FromInt(31173),
					Protocol:   v1.ProtocolTCP,
				},
			},
			SessionAffinity: v1.ServiceAffinityNone,
		},
	}

	_, err := c.EnsureLoadBalancer(context.TODO(), TestClusterName, fauxService, nodes)
	if err != nil {
		t.Errorf("EnsureLoadBalancer returned an error: %v", err)
	}
	for _, instances := range awsServices.elbv2.(*MockedFakeELBV2).RegisteredInstances {
		if len(instances) != 3 {
			t.Errorf("Expected 3 nodes registered with target group, saw %d", len(instances))
		}
	}

	_, err = c.EnsureLoadBalancer(context.TODO(), TestClusterName, fauxService, nodes[:2])
	if err != nil {
		t.Errorf("EnsureLoadBalancer returned an error: %v", err)
	}
	for _, instances := range awsServices.elbv2.(*MockedFakeELBV2).RegisteredInstances {
		if len(instances) != 2 {
			t.Errorf("Expected 2 nodes registered with target group, saw %d", len(instances))
		}
	}

	_, err = c.EnsureLoadBalancer(context.TODO(), TestClusterName, fauxService, nodes)
	if err != nil {
		t.Errorf("EnsureLoadBalancer returned an error: %v", err)
	}
	for _, instances := range awsServices.elbv2.(*MockedFakeELBV2).RegisteredInstances {
		if len(instances) != 3 {
			t.Errorf("Expected 3 nodes registered with target group, saw %d", len(instances))
		}
	}

	fauxService.Annotations[ServiceAnnotationLoadBalancerHealthCheckProtocol] = "http"
	tgARN := aws.StringValue(awsServices.elbv2.(*MockedFakeELBV2).Listeners[0].DefaultActions[0].TargetGroupArn)
	_, err = c.EnsureLoadBalancer(context.TODO(), TestClusterName, fauxService, nodes)
	if err != nil {
		t.Errorf("EnsureLoadBalancer returned an error: %v", err)
	}
	assert.Equal(t, 1, len(awsServices.elbv2.(*MockedFakeELBV2).Listeners))
	assert.NotEqual(t, tgARN, aws.StringValue(awsServices.elbv2.(*MockedFakeELBV2).Listeners[0].DefaultActions[0].TargetGroupArn))
}

func makeNamedNode(s *FakeAWSServices, offset int, name string) *v1.Node {
	instanceID := fmt.Sprintf("i-%x", int64(0x02bce90670bb0c7cd)+int64(offset))
	instance := &ec2.Instance{}
	instance.InstanceId = aws.String(instanceID)
	instance.Placement = &ec2.Placement{
		AvailabilityZone: aws.String("us-east-1c"),
	}
	instance.PrivateDnsName = aws.String(fmt.Sprintf("ip-172-20-0-%d.ec2.internal", 101+offset))
	instance.PrivateIpAddress = aws.String(fmt.Sprintf("192.168.0.%d", 1+offset))

	var tag ec2.Tag
	tag.Key = aws.String(TagNameKubernetesClusterLegacy)
	tag.Value = aws.String(TestClusterID)
	instance.Tags = []*ec2.Tag{&tag}

	s.instances = append(s.instances, instance)

	testProviderID := "aws:///us-east-1c/" + instanceID
	return &v1.Node{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
		},
		Spec: v1.NodeSpec{
			ProviderID: testProviderID,
		},
	}
}

func newMockedFakeAWSServices(id string) *FakeAWSServices {
	s := NewFakeAWSServices(id)
	s.ec2 = &MockedFakeEC2{FakeEC2Impl: s.ec2.(*FakeEC2Impl)}
	s.elb = &MockedFakeELB{FakeELB: s.elb.(*FakeELB)}
	return s
}

func TestAzToRegion(t *testing.T) {
	testCases := []struct {
		az     string
		region string
	}{
		{"us-west-2a", "us-west-2"},
		{"us-west-2-lax-1a", "us-west-2"},
		{"ap-northeast-2a", "ap-northeast-2"},
		{"us-gov-east-1a", "us-gov-east-1"},
		{"us-iso-east-1a", "us-iso-east-1"},
		{"us-isob-east-1a", "us-isob-east-1"},
	}

	for _, testCase := range testCases {
		result, err := azToRegion(testCase.az)
		assert.NoError(t, err)
		assert.Equal(t, testCase.region, result)
	}
}

func TestCloud_sortELBSecurityGroupList(t *testing.T) {
	type args struct {
		securityGroupIDs []string
		annotations      map[string]string
	}
	tests := []struct {
		name                 string
		args                 args
		wantSecurityGroupIDs []string
	}{
		{
			name: "with no annotation",
			args: args{
				securityGroupIDs: []string{"sg-1"},
				annotations:      map[string]string{},
			},
			wantSecurityGroupIDs: []string{"sg-1"},
		},
		{
			name: "with service.beta.kubernetes.io/aws-load-balancer-security-groups",
			args: args{
				securityGroupIDs: []string{"sg-2", "sg-1", "sg-3"},
				annotations: map[string]string{
					"service.beta.kubernetes.io/aws-load-balancer-security-groups": "sg-3,sg-2,sg-1",
				},
			},
			wantSecurityGroupIDs: []string{"sg-3", "sg-2", "sg-1"},
		},
		{
			name: "with service.beta.kubernetes.io/aws-load-balancer-extra-security-groups",
			args: args{
				securityGroupIDs: []string{"sg-2", "sg-1", "sg-3", "sg-4"},
				annotations: map[string]string{
					"service.beta.kubernetes.io/aws-load-balancer-extra-security-groups": "sg-3,sg-2,sg-1",
				},
			},
			wantSecurityGroupIDs: []string{"sg-4", "sg-3", "sg-2", "sg-1"},
		},
		{
			name: "with both annotation",
			args: args{
				securityGroupIDs: []string{"sg-2", "sg-1", "sg-3", "sg-4", "sg-5", "sg-6"},
				annotations: map[string]string{
					"service.beta.kubernetes.io/aws-load-balancer-security-groups":       "sg-3,sg-2,sg-1",
					"service.beta.kubernetes.io/aws-load-balancer-extra-security-groups": "sg-6,sg-5",
				},
			},
			wantSecurityGroupIDs: []string{"sg-3", "sg-2", "sg-1", "sg-4", "sg-6", "sg-5"},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			c := &Cloud{}
			c.sortELBSecurityGroupList(tt.args.securityGroupIDs, tt.args.annotations)
			assert.Equal(t, tt.wantSecurityGroupIDs, tt.args.securityGroupIDs)
		})
	}
}

func TestCloud_buildNLBHealthCheckConfiguration(t *testing.T) {
	tests := []struct {
		name        string
		annotations map[string]string
		service     *v1.Service
		want        healthCheckConfig
		wantError   bool
	}{
		{
			name:        "default cluster",
			annotations: map[string]string{},
			service: &v1.Service{
				ObjectMeta: metav1.ObjectMeta{
					Name: "test-svc",
					UID:  "UID",
				},
				Spec: v1.ServiceSpec{
					Ports: []v1.ServicePort{
						{
							Name:       "http",
							Protocol:   v1.ProtocolTCP,
							Port:       8080,
							TargetPort: intstr.FromInt(8880),
							NodePort:   32205,
						},
					},
				},
			},
			want: healthCheckConfig{
				Port:               "traffic-port",
				Protocol:           elbv2.ProtocolEnumTcp,
				Interval:           30,
				Timeout:            10,
				HealthyThreshold:   3,
				UnhealthyThreshold: 3,
			},
			wantError: false,
		},
		{
			name:        "default local",
			annotations: map[string]string{},
			service: &v1.Service{
				ObjectMeta: metav1.ObjectMeta{
					Name: "test-svc",
					UID:  "UID",
				},
				Spec: v1.ServiceSpec{
					Type: v1.ServiceTypeLoadBalancer,
					Ports: []v1.ServicePort{
						{
							Name:       "http",
							Protocol:   v1.ProtocolTCP,
							Port:       8080,
							TargetPort: intstr.FromInt(8880),
							NodePort:   32205,
						},
					},
					ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal,
					HealthCheckNodePort:   32213,
				},
			},
			want: healthCheckConfig{
				Port:               "32213",
				Path:               "/healthz",
				Protocol:           elbv2.ProtocolEnumHttp,
				Interval:           10,
				Timeout:            10,
				HealthyThreshold:   2,
				UnhealthyThreshold: 2,
			},
			wantError: false,
		},
		{
			name: "with TCP healthcheck",
			service: &v1.Service{
				ObjectMeta: metav1.ObjectMeta{
					Name: "test-svc",
					UID:  "UID",
					Annotations: map[string]string{
						ServiceAnnotationLoadBalancerHealthCheckProtocol:  "TCP",
						ServiceAnnotationLoadBalancerHealthCheckPort:      "8001",
						ServiceAnnotationLoadBalancerHealthCheckPath:      "/healthz",
						ServiceAnnotationLoadBalancerHCHealthyThreshold:   "4",
						ServiceAnnotationLoadBalancerHCUnhealthyThreshold: "4",
						ServiceAnnotationLoadBalancerHCInterval:           "10",
						ServiceAnnotationLoadBalancerHCTimeout:            "5",
					},
				},
				Spec: v1.ServiceSpec{
					Type: v1.ServiceTypeLoadBalancer,
					Ports: []v1.ServicePort{
						{
							Name:       "http",
							Protocol:   v1.ProtocolTCP,
							Port:       8080,
							TargetPort: intstr.FromInt(8880),
							NodePort:   32205,
						},
					},
					ExternalTrafficPolicy: v1.ServiceExternalTrafficPolicyTypeLocal,
					HealthCheckNodePort:   32213,
				},
			},
			want: healthCheckConfig{
				Interval:           10,
				Timeout:            5,
				Protocol:           "TCP",
				Port:               "8001",
				HealthyThreshold:   4,
				UnhealthyThreshold: 4,
			},
			wantError: false,
		},
		{
			name: "with HTTP healthcheck",
			service: &v1.Service{
				ObjectMeta: metav1.ObjectMeta{
					Name: "test-svc",
					UID:  "UID",
					Annotations: map[string]string{
						ServiceAnnotationLoadBalancerHealthCheckProtocol:  "HTTP",
						ServiceAnnotationLoadBalancerHealthCheckPort:      "41233",
						ServiceAnnotationLoadBalancerHealthCheckPath:      "/healthz",
						ServiceAnnotationLoadBalancerHCHealthyThreshold:   "5",
						ServiceAnnotationLoadBalancerHCUnhealthyThreshold: "5",
						ServiceAnnotationLoadBalancerHCInterval:           "30",
						ServiceAnnotationLoadBalancerHCTimeout:            "6",
					},
				},
				Spec: v1.ServiceSpec{
					Ports: []v1.ServicePort{
						{
							Name:       "http",
							Protocol:   v1.ProtocolTCP,
							Port:       8080,
							TargetPort: intstr.FromInt(8880),
							NodePort:   32205,
						},
					},
				},
			},
			want: healthCheckConfig{
				Interval:           30,
				Timeout:            6,
				Protocol:           "HTTP",
				Port:               "41233",
				Path:               "/healthz",
				HealthyThreshold:   5,
				UnhealthyThreshold: 5,
			},
			wantError: false,
		},
		{
			name: "HTTP healthcheck default path",
			service: &v1.Service{
				ObjectMeta: metav1.ObjectMeta{
					Name: "test-svc",
					UID:  "UID",
					Annotations: map[string]string{
						ServiceAnnotationLoadBalancerHealthCheckProtocol: "Http",
					},
				},
				Spec: v1.ServiceSpec{
					Ports: []v1.ServicePort{
						{
							Name:       "http",
							Protocol:   v1.ProtocolTCP,
							Port:       8080,
							TargetPort: intstr.FromInt(8880),
							NodePort:   32205,
						},
					},
				},
			},
			want: healthCheckConfig{
				Interval:           30,
				Timeout:            10,
				Protocol:           "HTTP",
				Path:               "/",
				Port:               "traffic-port",
				HealthyThreshold:   3,
				UnhealthyThreshold: 3,
			},
			wantError: false,
		},
		{
			name: "interval not 10 or 30",
			service: &v1.Service{
				ObjectMeta: metav1.ObjectMeta{
					Name: "test-svc",
					UID:  "UID",
					Annotations: map[string]string{
						ServiceAnnotationLoadBalancerHCInterval: "23",
					},
				},
				Spec: v1.ServiceSpec{
					Ports: []v1.ServicePort{
						{
							Name:       "http",
							Protocol:   v1.ProtocolTCP,
							Port:       8080,
							TargetPort: intstr.FromInt(8880),
							NodePort:   32205,
						},
					},
				},
			},
			want: healthCheckConfig{
				Port:               "traffic-port",
				Protocol:           elbv2.ProtocolEnumTcp,
				Interval:           23,
				Timeout:            10,
				HealthyThreshold:   3,
				UnhealthyThreshold: 3,
			},
			wantError: false,
		},
		{
			name: "invalid timeout",
			service: &v1.Service{
				ObjectMeta: metav1.ObjectMeta{
					Name: "test-svc",
					UID:  "UID",
					Annotations: map[string]string{
						ServiceAnnotationLoadBalancerHCTimeout: "non-numeric",
					},
				},
				Spec: v1.ServiceSpec{
					Ports: []v1.ServicePort{
						{
							Name:       "http",
							Protocol:   v1.ProtocolTCP,
							Port:       8080,
							TargetPort: intstr.FromInt(8880),
							NodePort:   32205,
						},
					},
				},
			},
			want:      healthCheckConfig{},
			wantError: true,
		},
		{
			name: "mismatch healthy and unhealthy targets",
			service: &v1.Service{
				ObjectMeta: metav1.ObjectMeta{
					Name: "test-svc",
					UID:  "UID",
					Annotations: map[string]string{
						ServiceAnnotationLoadBalancerHCHealthyThreshold:   "7",
						ServiceAnnotationLoadBalancerHCUnhealthyThreshold: "5",
					},
				},
				Spec: v1.ServiceSpec{
					Ports: []v1.ServicePort{
						{
							Name:       "http",
							Protocol:   v1.ProtocolTCP,
							Port:       8080,
							TargetPort: intstr.FromInt(8880),
							NodePort:   32205,
						},
					},
				},
			},
			want: healthCheckConfig{
				Port:               "traffic-port",
				Protocol:           elbv2.ProtocolEnumTcp,
				Interval:           30,
				Timeout:            10,
				HealthyThreshold:   7,
				UnhealthyThreshold: 5,
			},
			wantError: false,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			c := &Cloud{}
			hc, err := c.buildNLBHealthCheckConfiguration(tt.service)
			if !tt.wantError {
				assert.Equal(t, tt.want, hc)
			} else {
				assert.NotNil(t, err)
			}
		})
	}
}

func Test_parseStringSliceAnnotation(t *testing.T) {
	tests := []struct {
		name        string
		annotation  string
		annotations map[string]string
		want        []string
		wantExist   bool
	}{
		{
			name:       "empty annotation",
			annotation: "test.annotation",
			wantExist:  false,
		},
		{
			name:       "empty value",
			annotation: "a1",
			annotations: map[string]string{
				"a1": "\t, ,,",
			},
			want:      nil,
			wantExist: true,
		},
		{
			name:       "single value",
			annotation: "a1",
			annotations: map[string]string{
				"a1": "   value 1 ",
			},
			want:      []string{"value 1"},
			wantExist: true,
		},
		{
			name:       "multiple values",
			annotation: "a1",
			annotations: map[string]string{
				"a1": "subnet-1, subnet-2, My Subnet ",
			},
			want:      []string{"subnet-1", "subnet-2", "My Subnet"},
			wantExist: true,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			var gotValue []string
			gotExist := parseStringSliceAnnotation(tt.annotations, tt.annotation, &gotValue)
			assert.Equal(t, tt.wantExist, gotExist)
			assert.Equal(t, tt.want, gotValue)
		})
	}
}
