Skip to content

Commit

Permalink
test, e2e: test kubevirt live migration
Browse files Browse the repository at this point in the history 
This change add e2e tests to check pre and post copy live migration, for
that the "kind.sh" script has learn to install kubevirt. Github actions
also include jobs to exercise it, the post-copy test has being disabled
for them since it's not working at that env.

To test that tcp connection survives live migration http connection
reuse is check.

Signed-off-by: Enrique Llorente <[email protected]>
  • Loading branch information
@qinqon
qinqon committed Jul 18, 2023
1 parent dadf621 commit 7dfaabe
Show file tree
Hide file tree
Showing 12 changed files with 1,570 additions and 37 deletions.
5 changes: 5 additions & 0 deletions .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,8 @@ jobs:
- {"target": "multi-node-zones", "ha": "noHA", "gateway-mode": "local", "ipfamily": "ipv4", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "ic": "ic-multi-node-zones", "num-workers": "3", "num-nodes-per-zone": "2"}
- {"target": "external-gateway", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "2br", "ic": "ic-single-node-zones"}
- {"target": "external-gateway", "ha": "noHA", "gateway-mode": "local", "ipfamily": "ipv4", "disable-snat-multiple-gws": "noSnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"}
- {"target": "kv-live-migration", "ha": "noHA", "gateway-mode": "local", "ipfamily": "ipv4", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "ic": "ic-disabled"}
- {"target": "kv-live-migration", "ha": "noHA", "gateway-mode": "shared", "ipfamily": "dualstack", "disable-snat-multiple-gws": "SnatGW", "second-bridge": "1br", "ic": "ic-single-node-zones"}
needs: [ build-pr ]
env:
JOB_NAME: "${{ matrix.target }}-${{ matrix.ha }}-${{ matrix.gateway-mode }}-${{ matrix.ipfamily }}-${{ matrix.disable-snat-multiple-gws }}-${{ matrix.second-bridge }}"
Expand All @@ -406,6 +408,7 @@ jobs:
KIND_IPV4_SUPPORT: "${{ matrix.ipfamily == 'IPv4' || matrix.ipfamily == 'dualstack' }}"
KIND_IPV6_SUPPORT: "${{ matrix.ipfamily == 'IPv6' || matrix.ipfamily == 'dualstack' }}"
ENABLE_MULTI_NET: "${{ matrix.target == 'multi-homing' }}"
KIND_INSTALL_KUBEVIRT: "${{ matrix.target == 'kv-live-migration' }}"
OVN_COMPACT_MODE: "${{ matrix.target == 'compact-mode' }}"
OVN_DUMMY_GATEWAY_BRIDGE: "${{ matrix.target == 'compact-mode' }}"
OVN_ENABLE_INTERCONNECT: "${{ matrix.ic == 'ic-single-node-zones' || matrix.ic == 'ic-multi-node-zones'}}"
Expand Down Expand Up @@ -471,6 +474,8 @@ jobs:
make -C test control-plane WHAT="Multi node zones interconnect"
elif [ "${{ matrix.target }}" == "external-gateway" ]; then
make -C test control-plane WHAT="External Gateway"
elif [ "${{ matrix.target }}" == "kv-live-migration" ]; then
make -C test control-plane WHAT="Kubevirt Virtual Machines"
else
make -C test ${{ matrix.target }}
fi
Expand Down
61 changes: 58 additions & 3 deletions contrib/kind.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,8 @@ parse_args() {
;;
-pl | --install-cni-plugins ) KIND_INSTALL_PLUGINS=true
;;
-ikv | --install-kubevirt) KIND_INSTALL_KUBEVIRT=true
;;
-ha | --ha-enabled ) OVN_HA=true
;;
-me | --multicast-enabled) OVN_MULTICAST_ENABLE=true
Expand Down Expand Up @@ -358,6 +360,7 @@ print_params() {
echo "KIND_INSTALL_INGRESS = $KIND_INSTALL_INGRESS"
echo "KIND_INSTALL_METALLB = $KIND_INSTALL_METALLB"
echo "KIND_INSTALL_PLUGINS = $KIND_INSTALL_PLUGINS"
echo "KIND_INSTALL_KUBEVIRT = $KIND_INSTALL_KUBEVIRT"
echo "OVN_HA = $OVN_HA"
echo "RUN_IN_CONTAINER = $RUN_IN_CONTAINER"
echo "KIND_CLUSTER_NAME = $KIND_CLUSTER_NAME"
Expand Down Expand Up @@ -499,11 +502,12 @@ set_default_params() {
fi
RUN_IN_CONTAINER=${RUN_IN_CONTAINER:-false}
KIND_IMAGE=${KIND_IMAGE:-kindest/node}
K8S_VERSION=${K8S_VERSION:-v1.26.0}
K8S_VERSION=${K8S_VERSION:-v1.26.3}
OVN_GATEWAY_MODE=${OVN_GATEWAY_MODE:-shared}
KIND_INSTALL_INGRESS=${KIND_INSTALL_INGRESS:-false}
KIND_INSTALL_METALLB=${KIND_INSTALL_METALLB:-false}
KIND_INSTALL_PLUGINS=${KIND_INSTALL_PLUGINS:-false}
KIND_INSTALL_KUBEVIRT=${KIND_INSTALL_KUBEVIRT:-false}
OVN_HA=${OVN_HA:-false}
KIND_LOCAL_REGISTRY=${KIND_LOCAL_REGISTRY:-false}
KIND_LOCAL_REGISTRY_NAME=${KIND_LOCAL_REGISTRY_NAME:-kind-registry}
Expand Down Expand Up @@ -580,7 +584,7 @@ set_default_params() {
OVN_HOST_NETWORK_NAMESPACE=${OVN_HOST_NETWORK_NAMESPACE:-ovn-host-network}
OVN_EGRESSIP_HEALTHCHECK_PORT=${OVN_EGRESSIP_HEALTHCHECK_PORT:-9107}
OCI_BIN=${KIND_EXPERIMENTAL_PROVIDER:-docker}
OVN_DEPLOY_PODS=${OVN_DEPLOY_PODS:-"ovnkube-master ovnkube-node"}
OVN_DEPLOY_PODS=${OVN_DEPLOY_PODS:-"ovnkube-zone-controller ovnkube-control-plane ovnkube-master ovnkube-node"}
OVN_METRICS_SCALE_ENABLE=${OVN_METRICS_SCALE_ENABLE:-false}
OVN_ISOLATED=${OVN_ISOLATED:-false}
OVN_GATEWAY_OPTS=${OVN_GATEWAY_OPTS:-""}
Expand Down Expand Up @@ -749,7 +753,9 @@ docker_disable_ipv6() {

coredns_patch() {
dns_server="8.8.8.8"
if [ "$KIND_IPV6_SUPPORT" == true ]; then
# No need for ipv6 nameserver for dual stack, it will ask for
# A and AAAA records
if [ "$IP_FAMILY" == "ipv6" ]; then
dns_server="2001:4860:4860::8888"
fi

Expand Down Expand Up @@ -806,6 +812,12 @@ build_ovn_image() {
$OCI_BIN push "${OVN_IMAGE}"
fi
popd
# We should push to local registry if image is not remote
elif [ "${OVN_IMAGE}" != "" -a "${KIND_LOCAL_REGISTRY}" == true ] && (echo "$OVN_IMAGE" | grep / -vq); then
local local_registry_ovn_image="localhost:5000/${OVN_IMAGE}"
$OCI_BIN tag "$OVN_IMAGE" $local_registry_ovn_image
OVN_IMAGE=$local_registry_ovn_image
$OCI_BIN push $OVN_IMAGE
fi
}

Expand Down Expand Up @@ -1239,6 +1251,46 @@ add_dns_hostnames() {
done
}

function is_nested_virt_enabled() {
local kvm_nested="unknown"
if [ -f "/sys/module/kvm_intel/parameters/nested" ]; then
kvm_nested=$( cat /sys/module/kvm_intel/parameters/nested )
elif [ -f "/sys/module/kvm_amd/parameters/nested" ]; then
kvm_nested=$( cat /sys/module/kvm_amd/parameters/nested )
fi
[ "$kvm_nested" == "1" ] || [ "$kvm_nested" == "Y" ] || [ "$kvm_nested" == "y" ]
}

function install_kubevirt() {
for node in $(kubectl get node --no-headers -o custom-columns=":metadata.name"); do
$OCI_BIN exec -t $node bash -c "echo 'fs.inotify.max_user_watches=1048576' >> /etc/sysctl.conf"
$OCI_BIN exec -t $node bash -c "echo 'fs.inotify.max_user_instances=512' >> /etc/sysctl.conf"
$OCI_BIN exec -i $node bash -c "sysctl -p /etc/sysctl.conf"
if [[ "${node}" =~ worker ]]; then
kubectl label nodes $node node-role.kubernetes.io/worker="" --overwrite=true
fi
done
local nightly_build_base_url="https://storage.googleapis.com/kubevirt-prow/devel/nightly/release/kubevirt/kubevirt"
local latest=$(curl -sL "${nightly_build_base_url}/latest")

echo "Deploy latest nighly build Kubevirt"
if [ "$(kubectl get kubevirts -n kubevirt kubevirt -ojsonpath='{.status.phase}')" != "Deployed" ]; then
kubectl apply -f "${nightly_build_base_url}/${latest}/kubevirt-operator.yaml"
kubectl apply -f "${nightly_build_base_url}/${latest}/kubevirt-cr.yaml"
if ! is_nested_virt_enabled; then
kubectl -n kubevirt patch kubevirt kubevirt --type=merge --patch '{"spec":{"configuration":{"developerConfiguration":{"useEmulation":true}}}}'
fi
fi
if ! kubectl wait -n kubevirt kv kubevirt --for condition=Available --timeout 15m; then
kubectl get pod -n kubevirt -l || true
kubectl describe pod -n kubevirt -l || true
for p in $(kubectl get pod -n kubevirt -l -o name |sed "s#pod/##"); do
kubectl logs -p --all-containers=true -n kubevirt $p || true
kubectl logs --all-containers=true -n kubevirt $p || true
done
fi
}

check_dependencies
# In order to allow providing arguments with spaces, e.g. "-vconsole:info -vfile:info"
# the original command <parse_args $*> was replaced by <parse_args "$@">
Expand Down Expand Up @@ -1301,3 +1353,6 @@ fi
if [ "$KIND_INSTALL_PLUGINS" == true ]; then
install_plugins
fi
if [ "$KIND_INSTALL_KUBEVIRT" == true ]; then
install_kubevirt
fi
1 change: 1 addition & 0 deletions dist/images/Dockerfile.fedora.dev
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ RUN INSTALL_PKGS=" \
python3-openvswitch python3-pyOpenSSL \
autoconf automake libtool g++ gcc fedora-packager rpmdevtools \
unbound unbound-devel groff python3-sphinx graphviz openssl openssl-devel \
libbpf-devel libxdp-devel numactl-devel \
checkpolicy libcap-ng-devel selinux-policy-devel" && \
dnf install --best --refresh -y --setopt=tsflags=nodocs $INSTALL_PKGS && \
dnf clean all && rm -rf /var/cache/dnf/*
Expand Down
2 changes: 1 addition & 1 deletion go-controller/pkg/libovsdbops/dhcp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ type DHCPOptionsPredicate func(*nbdb.DHCPOptions) bool

// CreateOrUpdateDhcpOptionsOps will configure logical switch port DHCPv4Options and DHCPv6Options fields with
// options at dhcpv4Options and dhcpv6Options arguments and create/update DHCPOptions objects that matches the
// pv4 and pv6 predicates. The DHCP options not provided will be reset to nil the LSP fields.
// pv4 and pv6 predicates. The missing DHCP options will default to nil in the LSP attributes.
func CreateOrUpdateDhcpOptionsOps(nbClient libovsdbclient.Client, ops []libovsdb.Operation, lsp *nbdb.LogicalSwitchPort, dhcpIPv4Options, dhcpIPv6Options *nbdb.DHCPOptions) ([]libovsdb.Operation, error) {
opModels := []operationModel{}
if dhcpIPv4Options != nil {
Expand Down
7 changes: 7 additions & 0 deletions go-controller/pkg/ovn/kubevirt_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -631,6 +631,13 @@ var _ = Describe("OVN Kubevirt Operations", func() {
podToCreate.CreationTimestamp = metav1.NewTime(time.Now())
_, err = fakeOvn.fakeClient.KubeClient.CoreV1().Pods(t.namespace).Create(context.TODO(), podToCreate, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())
Eventually(func() error {
_, err := fakeOvn.controller.watchFactory.GetPod(podToCreate.Namespace, podToCreate.Name)
return err
}).
WithTimeout(time.Minute).
WithPolling(time.Second).
Should(Succeed(), "should fill in the cache with the pod")
}

expectedOVN := []libovsdbtest.TestData{}
Expand Down
51 changes: 41 additions & 10 deletions test/e2e/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,53 +42,74 @@ require (
golang.org/x/sync v0.1.0
k8s.io/api v0.26.1
k8s.io/apimachinery v0.26.1
k8s.io/client-go v0.26.1
k8s.io/client-go v12.0.0+incompatible
k8s.io/klog v1.0.0
k8s.io/kubernetes v1.24.0
k8s.io/pod-security-admission v0.24.0
k8s.io/utils v0.0.0-20230209194617-a36077c30491
)

require (
github.com/aws/aws-sdk-go v1.44.204 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/clarketm/json v1.17.1 // indirect
github.com/coreos/go-json v0.0.0-20230131223807-18775e0fb4fb // indirect
github.com/coreos/go-semver v0.3.1 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/coreos/ignition/v2 v2.15.0 // indirect
github.com/coreos/prometheus-operator v0.38.0 // indirect
github.com/coreos/vcontext v0.0.0-20230201181013-d72178a18687 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/distribution v2.8.2+incompatible // indirect
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/felixge/httpsnoop v1.0.1 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-kit/kit v0.9.0 // indirect
github.com/go-logfmt/logfmt v0.5.1 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.20.0 // indirect
github.com/go-openapi/swag v0.19.14 // indirect
github.com/go-openapi/swag v0.21.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/glog v1.0.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/gnostic v0.5.7-v3refs // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.2.0 // indirect
github.com/google/goterm v0.0.0-20190703233501-fc88cf888a3f // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/imdario/mergo v0.3.12 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
github.com/moby/spdystream v0.2.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/openshift/api v0.0.0-20211217221424-8779abfbd571 // indirect
github.com/openshift/client-go v0.0.0-20210112165513-ebc401615f47 // indirect
github.com/openshift/custom-resource-status v1.1.2 // indirect
github.com/pborman/uuid v1.2.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.14.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/spf13/cobra v1.4.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/testify v1.8.0 // indirect
github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace // indirect
github.com/stretchr/testify v1.8.2 // indirect
github.com/vincent-petithory/dataurl v1.0.0 // indirect
go.opentelemetry.io/contrib v0.20.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 // indirect
go.opentelemetry.io/otel v0.20.0 // indirect
Expand All @@ -101,19 +122,19 @@ require (
go.opentelemetry.io/proto/otlp v0.7.0 // indirect
golang.org/x/crypto v0.1.0 // indirect
golang.org/x/net v0.7.0 // indirect
golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
golang.org/x/oauth2 v0.5.0 // indirect
golang.org/x/sys v0.5.0 // indirect
golang.org/x/term v0.5.0 // indirect
golang.org/x/text v0.7.0 // indirect
golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
google.golang.org/grpc v1.49.0 // indirect
google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apiextensions-apiserver v0.23.5 // indirect
k8s.io/apiserver v0.24.0 // indirect
k8s.io/cloud-provider v0.24.0 // indirect
k8s.io/component-base v0.24.0 // indirect
Expand All @@ -122,8 +143,18 @@ require (
k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
k8s.io/kubectl v0.24.0 // indirect
k8s.io/kubelet v0.24.2 // indirect
kubevirt.io/containerized-data-importer-api v1.55.0 // indirect
kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)

require (
github.com/coreos/butane v0.18.0
github.com/google/goexpect v0.0.0-20210430020637-ab937bf7fd6f
google.golang.org/grpc v1.53.0
kubevirt.io/api v1.0.0-alpha.0
kubevirt.io/client-go v1.0.0-alpha.0
)
Loading

0 comments on commit 7dfaabe

Please sign in to comment.