diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000000..fffc67955b55 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +root = true + +[*] +charset = utf-8 +end_of_line = lf +# indent_size = 4 +indent_style = space +insert_final_newline = true +# max_line_length = 120 +# tab_width = 4 diff --git a/.github/linters/codespell.txt b/.github/linters/codespell.txt index 27761c70f446..e954402a57c3 100644 --- a/.github/linters/codespell.txt +++ b/.github/linters/codespell.txt @@ -187,6 +187,7 @@ environmnet equivalant erro erronous +errorprone everthing everytime excute diff --git a/PendingReleaseNotes b/PendingReleaseNotes index 9670b6e7c13a..02d63811e36e 100644 --- a/PendingReleaseNotes +++ b/PendingReleaseNotes @@ -39,3 +39,17 @@ example.ver.1 > example.ver.2: which can now be attached to Instances. This is to prevent the Secondary Storage to grow to enormous sizes as Linux Distributions keep growing in size while a stripped down Linux should fit on a 2.88MB floppy. + +4.22.0.0 > 4.22.0.1: + * Disk-only instance snapshots for KVM UEFI VMs now include a sidecar copy of + the active NVRAM state so revert operations restore both disk and firmware + boot state consistently. + + * UEFI disk-only instance snapshots taken before this change do not contain an + NVRAM sidecar and cannot be safely reverted. Take a new snapshot after + upgrading before relying on revert for UEFI VMs. + + * Taking a disk-only instance snapshot for KVM UEFI VMs now briefly suspends + the guest while the NVRAM sidecar is copied, so that the captured firmware + state is consistent with the disk snapshot. Non-UEFI VMs are unaffected and + continue to snapshot live. diff --git a/agent/src/main/java/com/cloud/agent/mockvm/MockVmMgr.java b/agent/src/main/java/com/cloud/agent/mockvm/MockVmMgr.java index 54fdde3d3d28..d97207c9f30c 100644 --- a/agent/src/main/java/com/cloud/agent/mockvm/MockVmMgr.java +++ b/agent/src/main/java/com/cloud/agent/mockvm/MockVmMgr.java @@ -249,7 +249,7 @@ public void freeVncPort(int port) { public MockVm createVmFromSpec(VirtualMachineTO vmSpec) { String vmName = vmSpec.getName(); long ramSize = vmSpec.getMinRam(); - int utilizationPercent = randSeed.nextInt() % 100; + int utilizationPercent = randSeed.nextInt(100); MockVm vm = null; synchronized (this) { diff --git a/api/src/main/java/com/cloud/host/Host.java b/api/src/main/java/com/cloud/host/Host.java index b52348201516..a7b89b8a2b85 100644 --- a/api/src/main/java/com/cloud/host/Host.java +++ b/api/src/main/java/com/cloud/host/Host.java @@ -55,6 +55,7 @@ public static String[] toStrings(Host.Type... types) { } String HOST_UEFI_ENABLE = "host.uefi.enable"; + String HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM = "host.kvm.diskonlyvmsnapshot.nvram"; String HOST_VOLUME_ENCRYPTION = "host.volume.encryption"; String HOST_INSTANCE_CONVERSION = "host.instance.conversion"; String HOST_VDDK_SUPPORT = "host.vddk.support"; diff --git a/api/src/main/java/com/cloud/network/netris/NetrisService.java b/api/src/main/java/com/cloud/network/netris/NetrisService.java index 110e9f07105a..1e7c7d66399a 100644 --- a/api/src/main/java/com/cloud/network/netris/NetrisService.java +++ b/api/src/main/java/com/cloud/network/netris/NetrisService.java @@ -21,6 +21,8 @@ import com.cloud.network.SDNProviderNetworkRule; import com.cloud.network.vpc.StaticRoute; import com.cloud.network.vpc.Vpc; +import com.cloud.offering.NetworkOffering; +import com.cloud.utils.net.NetUtils; import java.util.List; @@ -81,18 +83,20 @@ public interface NetrisService { /** * Creates a virtual network (vNet) resource. * - * @param zoneId the ID of the zone - * @param accountId the ID of the account - * @param domainId the ID of the domain - * @param vpcName the name of the VPC - * @param vpcId the ID of the VPC - * @param networkName the name of the network - * @param networkId the ID of the network - * @param cidr the CIDR of the network - * @param globalRouting true if global routing is enabled + * @param zoneId the ID of the zone + * @param accountId the ID of the account + * @param domainId the ID of the domain + * @param vpcName the name of the VPC + * @param vpcId the ID of the VPC + * @param networkName the name of the network + * @param networkId the ID of the network + * @param cidr the CIDR of the network + * @param networkMode the network mode + * @param internetProtocol the internet protocol + * @param isL2 true if it is an L2 network * @return true if the operation is successful, false otherwise */ - boolean createVnetResource(Long zoneId, long accountId, long domainId, String vpcName, Long vpcId, String networkName, Long networkId, String cidr, Boolean globalRouting); + boolean createVnetResource(Long zoneId, long accountId, long domainId, String vpcName, Long vpcId, String networkName, Long networkId, String cidr, NetworkOffering.NetworkMode networkMode, NetUtils.InternetProtocol internetProtocol, boolean isL2); /** * Updates an existing vNet resource. @@ -105,9 +109,10 @@ public interface NetrisService { * @param networkName the new name of the network * @param networkId the ID of the network * @param prevNetworkName the previous name of the network + * @param isL2 true if it is an L2 network * @return true if the operation is successful, false otherwise */ - boolean updateVnetResource(Long zoneId, long accountId, long domainId, String vpcName, Long vpcId, String networkName, Long networkId, String prevNetworkName); + boolean updateVnetResource(Long zoneId, long accountId, long domainId, String vpcName, Long vpcId, String networkName, Long networkId, String prevNetworkName, boolean isL2); /** * Deletes an existing vNet resource. @@ -120,9 +125,10 @@ public interface NetrisService { * @param networkName the name of the network * @param networkId the ID of the network * @param cidr the CIDR of the network + * @param isL2 true if it is an L2 network * @return true if the operation is successful, false otherwise */ - boolean deleteVnetResource(long zoneId, long accountId, long domainId, String vpcName, Long vpcId, String networkName, Long networkId, String cidr); + boolean deleteVnetResource(long zoneId, long accountId, long domainId, String vpcName, Long vpcId, String networkName, Long networkId, String cidr, boolean isL2); /** * Creates a source NAT rule for a VPC or network. diff --git a/api/src/main/java/com/cloud/resource/ResourceService.java b/api/src/main/java/com/cloud/resource/ResourceService.java index 3cdf8fc64e99..202d7fe63c49 100644 --- a/api/src/main/java/com/cloud/resource/ResourceService.java +++ b/api/src/main/java/com/cloud/resource/ResourceService.java @@ -91,7 +91,7 @@ public interface ResourceService { DataCenter getZone(Long zoneId); - List getSupportedHypervisorTypes(long zoneId, boolean forVirtualRouter, Long podId); + List getSupportedHypervisorTypes(long zoneId, boolean forSystemVm, Long podId); boolean releaseHostReservation(Long hostId); diff --git a/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java b/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java index 5c78d6bedd64..c91f5b736514 100644 --- a/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java +++ b/api/src/main/java/com/cloud/vm/VirtualMachineProfile.java @@ -79,6 +79,7 @@ public static class Param { public static final Param PreserveNics = new Param("PreserveNics"); public static final Param ConsiderLastHost = new Param("ConsiderLastHost"); public static final Param ReturnAfterVolumePrepare = new Param("ReturnAfterVolumePrepare"); + public static final Param ResetPasswordOnRestore = new Param("ResetPasswordOnRestore"); private String name; diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index a8ff00c40ff3..c4de8bfcba22 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -452,6 +452,7 @@ public class ApiConstants { public static final String CURRENT_PASSWORD = "currentpassword"; public static final String SHOULD_UPDATE_PASSWORD = "update_passwd_on_host"; public static final String PASSWORD_ENABLED = "passwordenabled"; + public static final String RESET_PASSWORD = "resetpassword"; public static final String SSHKEY_ENABLED = "sshkeyenabled"; public static final String PATH = "path"; public static final String PATH_READY = "pathready"; @@ -1288,6 +1289,7 @@ public class ApiConstants { public static final String HAS_RULES = "hasrules"; public static final String NSX_DETAIL_KEY = "forNsx"; public static final String NETRIS_DETAIL_KEY = "forNetris"; + public static final String NETRIS_NAT_PUBLIC_IP_ID = "netrisnatpublicipid"; public static final String NETRIS_TAG = "netristag"; public static final String NETRIS_VXLAN_ID = "netrisvxlanid"; public static final String NETRIS_URL = "netrisurl"; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/UpdateBackupOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/UpdateBackupOfferingCmd.java index 9de06715ee74..fd77aeb47081 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/UpdateBackupOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/backup/UpdateBackupOfferingCmd.java @@ -83,8 +83,8 @@ public Boolean getAllowUserDrivenBackups() { public void execute() { try { if (StringUtils.isAllEmpty(getName(), getDescription()) && getAllowUserDrivenBackups() == null) { - throw new InvalidParameterValueException(String.format("Can't update Backup Offering [id: %s] because there are no parameters to be updated, at least one of the", - "following should be informed: name, description or allowUserDrivenBackups.", id)); + throw new InvalidParameterValueException(String.format("Can't update Backup Offering [id: %s] because there are no parameters to be updated," + + " at least one of the following should be passed: name, description or allowUserDrivenBackups.", id)); } BackupOffering result = backupManager.updateBackupOffering(this); diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/network/CreateNetworkOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/network/CreateNetworkOfferingCmd.java index a0559f57dab0..da71b6596c09 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/network/CreateNetworkOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/network/CreateNetworkOfferingCmd.java @@ -60,6 +60,7 @@ import static com.cloud.network.Network.Service.NetworkACL; import static com.cloud.network.Network.Service.UserData; import static com.cloud.network.Network.Service.Firewall; +import static com.cloud.network.Network.Service.Vpn; import static org.apache.cloudstack.api.command.utils.OfferingUtils.isNetrisNatted; import static org.apache.cloudstack.api.command.utils.OfferingUtils.isNetrisRouted; @@ -309,6 +310,10 @@ public List getSupportedServices() { } else { services.add(Firewall.getName()); } + + if (isNetrisNatted(getProvider(), getNetworkMode())) { + services.add(Vpn.getName()); + } return services; } } @@ -418,6 +423,9 @@ else if (NetworkOffering.NetworkMode.NATTED.name().equalsIgnoreCase(getNetworkMo serviceProviderMap.remove(Lb.getName()); } } + if (isNetrisNatted(getProvider(), getNetworkMode())) { + serviceProviderMap.put(Vpn.getName(), List.of(routerProvider)); + } } public Map getServiceCapabilities(Service service) { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java index d60c627163b8..fd18e44754a4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportUnmanagedInstanceCmd.java @@ -115,7 +115,7 @@ public class ImportUnmanagedInstanceCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.TEMPLATE_ID, type = CommandType.UUID, entityType = TemplateResponse.class, - description = "The ID of the Template for the Instance") + description = "The ID of the Template for the Instance. Required when the instance to import is an unmanaged KVM Instance.") private Long templateId; @Parameter(name = ApiConstants.SERVICE_OFFERING_ID, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java index db7dcc3fb44f..a32b8dd604e7 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ImportVmCmd.java @@ -92,7 +92,7 @@ public class ImportVmCmd extends ImportUnmanagedInstanceCmd { @Parameter(name = ApiConstants.DISK_PATH, type = CommandType.STRING, - description = "path of the disk image") + description = "path of the disk image. It is the file name on file based storage pools (NFS, Local, SharedMountPoint), and the image name on RBD storage pools") private String diskPath; @Parameter(name = ApiConstants.IMPORT_SOURCE, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java index 57c3ee586d35..9f29c0e9a6df 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/volume/ImportVolumeCmd.java @@ -63,7 +63,7 @@ public class ImportVolumeCmd extends BaseAsyncCmd { @Parameter(name = ApiConstants.PATH, type = BaseCmd.CommandType.STRING, required = true, - description = "the path of the volume") + description = "the path of the volume. It is the file name on file based storage pools (NFS, Local, SharedMountPoint), and the image name on RBD storage pools") private String path; @Parameter(name = ApiConstants.NAME, diff --git a/api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/CreateVPCOfferingCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/CreateVPCOfferingCmd.java index 608794842ed4..0897a2b1b428 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/CreateVPCOfferingCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/admin/vpc/CreateVPCOfferingCmd.java @@ -24,6 +24,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -61,7 +62,9 @@ import static com.cloud.network.Network.Service.NetworkACL; import static com.cloud.network.Network.Service.UserData; import static com.cloud.network.Network.Service.Gateway; +import static com.cloud.network.Network.Service.Vpn; +import static org.apache.cloudstack.api.command.utils.OfferingUtils.isNetris; import static org.apache.cloudstack.api.command.utils.OfferingUtils.isNetrisNatted; import static org.apache.cloudstack.api.command.utils.OfferingUtils.isNetrisRouted; import static org.apache.cloudstack.api.command.utils.OfferingUtils.isNsxWithoutLb; @@ -189,7 +192,7 @@ public List getSupportedServices() { NetworkACL.getName(), UserData.getName() )); - if (NetworkOffering.NetworkMode.NATTED.name().equalsIgnoreCase(getNetworkMode())) { + if (isNetrisNatted(getProvider(), getNetworkMode())) { supportedServices.addAll(Arrays.asList( StaticNat.getName(), SourceNat.getName(), @@ -198,9 +201,13 @@ public List getSupportedServices() { if (NetworkOffering.NetworkMode.ROUTED.name().equalsIgnoreCase(getNetworkMode())) { supportedServices.add(Gateway.getName()); } + if (getNsxSupportsLbService() || isNetrisNatted(getProvider(), getNetworkMode())) { supportedServices.add(Lb.getName()); } + if (isNetrisNatted(getProvider(), getNetworkMode())) { + supportedServices.add(Vpn.getName()); + } } return supportedServices; } @@ -210,7 +217,7 @@ public String getProvider() { } public String getNetworkMode() { - return networkMode; + return isNetris(getProvider()) && Objects.isNull(networkMode) ? NetworkOffering.NetworkMode.NATTED.name() : networkMode; } public boolean getNsxSupportsLbService() { @@ -248,7 +255,7 @@ public Map> getServiceProviders() { private void getServiceProviderMapForExternalProvider(Map> serviceProviderMap, String provider) { List unsupportedServices = new ArrayList<>(List.of("Vpn", "BaremetalPxeService", "SecurityGroup", "Connectivity", "Firewall")); - if (NetworkOffering.NetworkMode.NATTED.name().equalsIgnoreCase(getNetworkMode())) { + if (isNetrisNatted(getProvider(), getNetworkMode())) { unsupportedServices.add("Gateway"); } List routerSupported = List.of("Dhcp", "Dns", "UserData"); @@ -258,7 +265,7 @@ private void getServiceProviderMapForExternalProvider(Map> continue; if (routerSupported.contains(service)) serviceProviderMap.put(service, List.of(VirtualRouterProvider.Type.VPCVirtualRouter.name())); - else if (NetworkOffering.NetworkMode.NATTED.name().equalsIgnoreCase(getNetworkMode()) || + else if (isNetrisNatted(getProvider(), getNetworkMode()) || Stream.of(NetworkACL.getName(), Gateway.getName()).anyMatch(s -> s.equalsIgnoreCase(service))) { serviceProviderMap.put(service, List.of(provider)); } @@ -268,7 +275,20 @@ else if (NetworkOffering.NetworkMode.NATTED.name().equalsIgnoreCase(getNetworkMo } } + @SuppressWarnings("unchecked") public Map> getServiceCapabilityList() { + // For Netris NATTED, Gateway service is excluded so the redundant-router capability + // must reference SourceNat (not Gateway). + if (serviceCapabilityList != null + && isNetrisNatted(getProvider(), getNetworkMode())) { + for (Object capEntry : serviceCapabilityList.values()) { + Map cap = (Map) capEntry; + if (Network.Capability.RedundantRouter.getName().equalsIgnoreCase(cap.get("capabilitytype")) + && Gateway.getName().equalsIgnoreCase(cap.get("service"))) { + cap.put("service", SourceNat.getName()); + } + } + } return serviceCapabilityList; } diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java index 6192bfb4540a..7390d4d2da71 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vm/CreateVMFromBackupCmd.java @@ -70,6 +70,11 @@ public class CreateVMFromBackupCmd extends BaseDeployVMCmd { @Parameter(name = ApiConstants.PRESERVE_IP, type = CommandType.BOOLEAN, description = "Use the same IP/MAC addresses as stored in the backup metadata. Works only if the original Instance is deleted and the IP/MAC address is available.") private Boolean preserveIp; + @Parameter(name = ApiConstants.RESET_PASSWORD, type = CommandType.BOOLEAN, + description = "For a password enabled template, whether to generate a new password for the created Instance and return it in the response. " + + "If not specified, the zone setting `restore.vm.from.backup.reset.password` decides.", since = "4.22.1.0") + private Boolean resetPassword; + ///////////////////////////////////////////////////// /////////////////// Accessors /////////////////////// ///////////////////////////////////////////////////// @@ -90,6 +95,10 @@ public boolean getPreserveIp() { return (preserveIp != null) ? preserveIp : false; } + public Boolean getResetPassword() { + return resetPassword; + } + @Override public void create() { UserVm vm; diff --git a/api/src/main/java/org/apache/cloudstack/api/command/user/vmsnapshot/CreateVMSnapshotCmd.java b/api/src/main/java/org/apache/cloudstack/api/command/user/vmsnapshot/CreateVMSnapshotCmd.java index 6e1a7daf4c23..f2decf21a1db 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/user/vmsnapshot/CreateVMSnapshotCmd.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/user/vmsnapshot/CreateVMSnapshotCmd.java @@ -37,7 +37,8 @@ import com.cloud.vm.VirtualMachine; import com.cloud.vm.snapshot.VMSnapshot; -@APICommand(name = "createVMSnapshot", description = "Creates Snapshot for an Instance.", responseObject = VMSnapshotResponse.class, since = "4.2.0", entityType = {VMSnapshot.class}, +@APICommand(name = "createVMSnapshot", description = "Creates Snapshot for an Instance. Running KVM UEFI disk-only snapshots briefly suspend the Instance while copying NVRAM state.", + responseObject = VMSnapshotResponse.class, since = "4.2.0", entityType = {VMSnapshot.class}, requestHasSensitiveInfo = false, responseHasSensitiveInfo = false) public class CreateVMSnapshotCmd extends BaseAsyncCreateCmd { diff --git a/api/src/main/java/org/apache/cloudstack/api/command/utils/OfferingUtils.java b/api/src/main/java/org/apache/cloudstack/api/command/utils/OfferingUtils.java index 433a37c07cde..3885c30f0af4 100644 --- a/api/src/main/java/org/apache/cloudstack/api/command/utils/OfferingUtils.java +++ b/api/src/main/java/org/apache/cloudstack/api/command/utils/OfferingUtils.java @@ -23,8 +23,12 @@ public class OfferingUtils { private OfferingUtils() { } + public static boolean isNetris(String provider) { + return "Netris".equalsIgnoreCase(provider); + } + public static boolean isNetrisNatted(String provider, String networkMode) { - return "Netris".equalsIgnoreCase(provider) && + return isNetris(provider) && NetworkOffering.NetworkMode.NATTED.name().equalsIgnoreCase(networkMode); } @@ -33,6 +37,6 @@ public static boolean isNsxWithoutLb(String provider, boolean nsxSupportsLbServi } public static boolean isNetrisRouted(String provider, String networkMode) { - return "Netris".equalsIgnoreCase(provider) && NetworkOffering.NetworkMode.ROUTED.name().equalsIgnoreCase(networkMode); + return isNetris(provider) && NetworkOffering.NetworkMode.ROUTED.name().equalsIgnoreCase(networkMode); } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ImageStoreDetailResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ImageStoreDetailResponse.java index 0afef6166f8b..036f5f0d170b 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ImageStoreDetailResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ImageStoreDetailResponse.java @@ -16,6 +16,8 @@ // under the License. package org.apache.cloudstack.api.response; +import java.util.Objects; + import com.google.gson.annotations.SerializedName; import org.apache.cloudstack.api.BaseResponse; @@ -81,7 +83,7 @@ public boolean equals(Object obj) { return false; } else if (!oid.equals(other.getName())) return false; - else if (this.getValue().equals(other.getValue())) + else if (!Objects.equals(this.getValue(), other.getValue())) return false; return true; } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/NetworkOfferingResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/NetworkOfferingResponse.java index 87f960590283..740dd23604a9 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/NetworkOfferingResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/NetworkOfferingResponse.java @@ -91,6 +91,10 @@ public class NetworkOfferingResponse extends BaseResponseWithAnnotations { @Param(description = "The ID of the service offering used by virtual router provider") private String serviceOfferingId; + @SerializedName(ApiConstants.SERVICE_OFFERING_NAME) + @Param(description = "the name of the service offering used by virtual router provider") + private String serviceOfferingName; + @SerializedName(ApiConstants.SERVICE) @Param(description = "The list of supported services", responseObject = ServiceResponse.class) private List services; @@ -330,4 +334,12 @@ public String getRoutingMode() { public void setRoutingMode(String routingMode) { this.routingMode = routingMode; } + + public String getServiceOfferingName() { + return serviceOfferingName; + } + + public void setServiceOfferingName(String serviceOfferingName) { + this.serviceOfferingName = serviceOfferingName; + } } diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ProjectResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ProjectResponse.java index 40f9405d0fc5..135dcf97affc 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ProjectResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ProjectResponse.java @@ -452,7 +452,7 @@ public void setNetworkAvailable(String networkAvailable) { @Override public void setVpcLimit(String vpcLimit) { - this.vpcLimit = networkLimit; + this.vpcLimit = vpcLimit; } @Override diff --git a/api/src/main/java/org/apache/cloudstack/api/response/VpcOfferingResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/VpcOfferingResponse.java index a0516e660e48..6c26a45b4826 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/VpcOfferingResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/VpcOfferingResponse.java @@ -62,6 +62,10 @@ public class VpcOfferingResponse extends BaseResponse { @Param(description = "Indicates if the VPC offering supports distributed router for one-hop forwarding", since = "4.4") private Boolean supportsDistributedRouter; + @SerializedName(ApiConstants.REDUNDANT_ROUTER) + @Param(description = "Indicates if the VPC offering supports redundant routers") + private Boolean redundantRouter; + @SerializedName((ApiConstants.SUPPORTS_REGION_LEVEL_VPC)) @Param(description = "Indicated if the offering can support region level VPC", since = "4.4") private Boolean supportsRegionLevelVpc; @@ -201,4 +205,8 @@ public String getRoutingMode() { public void setRoutingMode(String routingMode) { this.routingMode = routingMode; } + + public void setRedundantRouter(Boolean redundantRouter) { + this.redundantRouter = redundantRouter; + } } diff --git a/api/src/main/java/org/apache/cloudstack/backup/Backup.java b/api/src/main/java/org/apache/cloudstack/backup/Backup.java index 951af9180e7f..04b71eebf14d 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/Backup.java +++ b/api/src/main/java/org/apache/cloudstack/backup/Backup.java @@ -31,7 +31,11 @@ public interface Backup extends ControlledEntity, InternalIdentity, Identity { enum Status { - Allocated, Queued, BackingUp, BackedUp, Error, Failed, Restoring, Removed, Expunged + Allocated, Queued, BackingUp, BackedUp, Error, Failed, Restoring, Removed, Expunged, + // Hidden: a chain backup kept as a tombstone after the user deleted it while it still has + // live descendants (incremental chains). Excluded from listBackups and from all backup + // operations (which require BackedUp); swept from the DB once its last descendant is gone. + Hidden } class Metric { diff --git a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java index 32a714370dfc..ed620cbb8fb9 100644 --- a/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java +++ b/api/src/main/java/org/apache/cloudstack/backup/BackupProvider.java @@ -87,6 +87,18 @@ public interface BackupProvider { */ boolean deleteBackup(Backup backup, boolean forced); + /** + * Whether {@link #deleteBackup(Backup, boolean)} owns DB-row removal and resource-count / + * usage accounting for every backup it physically removes. Providers that manage incremental + * chains (e.g. NAS) delete several backups per call — the leaf plus swept delete-pending + * ancestors — and decrement once per removed backup themselves, so the manager must NOT + * decrement or remove the row again. Defaults to {@code false}: the manager does the + * single-backup accounting (the historical behaviour for non-chain providers). + */ + default boolean handlesChainDeleteResourceAccounting() { + return false; + } + Pair restoreBackupToVM(VirtualMachine vm, Backup backup, String hostIp, String dataStoreUuid); /** diff --git a/api/src/test/java/com/cloud/network/IsolationMethodTest.java b/api/src/test/java/com/cloud/network/IsolationMethodTest.java index 2db3ec55db24..dd08f46b5937 100644 --- a/api/src/test/java/com/cloud/network/IsolationMethodTest.java +++ b/api/src/test/java/com/cloud/network/IsolationMethodTest.java @@ -25,7 +25,7 @@ public class IsolationMethodTest { @After public void cleanTheRegistry() { - PhysicalNetwork.IsolationMethod.registeredIsolationMethods.removeAll(PhysicalNetwork.IsolationMethod.registeredIsolationMethods); + PhysicalNetwork.IsolationMethod.registeredIsolationMethods.clear(); } @Test diff --git a/api/src/test/java/org/apache/cloudstack/api/response/ImageStoreDetailResponseTest.java b/api/src/test/java/org/apache/cloudstack/api/response/ImageStoreDetailResponseTest.java new file mode 100644 index 000000000000..8550eac9fc1d --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/response/ImageStoreDetailResponseTest.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import org.junit.Assert; +import org.junit.Test; + +public class ImageStoreDetailResponseTest { + + @Test + public void equalsIsTrueForSameNameAndValue() { + ImageStoreDetailResponse a = new ImageStoreDetailResponse("key", "value"); + ImageStoreDetailResponse b = new ImageStoreDetailResponse("key", "value"); + Assert.assertEquals(a, b); + Assert.assertEquals(a.hashCode(), b.hashCode()); + } + + @Test + public void equalsIsFalseWhenValueDiffers() { + ImageStoreDetailResponse a = new ImageStoreDetailResponse("key", "value"); + ImageStoreDetailResponse c = new ImageStoreDetailResponse("key", "other"); + Assert.assertNotEquals(a, c); + } + + @Test + public void equalsIsFalseWhenNameDiffers() { + ImageStoreDetailResponse a = new ImageStoreDetailResponse("key", "value"); + ImageStoreDetailResponse d = new ImageStoreDetailResponse("other", "value"); + Assert.assertNotEquals(a, d); + } +} diff --git a/api/src/test/java/org/apache/cloudstack/api/response/ProjectResponseTest.java b/api/src/test/java/org/apache/cloudstack/api/response/ProjectResponseTest.java new file mode 100644 index 000000000000..a6f5caa4a76a --- /dev/null +++ b/api/src/test/java/org/apache/cloudstack/api/response/ProjectResponseTest.java @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.api.response; + +import org.junit.Assert; +import org.junit.Test; +import org.springframework.test.util.ReflectionTestUtils; + +public class ProjectResponseTest { + + @Test + public void setVpcLimitStoresItsOwnParameterNotTheNetworkLimit() { + ProjectResponse response = new ProjectResponse(); + response.setNetworkLimit("5"); + response.setVpcLimit("10"); + + Assert.assertEquals("10", ReflectionTestUtils.getField(response, "vpcLimit")); + Assert.assertEquals("5", ReflectionTestUtils.getField(response, "networkLimit")); + } +} diff --git a/core/src/main/java/com/cloud/agent/api/storage/CreateDiskOnlyVmSnapshotAnswer.java b/core/src/main/java/com/cloud/agent/api/storage/CreateDiskOnlyVmSnapshotAnswer.java index 4d61249c7cbc..ffa4eaff2963 100644 --- a/core/src/main/java/com/cloud/agent/api/storage/CreateDiskOnlyVmSnapshotAnswer.java +++ b/core/src/main/java/com/cloud/agent/api/storage/CreateDiskOnlyVmSnapshotAnswer.java @@ -27,13 +27,24 @@ public class CreateDiskOnlyVmSnapshotAnswer extends Answer { protected Map> mapVolumeToSnapshotSizeAndNewVolumePath; + private String nvramSnapshotPath; public CreateDiskOnlyVmSnapshotAnswer(Command command, boolean success, String details, Map> mapVolumeToSnapshotSizeAndNewVolumePath) { + this(command, success, details, mapVolumeToSnapshotSizeAndNewVolumePath, null); + } + + public CreateDiskOnlyVmSnapshotAnswer(Command command, boolean success, String details, Map> mapVolumeToSnapshotSizeAndNewVolumePath, + String nvramSnapshotPath) { super(command, success, details); this.mapVolumeToSnapshotSizeAndNewVolumePath = mapVolumeToSnapshotSizeAndNewVolumePath; + this.nvramSnapshotPath = nvramSnapshotPath; } public Map> getMapVolumeToSnapshotSizeAndNewVolumePath() { return mapVolumeToSnapshotSizeAndNewVolumePath; } + + public String getNvramSnapshotPath() { + return nvramSnapshotPath; + } } diff --git a/core/src/main/java/com/cloud/agent/api/storage/CreateDiskOnlyVmSnapshotCommand.java b/core/src/main/java/com/cloud/agent/api/storage/CreateDiskOnlyVmSnapshotCommand.java index 952bf0c971de..7f328bab81db 100644 --- a/core/src/main/java/com/cloud/agent/api/storage/CreateDiskOnlyVmSnapshotCommand.java +++ b/core/src/main/java/com/cloud/agent/api/storage/CreateDiskOnlyVmSnapshotCommand.java @@ -29,13 +29,30 @@ public class CreateDiskOnlyVmSnapshotCommand extends VMSnapshotBaseCommand { protected VirtualMachine.State vmState; + private final String vmUuid; + private final boolean uefiEnabled; public CreateDiskOnlyVmSnapshotCommand(String vmName, VMSnapshotTO snapshot, List volumeTOs, String guestOSType, VirtualMachine.State vmState) { + this(vmName, null, snapshot, volumeTOs, guestOSType, vmState, false); + } + + public CreateDiskOnlyVmSnapshotCommand(String vmName, String vmUuid, VMSnapshotTO snapshot, List volumeTOs, String guestOSType, + VirtualMachine.State vmState, boolean uefiEnabled) { super(vmName, snapshot, volumeTOs, guestOSType); + this.vmUuid = vmUuid; this.vmState = vmState; + this.uefiEnabled = uefiEnabled; } public VirtualMachine.State getVmState() { return vmState; } + + public String getVmUuid() { + return vmUuid; + } + + public boolean isUefiEnabled() { + return uefiEnabled; + } } diff --git a/core/src/main/java/com/cloud/agent/api/storage/DeleteDiskOnlyVmSnapshotCommand.java b/core/src/main/java/com/cloud/agent/api/storage/DeleteDiskOnlyVmSnapshotCommand.java index bf7bdd597360..1ac0a0ff719b 100644 --- a/core/src/main/java/com/cloud/agent/api/storage/DeleteDiskOnlyVmSnapshotCommand.java +++ b/core/src/main/java/com/cloud/agent/api/storage/DeleteDiskOnlyVmSnapshotCommand.java @@ -19,24 +19,43 @@ package com.cloud.agent.api.storage; import com.cloud.agent.api.Command; - import com.cloud.agent.api.to.DataTO; - +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import java.util.List; public class DeleteDiskOnlyVmSnapshotCommand extends Command { - List snapshots; + private final List snapshots; + private final String nvramSnapshotPath; + private final PrimaryDataStoreTO primaryDataStore; public DeleteDiskOnlyVmSnapshotCommand(List snapshots) { + this(snapshots, null); + } + + public DeleteDiskOnlyVmSnapshotCommand(List snapshots, String nvramSnapshotPath) { + this(snapshots, nvramSnapshotPath, null); + } + + public DeleteDiskOnlyVmSnapshotCommand(List snapshots, String nvramSnapshotPath, PrimaryDataStoreTO primaryDataStore) { this.snapshots = snapshots; + this.nvramSnapshotPath = nvramSnapshotPath; + this.primaryDataStore = primaryDataStore; } public List getSnapshots() { return snapshots; } + public String getNvramSnapshotPath() { + return nvramSnapshotPath; + } + + public PrimaryDataStoreTO getPrimaryDataStore() { + return primaryDataStore; + } + @Override public boolean executeInSequence() { return false; diff --git a/core/src/main/java/com/cloud/agent/api/storage/RevertDiskOnlyVmSnapshotCommand.java b/core/src/main/java/com/cloud/agent/api/storage/RevertDiskOnlyVmSnapshotCommand.java index 72bb92bcb10d..3c9859aa44af 100644 --- a/core/src/main/java/com/cloud/agent/api/storage/RevertDiskOnlyVmSnapshotCommand.java +++ b/core/src/main/java/com/cloud/agent/api/storage/RevertDiskOnlyVmSnapshotCommand.java @@ -27,11 +27,21 @@ public class RevertDiskOnlyVmSnapshotCommand extends Command { private List snapshotObjectTos; private String vmName; + private final String vmUuid; + private final boolean uefiEnabled; + private final String nvramSnapshotPath; public RevertDiskOnlyVmSnapshotCommand(List snapshotObjectTos, String vmName) { + this(snapshotObjectTos, vmName, null, false, null); + } + + public RevertDiskOnlyVmSnapshotCommand(List snapshotObjectTos, String vmName, String vmUuid, boolean uefiEnabled, String nvramSnapshotPath) { super(); this.snapshotObjectTos = snapshotObjectTos; this.vmName = vmName; + this.vmUuid = vmUuid; + this.uefiEnabled = uefiEnabled; + this.nvramSnapshotPath = nvramSnapshotPath; } public List getSnapshotObjectTos() { @@ -42,6 +52,18 @@ public String getVmName() { return vmName; } + public String getVmUuid() { + return vmUuid; + } + + public boolean isUefiEnabled() { + return uefiEnabled; + } + + public String getNvramSnapshotPath() { + return nvramSnapshotPath; + } + @Override public boolean executeInSequence() { return false; diff --git a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/AbstractConfigItemFacade.java b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/AbstractConfigItemFacade.java index 83dfa2a62caa..64034e444eaf 100644 --- a/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/AbstractConfigItemFacade.java +++ b/core/src/main/java/com/cloud/agent/resource/virtualnetwork/facade/AbstractConfigItemFacade.java @@ -106,7 +106,7 @@ public abstract class AbstractConfigItemFacade { public static AbstractConfigItemFacade getInstance(final Class key) { if (!flyweight.containsKey(key)) { - throw new CloudRuntimeException("Unable to process the configuration for " + key.getClass().getName()); + throw new CloudRuntimeException("Unable to process the configuration for " + key.getName()); } final AbstractConfigItemFacade instance = flyweight.get(key); diff --git a/core/src/main/java/com/cloud/agent/transport/compat/AbstractTOAdaptor.java b/core/src/main/java/com/cloud/agent/transport/compat/AbstractTOAdaptor.java new file mode 100644 index 000000000000..ed5630183ead --- /dev/null +++ b/core/src/main/java/com/cloud/agent/transport/compat/AbstractTOAdaptor.java @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.transport.compat; + +import com.cloud.utils.StringUtils; +import com.cloud.utils.exception.CloudRuntimeException; +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; + +import java.lang.reflect.Type; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * JSON serializer adapter for transport classes (com.cloud.agent.api.to.*) that ensures backward compatibility + * with older Agent versions due to rename of the fields + * (see https://github.com/apache/cloudstack/pull/10514) + * + * This class does not build its own Gson instance: doing so would silently drop whichever exclusion + * strategy (e.g. log redaction) and sibling compat adaptors (for nested TOs) the enclosing Gson was + * configured with. Instead, whoever registers an instance of this class into a GsonBuilder is + * responsible for also calling {@link #initGson(Gson)} with a Gson that (a) carries that same + * exclusion strategy and (b) has adapters registered for any nested TO types that also need field + * renaming, but not for this adaptor's own type (to avoid infinite recursion). See + * {@link com.cloud.serializer.GsonHelper#setDefaultGsonConfig(com.google.gson.GsonBuilder)}. + */ +public class AbstractTOAdaptor implements JsonSerializer { + private Gson gson; + private Map fieldMappings; + + protected AbstractTOAdaptor(String... fields) { + this.fieldMappings = new LinkedHashMap<>(); + for (int i = 0; i + 1 < fields.length; i += 2) { + String sourceField = fields[i]; + String destinationField = fields[i + 1]; + // skip empty fields + if (StringUtils.isBlank(sourceField) || StringUtils.isBlank(destinationField)) { + continue; + } + this.fieldMappings.put(sourceField, destinationField); + } + if (this.fieldMappings.isEmpty()) { + throw new CloudRuntimeException("Field mappings must not be empty"); + } + } + + public void initGson(Gson gson) { + this.gson = gson; + } + + @Override + public JsonElement serialize(T src, Type typeOfSrc, JsonSerializationContext context) { + if (src == null) { + return null; + } + JsonElement tree = gson.toJsonTree(src); + if (tree.isJsonObject()) { + JsonObject obj = tree.getAsJsonObject(); + for (Map.Entry field : fieldMappings.entrySet()) { + String sourceField = field.getKey(); + String destinationField = field.getValue(); + if (obj.has(sourceField)) { + obj.add(destinationField, obj.get(sourceField)); + } + } + } + return tree; + } +} diff --git a/core/src/main/java/com/cloud/agent/transport/compat/DiskTOAdaptor.java b/core/src/main/java/com/cloud/agent/transport/compat/DiskTOAdaptor.java new file mode 100644 index 000000000000..6b45529de05f --- /dev/null +++ b/core/src/main/java/com/cloud/agent/transport/compat/DiskTOAdaptor.java @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.transport.compat; + +import com.cloud.agent.api.to.DiskTO; + +/** + * See {@link AbstractTOAdaptor}. + */ +public class DiskTOAdaptor extends AbstractTOAdaptor { + public DiskTOAdaptor() { + super("details", "_details"); + } +} diff --git a/core/src/main/java/com/cloud/agent/transport/compat/MigrateCommandAdaptor.java b/core/src/main/java/com/cloud/agent/transport/compat/MigrateCommandAdaptor.java new file mode 100644 index 000000000000..ddac8afe5975 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/transport/compat/MigrateCommandAdaptor.java @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.transport.compat; + +import com.cloud.agent.api.MigrateCommand; + +/** + * See {@link AbstractTOAdaptor}. + */ +public class MigrateCommandAdaptor extends AbstractTOAdaptor { + public MigrateCommandAdaptor() { + super("destinationIp", "destIp", "windows", "isWindows", "virtualMachine", "vmTO"); + } +} diff --git a/core/src/main/java/com/cloud/agent/transport/compat/NetworkTOAdaptor.java b/core/src/main/java/com/cloud/agent/transport/compat/NetworkTOAdaptor.java new file mode 100644 index 000000000000..01b0cdb4cdf9 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/transport/compat/NetworkTOAdaptor.java @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.transport.compat; + +import com.cloud.agent.api.to.NetworkTO; + +/** + * See {@link AbstractTOAdaptor}. + */ +public class NetworkTOAdaptor extends AbstractTOAdaptor { + public NetworkTOAdaptor() { + super("securityGroupEnabled", "isSecurityGroupEnabled"); + } +} diff --git a/core/src/main/java/com/cloud/agent/transport/compat/VirtualMachineTOAdaptor.java b/core/src/main/java/com/cloud/agent/transport/compat/VirtualMachineTOAdaptor.java new file mode 100644 index 000000000000..24a8c02a59c5 --- /dev/null +++ b/core/src/main/java/com/cloud/agent/transport/compat/VirtualMachineTOAdaptor.java @@ -0,0 +1,29 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.transport.compat; + +import com.cloud.agent.api.to.VirtualMachineTO; + +/** + * See {@link AbstractTOAdaptor}. + */ +public class VirtualMachineTOAdaptor extends AbstractTOAdaptor { + + public VirtualMachineTOAdaptor() { + super("details", "params"); + } +} diff --git a/core/src/main/java/com/cloud/network/HAProxyConfigurator.java b/core/src/main/java/com/cloud/network/HAProxyConfigurator.java index ff9546529bd8..1879e6ee2a6d 100644 --- a/core/src/main/java/com/cloud/network/HAProxyConfigurator.java +++ b/core/src/main/java/com/cloud/network/HAProxyConfigurator.java @@ -538,7 +538,7 @@ private List getRulesForPool(final LoadBalancerTO lbTO, final LoadBalanc } dstSubRule.add(sb.toString()); if (stickinessSubRule != null) { - sb.append(" cookie ").append(dest.getDestIp().replace(".", "_")).append('-').append(dest.getDestPort()).toString(); + sb.append(" cookie ").append(dest.getDestIp().replace(".", "_")).append('-').append(dest.getDestPort()); dstWithCookieSubRule.add(sb.toString()); } destsAvailable = true; diff --git a/core/src/main/java/com/cloud/resource/RequestWrapper.java b/core/src/main/java/com/cloud/resource/RequestWrapper.java index 54d8b289c8d6..3c375195239c 100644 --- a/core/src/main/java/com/cloud/resource/RequestWrapper.java +++ b/core/src/main/java/com/cloud/resource/RequestWrapper.java @@ -86,7 +86,7 @@ protected CommandWrapper retrieveCommands(final keepCommandClass = commandClass2; } catch (final ClassCastException e) { - throw new CommandNotSupported("No key found for '" + keepCommandClass.getClass() + "' in the Map!"); + throw new CommandNotSupported("No key found for '" + keepCommandClass + "' in the Map!"); } catch (final NullPointerException e) { // Will now traverse all the resource hierarchy. Returning null // is not a problem. diff --git a/core/src/main/java/com/cloud/serializer/GsonHelper.java b/core/src/main/java/com/cloud/serializer/GsonHelper.java index 7de98c08b7e0..7fcfb28f983c 100644 --- a/core/src/main/java/com/cloud/serializer/GsonHelper.java +++ b/core/src/main/java/com/cloud/serializer/GsonHelper.java @@ -21,6 +21,14 @@ import java.util.List; +import com.cloud.agent.api.MigrateCommand; +import com.cloud.agent.api.to.DiskTO; +import com.cloud.agent.api.to.NetworkTO; +import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.agent.transport.compat.DiskTOAdaptor; +import com.cloud.agent.transport.compat.MigrateCommandAdaptor; +import com.cloud.agent.transport.compat.NetworkTOAdaptor; +import com.cloud.agent.transport.compat.VirtualMachineTOAdaptor; import com.cloud.hypervisor.Hypervisor; import org.apache.cloudstack.transport.HypervisorTypeAdaptor; import org.apache.logging.log4j.Logger; @@ -78,6 +86,42 @@ public static Gson setDefaultGsonConfig(GsonBuilder builder) { }.getType(), new NwGroupsCommandTypeAdaptor()); builder.registerTypeAdapter(Storage.StoragePoolType.class, new StoragePoolTypeAdaptor()); builder.registerTypeAdapter(Hypervisor.HypervisorType.class, new HypervisorTypeAdaptor()); + + // added for compatibility purposes, remove after all Agents migrate to the new version + // + // Each compat adaptor below needs a "base" Gson to run its own reflective (pre-rename) + // serialization through, so that nested TOs are renamed too and the exclusion strategy set + // on `builder` (e.g. log redaction) is honoured consistently at every nesting level. That base + // Gson is built incrementally off the same builder, snapshotted (via builder.create()) just + // before each adaptor's own type is registered on it, so it carries every sibling adaptor it + // can nest without ever routing back into itself and recursing forever. + DiskTOAdaptor diskAdaptor = new DiskTOAdaptor(); + NetworkTOAdaptor netAdaptor = new NetworkTOAdaptor(); + VirtualMachineTOAdaptor vmAdaptor = new VirtualMachineTOAdaptor(); + MigrateCommandAdaptor migrateAdaptor = new MigrateCommandAdaptor(); + + // DiskTO and NetworkTO don't nest any other compat TO, so the plain config built so far is + // already the correct base Gson for them. + Gson leafDelegateGson = builder.create(); + diskAdaptor.initGson(leafDelegateGson); + netAdaptor.initGson(leafDelegateGson); + + // VirtualMachineTO nests DiskTO[] and NicTO[] (NicTO extends NetworkTO), so its base Gson needs + // Disk/Network adapters too. registerTypeHierarchyAdapter is used for NetworkTO so that the + // NicTO[]-declared "nics" field is matched via its supertype. + builder.registerTypeAdapter(DiskTO.class, diskAdaptor); + builder.registerTypeHierarchyAdapter(NetworkTO.class, netAdaptor); + Gson vmDelegateGson = builder.create(); + vmAdaptor.initGson(vmDelegateGson); + + // MigrateCommand nests a VirtualMachineTO, so its base Gson needs the VirtualMachineTO adapter + // (which already renames the nested disks/nics above). + builder.registerTypeAdapter(VirtualMachineTO.class, vmAdaptor); + Gson migrateDelegateGson = builder.create(); + migrateAdaptor.initGson(migrateDelegateGson); + + builder.registerTypeAdapter(MigrateCommand.class, migrateAdaptor); + Gson gson = builder.create(); dsAdaptor.initGson(gson); dtAdaptor.initGson(gson); diff --git a/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java b/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java index 65d54fe81992..545169abd001 100644 --- a/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java +++ b/core/src/main/java/org/apache/cloudstack/agent/directdownload/DirectDownloadCommand.java @@ -53,7 +53,6 @@ protected DirectDownloadCommand (final String url, final Long templateId, final final Integer soTimeout, final Integer connectionRequestTimeout, final boolean followRedirects) { this.url = url; this.templateId = templateId; - this.destData = destData; this.destPool = destPool; this.checksum = checksum; this.headers = headers; diff --git a/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java b/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java index ffc67b628a7e..abe78ee5553d 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java +++ b/core/src/main/java/org/apache/cloudstack/backup/BackupAnswer.java @@ -29,6 +29,12 @@ public class BackupAnswer extends Answer { private Long virtualSize; private Map volumes; Boolean needsCleanup; + // Set by the NAS backup provider after a checkpoint/bitmap was created during this backup. + // The provider persists it in backup_details under NASBackupChainKeys.BITMAP_NAME. + private String bitmapCreated; + // Set when an incremental was requested but the agent had to fall back to a full + // (e.g. VM was stopped). Provider should record this backup as type=full. + private Boolean incrementalFallback; public BackupAnswer(final Command command, final boolean success, final String details) { super(command, success, details); @@ -68,4 +74,21 @@ public Boolean getNeedsCleanup() { public void setNeedsCleanup(Boolean needsCleanup) { this.needsCleanup = needsCleanup; } + + public String getBitmapCreated() { + return bitmapCreated; + } + + public void setBitmapCreated(String bitmapCreated) { + this.bitmapCreated = bitmapCreated; + } + + public Boolean getIncrementalFallback() { + return incrementalFallback != null && incrementalFallback; + } + + public void setIncrementalFallback(Boolean incrementalFallback) { + this.incrementalFallback = incrementalFallback; + } + } diff --git a/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java b/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java index 5402b6b24760..7c2d3fc76fc1 100644 --- a/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java +++ b/core/src/main/java/org/apache/cloudstack/backup/TakeBackupCommand.java @@ -33,9 +33,21 @@ public class TakeBackupCommand extends Command { private List volumePools; private List volumePaths; private Boolean quiesce; + private Integer quiesceTimeout; @LogLevel(LogLevel.Log4jLevel.Off) private String mountOptions; + // Incremental backup fields (NAS provider; null/empty for legacy full-only callers). + private String mode; // "full" or "incremental"; null => legacy behaviour (script default) + private String bitmapNew; // Checkpoint/bitmap name to create with this backup (timestamp-based) + private String bitmapParent; // Incremental: parent bitmap to read changes since + + // Per-volume parent backup file paths (one per VM volume, ordered by deviceId — same + // order as volumePaths). The script rebases each new qcow2 onto the matching parent. + // Backup file UUIDs differ across volumes, so a single parentPath would have rebased + // every data disk onto the root file. New callers MUST populate parentPaths. + private List parentPaths; + public TakeBackupCommand(String vmName, String backupPath) { super(); this.vmName = vmName; @@ -106,6 +118,46 @@ public void setQuiesce(Boolean quiesce) { this.quiesce = quiesce; } + public Integer getQuiesceTimeout() { + return quiesceTimeout; + } + + public void setQuiesceTimeout(Integer quiesceTimeout) { + this.quiesceTimeout = quiesceTimeout; + } + + public String getMode() { + return mode; + } + + public void setMode(String mode) { + this.mode = mode; + } + + public String getBitmapNew() { + return bitmapNew; + } + + public void setBitmapNew(String bitmapNew) { + this.bitmapNew = bitmapNew; + } + + public String getBitmapParent() { + return bitmapParent; + } + + public void setBitmapParent(String bitmapParent) { + this.bitmapParent = bitmapParent; + } + + public List getParentPaths() { + return parentPaths; + } + + public void setParentPaths(List parentPaths) { + this.parentPaths = parentPaths; + } + @Override public boolean executeInSequence() { return true; diff --git a/core/src/test/java/com/cloud/agent/transport/RequestTest.java b/core/src/test/java/com/cloud/agent/transport/RequestTest.java index 0fe42c7cede8..d68107e64917 100644 --- a/core/src/test/java/com/cloud/agent/transport/RequestTest.java +++ b/core/src/test/java/com/cloud/agent/transport/RequestTest.java @@ -19,14 +19,21 @@ package com.cloud.agent.transport; +import java.lang.reflect.Field; import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; import junit.framework.TestCase; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.config.Configurator; import org.junit.Assert; import org.mockito.Mockito; +import com.google.gson.Gson; + import org.apache.cloudstack.storage.command.DownloadCommand; import org.apache.cloudstack.storage.to.TemplateObjectTO; @@ -35,19 +42,28 @@ import com.cloud.agent.api.Command; import com.cloud.agent.api.GetHostStatsCommand; import com.cloud.agent.api.GetVolumeStatsCommand; +import com.cloud.agent.api.MigrateCommand; import com.cloud.agent.api.SecStorageFirewallCfgCommand; +import com.cloud.agent.api.StartCommand; import com.cloud.agent.api.UpdateHostPasswordCommand; import com.cloud.agent.api.storage.DownloadAnswer; import com.cloud.agent.api.storage.ListTemplateCommand; +import com.cloud.agent.api.to.DiskTO; import com.cloud.agent.api.to.NfsTO; +import com.cloud.agent.api.to.NicTO; +import com.cloud.agent.api.to.VirtualMachineTO; import com.cloud.agent.transport.Request.Version; import com.cloud.exception.UnsupportedVersionException; +import com.cloud.host.Host; import com.cloud.hypervisor.Hypervisor.HypervisorType; +import com.cloud.serializer.GsonHelper; import com.cloud.storage.DataStoreRole; import com.cloud.storage.Storage.ImageFormat; import com.cloud.storage.Storage.TemplateType; import com.cloud.storage.VMTemplateStorageResourceAssoc.Status; import com.cloud.template.VirtualMachineTemplate; +import com.cloud.template.VirtualMachineTemplate.BootloaderType; +import com.cloud.vm.VirtualMachine; /** * @@ -57,7 +73,28 @@ */ public class RequestTest extends TestCase { - protected Logger logger = LogManager.getLogger(getClass()); + private static final Logger logger = LogManager.getLogger(RequestTest.class); + + /** + * Changes GsonHelper's logging Gson level AND clears its cached TypeAdapters. + * + * Gson bakes each Command class's shouldSkipClass/shouldSkipField decision into a TypeAdapter + * the first time that class is serialized, then caches it (Gson#typeTokenCache) for the life + * of the Gson instance. Since GsonHelper.getGsonLogger() is a static singleton shared for the + * whole test JVM, just calling Configurator.setLevel() again has no effect on classes already + * cached - they'd keep reflecting whichever level was active the first time they were logged. + * Clearing the cache after every level change forces a fresh (correct) evaluation. + */ + private static void setGsonLoggerLevel(String loggerName, Level level) { + Configurator.setLevel(loggerName, level); + try { + Field field = Gson.class.getDeclaredField("typeTokenCache"); + field.setAccessible(true); + ((Map)field.get(GsonHelper.getGsonLogger())).clear(); + } catch (ReflectiveOperationException e) { + throw new RuntimeException(e); + } + } public void testSerDeser() { logger.info("Testing serializing and deserializing works as expected"); @@ -73,6 +110,32 @@ public void testSerDeser() { Request sreq = new Request(2, 3, new Command[] {cmd1, cmd2, cmd3}, true, true); sreq.setSequence(892403717); + Logger gsonLogger = LogManager.getLogger(GsonHelper.class); + Level level = gsonLogger.getLevel(); + + setGsonLoggerLevel(gsonLogger.getName(), Level.DEBUG); + String log = sreq.log("Debug", true, Level.DEBUG); + assert (log.contains(UpdateHostPasswordCommand.class.getSimpleName())); + assert (log.contains(SecStorageFirewallCfgCommand.class.getSimpleName())); + assert (!log.contains(GetHostStatsCommand.class.getSimpleName())); + assert (!log.contains("username")); + assert (!log.contains("password")); + + setGsonLoggerLevel(gsonLogger.getName(), Level.TRACE); + log = sreq.log("Trace", true, Level.TRACE); + System.out.println(log); + assert (log.contains(UpdateHostPasswordCommand.class.getSimpleName())); + assert (log.contains(SecStorageFirewallCfgCommand.class.getSimpleName())); + assert (log.contains(GetHostStatsCommand.class.getSimpleName())); + assert (!log.contains("username")); + assert (!log.contains("password")); + + setGsonLoggerLevel(gsonLogger.getName(), Level.INFO); + log = sreq.log("Info", true, Level.INFO); + assert (log == null); + + setGsonLoggerLevel(GsonHelper.class.getName(), level); + byte[] bytes = sreq.getBytes(); assert Request.getSequence(bytes) == 892403717; @@ -186,6 +249,79 @@ public void testCompress() { } } + public void testLogging() { + logger.info("Testing Logging"); + GetHostStatsCommand cmd3 = new GetHostStatsCommand("hostguid", "hostname", 101); + Request sreq = new Request(2, 3, new Command[] {cmd3}, true, true); + sreq.setSequence(1); + Logger gsonLogger = LogManager.getLogger(GsonHelper.class); + Level level = gsonLogger.getLevel(); + + setGsonLoggerLevel(GsonHelper.class.getName(), Level.DEBUG); + String log = sreq.log("Debug", true, Level.DEBUG); + assert (log == null); + + log = sreq.log("Debug", false, Level.DEBUG); + assert (log != null); + + setGsonLoggerLevel(GsonHelper.class.getName(), Level.TRACE); + log = sreq.log("Trace", true, Level.TRACE); + assert (log != null); + + assert (log.contains(GetHostStatsCommand.class.getSimpleName())); + logger.debug(log); + + setGsonLoggerLevel(GsonHelper.class.getName(), level); + } + + public void testCompatFieldRenamingNestedTOs() { + logger.info("Testing that renamed fields are restored on nested TOs too, for backward compatibility with older Agents"); + + DiskTO diskTO = new DiskTO(); + diskTO.setDetails(new HashMap()); + + NicTO nicTO = new NicTO(); + nicTO.setSecurityGroupEnabled(true); + + VirtualMachineTO vmTO = new VirtualMachineTO(1, "i-2-3-VM", VirtualMachine.Type.User, 1, 512, 512L * 1024 * 1024, 512L * 1024 * 1024, + BootloaderType.HVM, "Other PV (64-bit)", true, true, "vncpassword123"); + vmTO.setDetails(new HashMap()); + vmTO.setDisks(new DiskTO[] {diskTO}); + vmTO.setNics(new NicTO[] {nicTO}); + + Host host = Mockito.mock(Host.class); + Mockito.when(host.getPrivateIpAddress()).thenReturn("10.1.1.1"); + StartCommand startCmd = new StartCommand(vmTO, host, false); + + Request startReq = new Request(1, 1, startCmd, true); + String startWireJson = GsonHelper.getGson().toJson(new Command[] {startCmd}); + assert startWireJson.contains("\"params\"") : "VirtualMachineTO.details should be serialized under its old name 'params'"; + assert startWireJson.contains("\"_details\"") : "nested DiskTO.details should be serialized under its old name '_details'"; + assert startWireJson.contains("\"isSecurityGroupEnabled\"") : "nested NicTO.securityGroupEnabled should be serialized under its old name 'isSecurityGroupEnabled'"; + assert startWireJson.contains("vncpassword123") : "wire serialization should still contain the real vncPassword value"; + + Logger gsonLogger = LogManager.getLogger(GsonHelper.class); + Level gsonLoggerLevel = gsonLogger.getLevel(); + setGsonLoggerLevel(GsonHelper.class.getName(), Level.TRACE); + String startLogJson; + try { + startLogJson = startReq.log("Trace", true, Level.TRACE); + } finally { + setGsonLoggerLevel(GsonHelper.class.getName(), gsonLoggerLevel); + } + assert startLogJson.contains("\"isSecurityGroupEnabled\"") : "renamed fields should still show up in the logging serialization"; + assert !startLogJson.contains("vncpassword123") : "logging serialization should never contain the plaintext vncPassword value"; + + MigrateCommand migrateCmd = new MigrateCommand("i-2-3-VM", "10.1.1.2", true, vmTO, false); + String migrateWireJson = GsonHelper.getGson().toJson(new Command[] {migrateCmd}); + assert migrateWireJson.contains("\"destIp\"") : "MigrateCommand.destinationIp should be serialized under its old name 'destIp'"; + assert migrateWireJson.contains("\"isWindows\"") : "MigrateCommand.windows should be serialized under its old name 'isWindows'"; + assert migrateWireJson.contains("\"vmTO\"") : "MigrateCommand.virtualMachine should be serialized under its old name 'vmTO'"; + assert migrateWireJson.contains("\"params\"") : "VirtualMachineTO nested in MigrateCommand should still be renamed"; + assert migrateWireJson.contains("\"_details\"") : "DiskTO nested inside the VirtualMachineTO nested in MigrateCommand should still be renamed"; + assert migrateWireJson.contains("\"isSecurityGroupEnabled\"") : "NicTO nested inside the VirtualMachineTO nested in MigrateCommand should still be renamed"; + } + protected void compareRequest(Request req1, Request req2) { assert req1.getSequence() == req2.getSequence(); assert req1.getAgentId() == req2.getAgentId(); diff --git a/core/src/test/java/com/cloud/resource/ServerResourceBaseTest.java b/core/src/test/java/com/cloud/resource/ServerResourceBaseTest.java index ed64e1482a62..68bfb5e9b3bf 100644 --- a/core/src/test/java/com/cloud/resource/ServerResourceBaseTest.java +++ b/core/src/test/java/com/cloud/resource/ServerResourceBaseTest.java @@ -182,36 +182,36 @@ public void tryToAutoDiscoverResourcePrivateNetworkInterfaceTestReturnNic() thro @Test public void defineResourceNetworkInterfacesTestUseXenbr0WhenPrivateNetworkInterfaceNotConfigured() { Map params = createParamsMap(null, "cloudbr1", "cloudbr2", "cloudbr3"); - try (MockedStatic ignored = Mockito.mockStatic(NetUtils.class)) { - Mockito.when(NetUtils.getNetworkInterface(Mockito.anyString())).thenReturn(networkInterfaceMock1, networkInterfaceMock2, networkInterfaceMock3, networkInterfaceMock4); + try (MockedStatic mockedNetUtils = Mockito.mockStatic(NetUtils.class)) { + mockedNetUtils.when(() -> NetUtils.getNetworkInterface(Mockito.anyString())).thenReturn(networkInterfaceMock1, networkInterfaceMock2, networkInterfaceMock3, networkInterfaceMock4); serverResourceBaseSpy.defineResourceNetworkInterfaces(params); - verifyAndAssertNetworkInterfaces("xenbr0", "cloudbr1", "cloudbr2", "cloudbr3"); + verifyAndAssertNetworkInterfaces(mockedNetUtils, "xenbr0", "cloudbr1", "cloudbr2", "cloudbr3"); } } @Test public void defineResourceNetworkInterfacesTestUseXenbr1WhenPublicNetworkInterfaceNotConfigured() { Map params = createParamsMap("cloudbr0", null, "cloudbr2", "cloudbr3"); - try (MockedStatic ignored = Mockito.mockStatic(NetUtils.class)) { - Mockito.when(NetUtils.getNetworkInterface(Mockito.anyString())).thenReturn(networkInterfaceMock1, networkInterfaceMock2, networkInterfaceMock3, networkInterfaceMock4); + try (MockedStatic mockedNetUtils = Mockito.mockStatic(NetUtils.class)) { + mockedNetUtils.when(() -> NetUtils.getNetworkInterface(Mockito.anyString())).thenReturn(networkInterfaceMock1, networkInterfaceMock2, networkInterfaceMock3, networkInterfaceMock4); serverResourceBaseSpy.defineResourceNetworkInterfaces(params); - verifyAndAssertNetworkInterfaces("cloudbr0", "xenbr1", "cloudbr2", "cloudbr3"); + verifyAndAssertNetworkInterfaces(mockedNetUtils, "cloudbr0", "xenbr1", "cloudbr2", "cloudbr3"); } } @Test public void defineResourceNetworkInterfacesTestUseConfiguredNetworkInterfaces() { Map params = createParamsMap("cloudbr0", "cloudbr1", "cloudbr2", "cloudbr3"); - try (MockedStatic ignored = Mockito.mockStatic(NetUtils.class)) { - Mockito.when(NetUtils.getNetworkInterface(Mockito.anyString())).thenReturn(networkInterfaceMock1, networkInterfaceMock2, networkInterfaceMock3, networkInterfaceMock4); + try (MockedStatic mockedNetUtils = Mockito.mockStatic(NetUtils.class)) { + mockedNetUtils.when(() -> NetUtils.getNetworkInterface(Mockito.anyString())).thenReturn(networkInterfaceMock1, networkInterfaceMock2, networkInterfaceMock3, networkInterfaceMock4); serverResourceBaseSpy.defineResourceNetworkInterfaces(params); - verifyAndAssertNetworkInterfaces("cloudbr0", "cloudbr1", "cloudbr2", "cloudbr3"); + verifyAndAssertNetworkInterfaces(mockedNetUtils, "cloudbr0", "cloudbr1", "cloudbr2", "cloudbr3"); } } @@ -224,9 +224,8 @@ private Map createParamsMap(String... params) { return result; } - private void verifyAndAssertNetworkInterfaces(String... expectedResults) { - Mockito.verify(NetUtils.class, Mockito.times(4)); - NetUtils.getNetworkInterface(keyCaptor.capture()); + private void verifyAndAssertNetworkInterfaces(MockedStatic mockedNetUtils, String... expectedResults) { + mockedNetUtils.verify(() -> NetUtils.getNetworkInterface(keyCaptor.capture()), Mockito.times(4)); List keys = keyCaptor.getAllValues(); for (int i = 0; i < expectedResults.length; i++) { diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java index 8686e4dd3a5a..fc9245b89704 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/orchestration/service/VolumeOrchestrationService.java @@ -183,10 +183,12 @@ List allocateTemplatedVolumes(Type type, String name, DiskOffering */ DiskProfile importVolume(Type type, String name, DiskOffering offering, Long sizeInBytes, Long minIops, Long maxIops, Long zoneId, HypervisorType hypervisorType, VirtualMachine vm, VirtualMachineTemplate template, - Account owner, Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo); + Account owner, Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, + Storage.ImageFormat format); DiskProfile updateImportedVolume(Type type, DiskOffering offering, VirtualMachine vm, VirtualMachineTemplate template, - Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, DiskProfile diskProfile); + Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, DiskProfile diskProfile, + Storage.ImageFormat format); /** * Unmanage VM volumes diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index 5c7348cbe6c3..7a29ce81002b 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -432,4 +432,8 @@ void connectHostsToPool(DataStore primaryStore, List hostIds, Scope scope, String[] getStorageAccessGroups(Long zoneId, Long podId, Long clusterId, Long hostId); CapacityVO getObjectStorageUsedStats(Long zoneId); + + static ConfigKey getMountDisabledStoragePool() { + return MountDisabledStoragePool; + } } diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java index 0a6bbc876544..402bd2b6b9b9 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java @@ -24,6 +24,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Random; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -361,7 +362,8 @@ public boolean equals(final Object obj) { return false; } AgentAttache that = (AgentAttache)obj; - return _id == that._id; + return Objects.equals(_uuid, that._uuid) && + Objects.equals(_name, that._name); } public void send(final Request req, final Listener listener) throws AgentUnavailableException { @@ -530,11 +532,12 @@ protected synchronized void sendNext(final long seq) { logger.debug(LOG_SEQ_FORMATTED_STRING, req.getSequence(), "Sending now. is current sequence."); try { send(req); + _currentSequence = req.getSequence(); } catch (AgentUnavailableException e) { logger.debug(LOG_SEQ_FORMATTED_STRING, req.getSequence(), "Unable to send the next sequence"); cancel(req.getSequence()); + sendNext(req.getSequence()); } - _currentSequence = req.getSequence(); } public void process(final Answer[] answers) { @@ -561,6 +564,11 @@ public void process(final Answer[] answers) { */ protected abstract boolean isClosed(); + @Override + public int hashCode() { + return Objects.hash(_uuid, _name); + } + protected class Alarm extends ManagedContextRunnable { long _seq; diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java index 8c69dcdc4828..ecb789a15bd5 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java @@ -108,10 +108,12 @@ import com.cloud.exception.UnsupportedVersionException; import com.cloud.ha.HighAvailabilityManager; import com.cloud.host.Host; +import com.cloud.host.DetailVO; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.Status.Event; import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostDetailsDao; import com.cloud.hypervisor.Hypervisor.HypervisorType; import com.cloud.hypervisor.HypervisorGuruManager; import com.cloud.org.Cluster; @@ -167,6 +169,8 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl @Inject protected HostDao _hostDao = null; @Inject + protected HostDetailsDao _hostDetailsDao = null; + @Inject private ManagementServerHostDao _mshostDao; @Inject protected OutOfBandManagementDao outOfBandManagementDao; @@ -802,18 +806,24 @@ protected AgentAttache notifyMonitorsOfConnection(final AgentAttache attache, fi ReadyAnswer readyAnswer = (ReadyAnswer)answer; Map detailsMap = readyAnswer.getDetailsMap(); if (detailsMap != null) { + _hostDao.loadDetails(host); + if (host.getDetails() == null) { + host.setDetails(new HashMap<>()); + } String uefiEnabled = detailsMap.get(Host.HOST_UEFI_ENABLE); + String diskOnlyVmSnapshotNvramSupport = detailsMap.get(Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM); String virtv2vVersion = detailsMap.get(Host.HOST_VIRTV2V_VERSION); String ovftoolVersion = detailsMap.get(Host.HOST_OVFTOOL_VERSION); String vddkSupport = detailsMap.get(Host.HOST_VDDK_SUPPORT); String vddkLibDir = detailsMap.get(Host.HOST_VDDK_LIB_DIR); String vddkVersion = detailsMap.get(Host.HOST_VDDK_VERSION); logger.debug("Got HOST_UEFI_ENABLE [{}] for host [{}]:", uefiEnabled, host); - if (ObjectUtils.anyNotNull(uefiEnabled, virtv2vVersion, ovftoolVersion, vddkSupport, vddkLibDir, vddkVersion)) { - _hostDao.loadDetails(host); + if (ObjectUtils.anyNotNull(uefiEnabled, diskOnlyVmSnapshotNvramSupport, virtv2vVersion, ovftoolVersion, vddkSupport, vddkLibDir, vddkVersion)) { boolean updateNeeded = false; - if (StringUtils.isNotBlank(uefiEnabled) && !uefiEnabled.equals(host.getDetails().get(Host.HOST_UEFI_ENABLE))) { - host.getDetails().put(Host.HOST_UEFI_ENABLE, uefiEnabled); + if (syncBooleanHostCapability(host, Host.HOST_UEFI_ENABLE, uefiEnabled)) { + updateNeeded = true; + } + if (syncBooleanHostCapability(host, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM, diskOnlyVmSnapshotNvramSupport)) { updateNeeded = true; } if (StringUtils.isNotBlank(virtv2vVersion) && !virtv2vVersion.equals(host.getDetails().get(Host.HOST_VIRTV2V_VERSION))) { @@ -856,6 +866,26 @@ protected AgentAttache notifyMonitorsOfConnection(final AgentAttache attache, fi return attache; } + protected boolean syncBooleanHostCapability(HostVO host, String capabilityName, String advertisedValue) { + if (StringUtils.isNotBlank(advertisedValue)) { + if (!advertisedValue.equals(host.getDetails().get(capabilityName))) { + host.getDetails().put(capabilityName, advertisedValue); + return true; + } + return false; + } + + if (host.getDetails().containsKey(capabilityName)) { + host.getDetails().remove(capabilityName); + DetailVO hostDetail = _hostDetailsDao.findDetail(host.getId(), capabilityName); + if (hostDetail != null) { + _hostDetailsDao.remove(hostDetail.getId()); + } + return true; + } + return false; + } + @Override public boolean start() { ManagementServerHostVO msHost = _mshostDao.findByMsid(_nodeId); diff --git a/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java b/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java index ed18d1e82b7e..2edc9ad19bc3 100644 --- a/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java +++ b/engine/orchestration/src/main/java/com/cloud/agent/manager/DirectAgentAttache.java @@ -150,6 +150,11 @@ private synchronized void scheduleFromQueue() { } } + @Override + public int hashCode() { + return super.hashCode(); + } + protected class PingTask extends ManagedContextRunnable { @Override protected synchronized void runInContext() { diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java index 4bbb73014cf5..c98391a654db 100755 --- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java +++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java @@ -1630,9 +1630,10 @@ public void orchestrateStart(final String vmUuid, final Map> getVolumesToDisconnect(VirtualMachine vm) { return volumesToDisconnect; } - protected boolean sendStop(final VirtualMachineGuru guru, final VirtualMachineProfile profile, final boolean force, final boolean checkBeforeCleanup) { + protected Pair sendStop(final VirtualMachineGuru guru, final VirtualMachineProfile profile, final boolean force, final boolean checkBeforeCleanup) { final VirtualMachine vm = profile.getVirtualMachine(); Map vlanToPersistenceMap = getVlanToPersistenceMapForVM(vm.getId()); StopCommand stpCmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), checkBeforeCleanup); @@ -2241,7 +2242,7 @@ protected boolean sendStop(final VirtualMachineGuru guru, final VirtualMachinePr if (!answer.getResult()) { final String details = answer.getDetails(); logger.debug("Unable to stop VM due to {}", details); - return false; + return new Pair<>(false, details); } guru.finalizeStop(profile, answer); @@ -2254,21 +2255,23 @@ protected boolean sendStop(final VirtualMachineGuru guru, final VirtualMachinePr } } } else { - logger.error("Invalid answer received in response to a StopCommand for {}", vm.getInstanceName()); - return false; + String errorMsg = String.format("Invalid answer received in response to a StopCommand for %s", vm.getInstanceName()); + logger.error(errorMsg); + return new Pair<>(false, errorMsg); } } catch (final AgentUnavailableException | OperationTimedoutException e) { - logger.warn("Unable to stop {} due to [{}].", vm.toString(), e.getMessage(), e); + String errorMsg = String.format("Unable to stop %s due to [%s].", vm.toString(), e.getMessage()); + logger.warn(errorMsg, e); if (!force) { - return false; + return new Pair<>(false, errorMsg); } } - return true; + return new Pair<>(true, null); } - protected boolean cleanup(final VirtualMachineGuru guru, final VirtualMachineProfile profile, final ItWorkVO work, final Event event, final boolean cleanUpEvenIfUnableToStop) { + protected Pair cleanup(final VirtualMachineGuru guru, final VirtualMachineProfile profile, final ItWorkVO work, final Event event, final boolean cleanUpEvenIfUnableToStop) { final VirtualMachine vm = profile.getVirtualMachine(); final State state = vm.getState(); logger.debug("Cleaning up resources for the vm {} in {} state", vm, state); @@ -2277,57 +2280,63 @@ protected boolean cleanup(final VirtualMachineGuru guru, final VirtualMachinePro if (work != null) { final Step step = work.getStep(); if (step == Step.Starting && !cleanUpEvenIfUnableToStop) { - logger.warn("Unable to cleanup vm {}; work state is incorrect: {}", vm, step); - return false; + String errorMsg = String.format("Unable to cleanup vm %s; work state is incorrect: %s", vm, step); + logger.warn(errorMsg); + return new Pair<>(false, errorMsg); } if (step == Step.Started || step == Step.Starting || step == Step.Release) { if (vm.getHostId() != null) { - if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop, false)) { + Pair result = sendStop(guru, profile, cleanUpEvenIfUnableToStop, false); + if (!result.first()) { logger.warn("Failed to stop vm {} in {} state as a part of cleanup process", vm, State.Starting); - return false; + return result; } } } if (step != Step.Release && step != Step.Prepare && step != Step.Started && step != Step.Starting) { logger.debug("Cleanup is not needed for vm {}; work state is incorrect: {}", vm, step); - return true; + return new Pair<>(true, null); } } else { if (vm.getHostId() != null) { - if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop, false)) { + Pair result = sendStop(guru, profile, cleanUpEvenIfUnableToStop, false); + if (!result.first()) { logger.warn("Failed to stop vm {} in {} state as a part of cleanup process", vm, State.Starting); - return false; + return result; } } } } else if (state == State.Stopping) { if (vm.getHostId() != null) { - if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop, false)) { + Pair result = sendStop(guru, profile, cleanUpEvenIfUnableToStop, false); + if (!result.first()) { logger.warn("Failed to stop vm {} in {} state as a part of cleanup process", vm, State.Stopping); - return false; + return result; } } } else if (state == State.Migrating) { if (vm.getHostId() != null || vm.getLastHostId() != null) { - if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop, false)) { + Pair result = sendStop(guru, profile, cleanUpEvenIfUnableToStop, false); + if (!result.first()) { logger.warn("Failed to stop vm {} in {} state as a part of cleanup process", vm, State.Migrating); - return false; + return result; } } } else if (state == State.Running) { - if (!sendStop(guru, profile, cleanUpEvenIfUnableToStop, false)) { + Pair result = sendStop(guru, profile, cleanUpEvenIfUnableToStop, false); + if (!result.first()) { logger.warn("Failed to stop vm {} in {} state as a part of cleanup process", vm, State.Running); - return false; + return result; } } } finally { releaseVmResources(profile, cleanUpEvenIfUnableToStop); } - return true; + return new Pair<>(true, null); } protected void releaseVmResources(final VirtualMachineProfile profile, final boolean forced) { @@ -2509,7 +2518,8 @@ private void advanceStop(final VMInstanceVO vm, final boolean cleanUpEvenIfUnabl logger.warn("Unable to transition the state but we're moving on because it's forced stop", e1); if (doCleanup) { - if (cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.StopRequested, cleanUpEvenIfUnableToStop)) { + Pair cleanupResult = cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.StopRequested, cleanUpEvenIfUnableToStop); + if (cleanupResult.first()) { try { if (work != null) { logger.debug("Updating work item to Done, id: {}", work.getId()); @@ -2524,7 +2534,8 @@ private void advanceStop(final VMInstanceVO vm, final boolean cleanUpEvenIfUnabl } } else { logger.debug("Failed to cleanup VM: {}", vm); - throw new CloudRuntimeException("Failed to cleanup " + vm + " , current state " + vm.getState()); + String errorDetails = cleanupResult.second() != null ? " due to " + cleanupResult.second() : ""; + throw new CloudRuntimeException("Failed to cleanup " + vm + " , current state " + vm.getState() + errorDetails); } } } @@ -2545,6 +2556,7 @@ private void advanceStop(final VMInstanceVO vm, final boolean cleanUpEvenIfUnabl boolean stopped = false; Answer answer = null; + String agentExceptionDetail = null; try { answer = _agentMgr.send(vm.getHostId(), stop); if (answer != null) { @@ -2572,6 +2584,7 @@ private void advanceStop(final VMInstanceVO vm, final boolean cleanUpEvenIfUnabl } } catch (AgentUnavailableException | OperationTimedoutException e) { + agentExceptionDetail = e.getMessage(); logger.warn("Unable to stop {} due to [{}].", profile.toString(), e.toString(), e); } finally { if (!stopped) { @@ -2582,7 +2595,9 @@ private void advanceStop(final VMInstanceVO vm, final boolean cleanUpEvenIfUnabl } catch (final NoTransitionException e) { logger.warn("Unable to transition the state " + vm, e); } - throw new CloudRuntimeException("Unable to stop " + vm); + String errorDetail = (answer != null && answer.getDetails() != null) ? answer.getDetails() : agentExceptionDetail; + String errorDetails = errorDetail != null ? " due to " + errorDetail : ""; + throw new CloudRuntimeException("Unable to stop " + vm + errorDetails); } else { logger.warn("Unable to actually stop {} but continue with release because it's a force stop", vm); vmGuru.finalizeStop(profile, answer); @@ -3261,8 +3276,9 @@ protected void migrate(final VMInstanceVO vm, final long srcHostId, final Deploy } catch (final AgentUnavailableException e) { logger.error("AgentUnavailableException while cleanup on source host: {}", fromHost, e); } - cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.AgentReportStopped, true); - throw new CloudRuntimeException("Unable to complete migration for " + vm); + Pair cleanupResult = cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.AgentReportStopped, true); + String errorDetails = (cleanupResult.second() != null) ? " due to " + cleanupResult.second() : ""; + throw new CloudRuntimeException("Unable to complete migration for " + vm + errorDetails); } } catch (final OperationTimedoutException e) { logger.warn("Error while checking the vm {} on host {}", vm, dest.getHost(), e); @@ -3718,8 +3734,9 @@ private void orchestrateMigrateWithStorage(final String vmUuid, final long srcHo } catch (final AgentUnavailableException e) { logger.error("AgentUnavailableException while cleanup on source host: {}", srcHost, e); } - cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.AgentReportStopped, true); - throw new CloudRuntimeException("VM not found on destination host. Unable to complete migration for " + vm); + Pair cleanupResult = cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.AgentReportStopped, true); + String errorDetails = (cleanupResult.second() != null) ? " due to " + cleanupResult.second() : ""; + throw new CloudRuntimeException("VM not found on destination host. Unable to complete migration for " + vm + errorDetails); } } catch (final OperationTimedoutException e) { logger.error("Error while checking the vm {} is on host {}", vm, destHost, e); @@ -5003,8 +5020,9 @@ private void orchestrateMigrateForScale(final String vmUuid, final long srcHostI } catch (final AgentUnavailableException e) { logger.error("Unable to cleanup source host [{}] due to [{}].", fromHost, e.getMessage(), e); } - cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.AgentReportStopped, true); - throw new CloudRuntimeException("Unable to complete migration for " + vm); + Pair cleanupResult = cleanup(vmGuru, new VirtualMachineProfileImpl(vm), work, Event.AgentReportStopped, true); + String errorDetails = (cleanupResult.second() != null) ? " due to " + cleanupResult.second() : ""; + throw new CloudRuntimeException("Unable to complete migration for " + vm + errorDetails); } } catch (final OperationTimedoutException e) { logger.debug("Error while checking the {} on {}", vm, dstHost, e); @@ -5465,7 +5483,8 @@ private void handlePowerOffReportWithNoPendingJobsOnVM(final VMInstanceVO vm) { if (PowerState.PowerOff.equals(vm.getPowerState())) { final VirtualMachineGuru vmGuru = getVmGuru(vm); final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm); - if (!sendStop(vmGuru, profile, true, true)) { + Pair result = sendStop(vmGuru, profile, true, true); + if (!result.first()) { return; } else { // Release resources on StopCommand success @@ -5938,14 +5957,25 @@ public Outcome removeNicFromVmThroughJobQueue( final VirtualMachine vm, final Nic nic) { Long vmId = vm.getId(); String commandName = VmWorkRemoveNicFromVm.class.getName(); - Pair pendingWorkJob = retrievePendingWorkJob(vmId, commandName); - VmWorkJobVO workJob = pendingWorkJob.first(); + // The nic uuid must be part of the pending-job lookup key. Without it, a concurrent request + // to remove a different nic from the same vm matches this still-pending job and joins it + // instead of submitting its own, so only one nic is removed while both callers wait on the + // single job and both receive its success. Mirrors the symmetric addVmToNetworkThroughJobQueue. + final List pendingWorkJobs = _workJobDao.listPendingWorkJobs( + VirtualMachine.Type.Instance, vmId, commandName, nic.getUuid()); - if (workJob == null) { + VmWorkJobVO workJob; + if (pendingWorkJobs != null && pendingWorkJobs.size() > 0) { + if (pendingWorkJobs.size() > 1) { + throw new CloudRuntimeException(String.format("The number of jobs to remove nic %s from vm %s are %d", nic.getUuid(), vm.getInstanceName(), pendingWorkJobs.size())); + } + workJob = pendingWorkJobs.get(0); + } else { Pair newVmWorkJobAndInfo = createWorkJobAndWorkInfo(commandName, vmId); workJob = newVmWorkJobAndInfo.first(); + workJob.setSecondaryObjectIdentifier(nic.getUuid()); VmWorkRemoveNicFromVm workInfo = new VmWorkRemoveNicFromVm(newVmWorkJobAndInfo.second(), nic.getId()); setCmdInfoAndSubmitAsyncJob(workJob, workInfo, vmId); diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java index 55079173dc98..20b9bf125a4d 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/NetworkOrchestrator.java @@ -1767,7 +1767,7 @@ protected boolean reprogramNetworkRules(final long networkId, final Account call final List firewallEgressRulesToApply = _firewallDao.listByNetworkPurposeTrafficType(networkId, Purpose.Firewall, FirewallRule.TrafficType.Egress); final NetworkOfferingVO offering = _networkOfferingDao.findById(network.getNetworkOfferingId()); final DataCenter zone = _dcDao.findById(network.getDataCenterId()); - if (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall) && _networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall) + if (_networkModel.areServicesSupportedInNetwork(network.getId(), Service.Firewall) && (network.getGuestType() == Network.GuestType.Isolated || network.getGuestType() == Network.GuestType.Shared && zone.getNetworkType() == NetworkType.Advanced)) { // add default egress rule to accept the traffic _firewallMgr.applyDefaultEgressFirewallRule(network.getId(), offering.isEgressDefaultPolicy(), true); @@ -2302,6 +2302,11 @@ public void prepareNicForMigration(final VirtualMachineProfile vm, final DeployD } } + private boolean isReservedForExternalProvider(Long vlanId) { + VlanDetailsVO vlanDetail = vlanDetailsDao.findDetail(vlanId, ApiConstants.NETRIS_DETAIL_KEY); + return vlanDetail != null && BooleanUtils.toBoolean(vlanDetail.getValue()); + } + /* Prepare All Nics for migration including the nics dynamically created and not stored in DB This is a temporary workaround work KVM migration @@ -2350,6 +2355,10 @@ public void prepareAllNicsForMigration(final VirtualMachineProfile vm, final Dep final List publicIps = _ipAddressDao.listByAssociatedNetwork(guestNetworkId, null); for (final IPAddressVO userIp : publicIps) { final PublicIp publicIp = PublicIp.createFromAddrAndVlan(userIp, _vlanDao.findById(userIp.getVlanId())); + if (isReservedForExternalProvider(publicIp.getVlanId())) { + logger.debug("Skipping public IP {} for migration NIC profile: managed by external network provider", publicIp.getAddress()); + continue; + } final URI broadcastUri = BroadcastDomainType.Vlan.toUri(publicIp.getVlanTag()); final long ntwkId = publicIp.getNetworkId(); final Nic nic = _nicDao.findByNetworkIdInstanceIdAndBroadcastUri(ntwkId, vm.getId(), @@ -3175,7 +3184,7 @@ private boolean hasGuestBypassVlanOverlapCheck(final boolean bypassVlanOverlapCh /** * Checks for L2 network offering services. Only 2 cases allowed: * - No services - * - User Data service only, provided by ConfigDrive + * - UserData service only, provided by ConfigDrive * * @param ntwkOff network offering */ diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java index 933accbda524..da827b17ff34 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestrator.java @@ -2328,7 +2328,8 @@ public void updateVolumeDiskChain(long volumeId, String path, String chainInfo, @Override public DiskProfile importVolume(Type type, String name, DiskOffering offering, Long sizeInBytes, Long minIops, Long maxIops, Long zoneId, HypervisorType hypervisorType, VirtualMachine vm, VirtualMachineTemplate template, Account owner, - Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo) { + Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, + ImageFormat format) { if (sizeInBytes == null) { sizeInBytes = offering.getDiskSize(); } @@ -2367,7 +2368,9 @@ public DiskProfile importVolume(Type type, String name, DiskOffering offering, L vol.setDisplayVolume(userVm.isDisplayVm()); } - vol.setFormat(getSupportedImageFormatForCluster(hypervisorType)); + // The format the hypervisor actually reported for the existing image wins; pools such as RBD + // hold raw images even though QCOW2 is the cluster default for KVM. + vol.setFormat(format != null ? format : getSupportedImageFormatForCluster(hypervisorType)); vol.setPoolId(poolId); vol.setPoolType(poolType); vol.setPath(path); @@ -2379,7 +2382,8 @@ public DiskProfile importVolume(Type type, String name, DiskOffering offering, L @Override public DiskProfile updateImportedVolume(Type type, DiskOffering offering, VirtualMachine vm, VirtualMachineTemplate template, - Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, DiskProfile diskProfile) { + Long deviceId, Long poolId, Storage.StoragePoolType poolType, String path, String chainInfo, DiskProfile diskProfile, + ImageFormat format) { VolumeVO vol = _volsDao.findById(diskProfile.getVolumeId()); if (vm != null) { @@ -2411,7 +2415,9 @@ public DiskProfile updateImportedVolume(Type type, DiskOffering offering, Virtua vol.setDisplayVolume(userVm.isDisplayVm()); } - vol.setFormat(getSupportedImageFormatForCluster(vm.getHypervisorType())); + // The format the hypervisor actually reported for the existing image wins; pools such as RBD + // hold raw images even though QCOW2 is the cluster default for KVM. + vol.setFormat(format != null ? format : getSupportedImageFormatForCluster(vm.getHypervisorType())); vol.setPoolId(poolId); vol.setPoolType(poolType); vol.setPath(path); diff --git a/engine/orchestration/src/test/java/com/cloud/agent/manager/AgentAttacheSendNextTest.java b/engine/orchestration/src/test/java/com/cloud/agent/manager/AgentAttacheSendNextTest.java new file mode 100644 index 000000000000..5b504b57133c --- /dev/null +++ b/engine/orchestration/src/test/java/com/cloud/agent/manager/AgentAttacheSendNextTest.java @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.agent.manager; + +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import com.cloud.agent.transport.Request; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.host.Status; +import com.cloud.hypervisor.Hypervisor; + +public class AgentAttacheSendNextTest { + + /** + * Minimal concrete AgentAttache: send() fails for one designated sequence and succeeds otherwise, + * recording the sequence that was actually dispatched. + */ + static class TestAgentAttache extends AgentAttache { + Long sentSeq; + final long failSeq; + + TestAgentAttache(long failSeq) { + super(null, 1L, "uuid-1", "host-1", Hypervisor.HypervisorType.KVM, false); + this.failSeq = failSeq; + } + + @Override + public void send(Request req) throws AgentUnavailableException { + if (req.getSequence() == failSeq) { + throw new AgentUnavailableException("simulated transient link failure", _id); + } + sentSeq = req.getSequence(); + } + + @Override + public void disconnect(Status state) { + } + + @Override + protected boolean isClosed() { + return false; + } + } + + @Test + public void sendNextAdvancesPastAFailedCommandToTheNextQueued() { + long failSeq = 100L; + long goodSeq = 200L; + + Request failing = Mockito.mock(Request.class); + Mockito.when(failing.getSequence()).thenReturn(failSeq); + Request good = Mockito.mock(Request.class); + Mockito.when(good.getSequence()).thenReturn(goodSeq); + + TestAgentAttache attache = new TestAgentAttache(failSeq); + attache._requests.add(failing); + attache._requests.add(good); + + attache.sendNext(1L); + + // A command whose send() failed (and was cancelled) must NOT become _currentSequence: no answer + // will ever arrive for it, so every later in-sequence command to this host would queue behind it + // and time out. sendNext must move on and dispatch the next queued command instead. + Assert.assertEquals("the next queued command should have been dispatched", Long.valueOf(goodSeq), attache.sentSeq); + Assert.assertEquals("current sequence must be the successfully sent command, not the failed one", + Long.valueOf(goodSeq), attache._currentSequence); + Assert.assertTrue("the request queue should be drained", attache._requests.isEmpty()); + } +} diff --git a/engine/orchestration/src/test/java/com/cloud/agent/manager/AgentManagerImplTest.java b/engine/orchestration/src/test/java/com/cloud/agent/manager/AgentManagerImplTest.java index 43d83a672c0f..2377bbaefbc8 100644 --- a/engine/orchestration/src/test/java/com/cloud/agent/manager/AgentManagerImplTest.java +++ b/engine/orchestration/src/test/java/com/cloud/agent/manager/AgentManagerImplTest.java @@ -18,14 +18,17 @@ import com.cloud.agent.Listener; import com.cloud.agent.api.Answer; +import com.cloud.agent.api.ReadyAnswer; import com.cloud.agent.api.ReadyCommand; import com.cloud.agent.api.StartupCommand; import com.cloud.agent.api.StartupRoutingCommand; import com.cloud.exception.ConnectionException; +import com.cloud.host.DetailVO; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostDetailsDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.utils.Pair; import org.junit.Assert; @@ -34,10 +37,13 @@ import org.mockito.Mockito; import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; public class AgentManagerImplTest { private HostDao hostDao; + private HostDetailsDao hostDetailsDao; private Listener storagePoolMonitor; private AgentAttache attache; private AgentManagerImpl mgr = Mockito.spy(new AgentManagerImpl()); @@ -46,15 +52,18 @@ public class AgentManagerImplTest { @Before public void setUp() throws Exception { - host = new HostVO("some-Uuid"); + host = Mockito.spy(new HostVO("some-Uuid")); + Mockito.when(host.getId()).thenReturn(1L); host.setDataCenterId(1L); cmds = new StartupCommand[]{new StartupRoutingCommand()}; attache = new ConnectedAgentAttache(null, 1L, "uuid", "kvm-attache", Hypervisor.HypervisorType.KVM, null, false); hostDao = Mockito.mock(HostDao.class); + hostDetailsDao = Mockito.mock(HostDetailsDao.class); storagePoolMonitor = Mockito.mock(Listener.class); mgr._hostDao = hostDao; + mgr._hostDetailsDao = hostDetailsDao; mgr._hostMonitors = new ArrayList<>(); mgr._hostMonitors.add(new Pair<>(0, storagePoolMonitor)); } @@ -86,6 +95,32 @@ public void testNotifyMonitorsOfConnectionWhenStoragePoolConnectionHostFailure() Mockito.verify(mgr, Mockito.times(1)).handleDisconnectWithoutInvestigation(Mockito.any(attache.getClass()), Mockito.eq(Status.Event.AgentDisconnected), Mockito.eq(true), Mockito.eq(true)); } + @Test + public void testNotifyMonitorsOfConnectionClearsStaleNvramCapabilityOnReconnect() throws ConnectionException { + DetailVO staleNvramCapability = Mockito.mock(DetailVO.class); + ReadyAnswer readyAnswer = Mockito.mock(ReadyAnswer.class); + host.setDetails(new HashMap<>(Map.of(Host.HOST_UEFI_ENABLE, Boolean.TRUE.toString(), + Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM, Boolean.TRUE.toString()))); + + Mockito.when(staleNvramCapability.getId()).thenReturn(11L); + Mockito.when(hostDao.findById(Mockito.anyLong())).thenReturn(host); + Mockito.doNothing().when(hostDao).loadDetails(host); + Mockito.when(hostDetailsDao.findDetail(host.getId(), Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM)).thenReturn(staleNvramCapability); + Mockito.doNothing().when(storagePoolMonitor).processConnect(Mockito.eq(host), Mockito.eq(cmds[0]), Mockito.eq(false)); + Mockito.doReturn(true).when(mgr).handleDisconnectWithoutInvestigation(Mockito.any(attache.getClass()), Mockito.any(Status.Event.class), Mockito.anyBoolean(), Mockito.anyBoolean()); + Mockito.when(readyAnswer.getResult()).thenReturn(true); + Mockito.when(readyAnswer.getDetailsMap()).thenReturn(Map.of(Host.HOST_UEFI_ENABLE, Boolean.TRUE.toString())); + Mockito.doReturn(readyAnswer).when(mgr).easySend(Mockito.anyLong(), Mockito.any(ReadyCommand.class)); + Mockito.doReturn(true).when(mgr).agentStatusTransitTo(Mockito.eq(host), Mockito.eq(Status.Event.Ready), Mockito.anyLong()); + + final AgentAttache agentAttache = mgr.notifyMonitorsOfConnection(attache, cmds, false); + + Assert.assertTrue(agentAttache.isReady()); + Assert.assertFalse(host.getDetails().containsKey(Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM)); + Mockito.verify(hostDetailsDao).remove(11L); + Mockito.verify(hostDao).saveDetails(host); + } + @Test public void testGetTimeoutWithPositiveTimeout() { Commands commands = Mockito.mock(Commands.class); diff --git a/engine/orchestration/src/test/java/com/cloud/agent/manager/ConnectedAgentAttacheTest.java b/engine/orchestration/src/test/java/com/cloud/agent/manager/ConnectedAgentAttacheTest.java index 66e6bbae5e2c..72b217d0fdd9 100644 --- a/engine/orchestration/src/test/java/com/cloud/agent/manager/ConnectedAgentAttacheTest.java +++ b/engine/orchestration/src/test/java/com/cloud/agent/manager/ConnectedAgentAttacheTest.java @@ -61,12 +61,12 @@ public void testEqualsFalseDiffLink() throws Exception { } @Test - public void testEqualsFalseDiffId() throws Exception { + public void testEqualsFalseDiffUuid() throws Exception { Link link1 = mock(Link.class); - ConnectedAgentAttache agentAttache1 = new ConnectedAgentAttache(null, 1, "uuid", null, Hypervisor.HypervisorType.KVM, link1, false); - ConnectedAgentAttache agentAttache2 = new ConnectedAgentAttache(null, 2, "uuid", null, Hypervisor.HypervisorType.KVM, link1, false); + ConnectedAgentAttache agentAttache1 = new ConnectedAgentAttache(null, 1, "uuid1", null, Hypervisor.HypervisorType.KVM, link1, false); + ConnectedAgentAttache agentAttache2 = new ConnectedAgentAttache(null, 2, "uuid2", null, Hypervisor.HypervisorType.KVM, link1, false); assertFalse(agentAttache1.equals(agentAttache2)); } diff --git a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java index a07870d09af2..c9a404f9c89c 100644 --- a/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java +++ b/engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java @@ -70,6 +70,7 @@ import org.apache.cloudstack.framework.extensions.dao.ExtensionDetailsDao; import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; import org.apache.cloudstack.framework.extensions.vo.ExtensionDetailsVO; +import org.apache.cloudstack.framework.jobs.AsyncJobExecutionContext; import org.apache.cloudstack.framework.jobs.dao.VmWorkJobDao; import org.apache.cloudstack.framework.jobs.impl.VmWorkJobVO; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; @@ -394,9 +395,9 @@ public void testSendStopWithOkAnswer() throws Exception { when(vm.getHostId()).thenReturn(1L); when(agentManagerMock.send(anyLong(), (Command)any())).thenReturn(answer); - boolean actual = virtualMachineManagerImpl.sendStop(guru, profile, false, false); + Pair actual = virtualMachineManagerImpl.sendStop(guru, profile, false, false); - Assert.assertTrue(actual); + Assert.assertTrue(actual.first()); } @Test @@ -409,9 +410,10 @@ public void testSendStopWithFailAnswer() throws Exception { when(vm.getHostId()).thenReturn(1L); when(agentManagerMock.send(anyLong(), (Command)any())).thenReturn(answer); - boolean actual = virtualMachineManagerImpl.sendStop(guru, profile, false, false); + Pair actual = virtualMachineManagerImpl.sendStop(guru, profile, false, false); - assertFalse(actual); + assertFalse(actual.first()); + Assert.assertEquals("fail", actual.second()); } @Test @@ -421,11 +423,13 @@ public void testSendStopWithNullAnswer() throws Exception { VirtualMachineProfile profile = mock(VirtualMachineProfile.class); when(profile.getVirtualMachine()).thenReturn(vm); when(vm.getHostId()).thenReturn(1L); + when(vm.getInstanceName()).thenReturn("test-vm"); when(agentManagerMock.send(anyLong(), (Command)any())).thenReturn(null); - boolean actual = virtualMachineManagerImpl.sendStop(guru, profile, false, false); + Pair actual = virtualMachineManagerImpl.sendStop(guru, profile, false, false); - assertFalse(actual); + assertFalse(actual.first()); + Assert.assertNotNull(actual.second()); } @Test @@ -1865,6 +1869,97 @@ public void testUnmanagePendingWorkJobs() { virtualMachineManagerImpl.unmanage(vmMockUuid, null); } + /** + * A pending remove-nic job for a different nic on the same vm must not swallow the removal of + * this nic: the pending-job lookup has to be keyed on the nic uuid so that a new job is + * submitted for this nic instead of joining the other nic's job. Regression test for the + * concurrent-removeNic collapse. + */ + @Test + public void removeNicFromVmThroughJobQueueDoesNotJoinAnotherNicsPendingJob() { + String commandName = VmWorkRemoveNicFromVm.class.getName(); + String nicUuid = UUID.randomUUID().toString(); + + Nic nic = mock(Nic.class); + when(nic.getId()).thenReturn(42L); + when(nic.getUuid()).thenReturn(nicUuid); + + // A pending remove-nic job exists for the vm (e.g. for another nic); the nic-agnostic + // lookup would return it, but the nic-keyed lookup finds nothing for this nic. This stub is + // lenient because the fixed code never consults the nic-agnostic 3-arg lookup - only the + // buggy code does, where this job is what it wrongly joins. + Mockito.lenient().when(_workJobDao.listPendingWorkJobs(VirtualMachine.Type.Instance, vmInstanceVoMockId, commandName)) + .thenReturn(Collections.singletonList(mock(VmWorkJobVO.class))); + when(_workJobDao.listPendingWorkJobs(VirtualMachine.Type.Instance, vmInstanceVoMockId, commandName, nicUuid)) + .thenReturn(Collections.emptyList()); + + VmWorkJobVO newJob = mock(VmWorkJobVO.class); + when(newJob.getId()).thenReturn(100L); + doReturn(new Pair(newJob, mock(VmWork.class))) + .when(virtualMachineManagerImpl).createWorkJobAndWorkInfo(commandName, vmInstanceVoMockId); + doNothing().when(virtualMachineManagerImpl).setCmdInfoAndSubmitAsyncJob(any(), any(), anyLong()); + + AsyncJobExecutionContext execContext = mock(AsyncJobExecutionContext.class); + try (MockedStatic ignored = Mockito.mockStatic(AsyncJobExecutionContext.class)) { + when(AsyncJobExecutionContext.getCurrentExecutionContext()).thenReturn(execContext); + + virtualMachineManagerImpl.removeNicFromVmThroughJobQueue(vmInstanceMock, nic); + + // A new job must be created for this nic, stamped with the nic uuid, submitted and joined. + verify(virtualMachineManagerImpl, times(1)).createWorkJobAndWorkInfo(commandName, vmInstanceVoMockId); + verify(newJob, times(1)).setSecondaryObjectIdentifier(nicUuid); + verify(virtualMachineManagerImpl, times(1)).setCmdInfoAndSubmitAsyncJob(eq(newJob), any(), eq(vmInstanceVoMockId)); + verify(execContext, times(1)).joinJob(100L); + } + } + + /** + * When a pending remove-nic job already exists for this same nic, the request must join it + * rather than submit a duplicate: per-nic deduplication still holds. + */ + @Test + public void removeNicFromVmThroughJobQueueJoinsExistingJobForSameNic() { + String commandName = VmWorkRemoveNicFromVm.class.getName(); + String nicUuid = UUID.randomUUID().toString(); + + Nic nic = mock(Nic.class); + when(nic.getUuid()).thenReturn(nicUuid); + + VmWorkJobVO existingJob = mock(VmWorkJobVO.class); + when(existingJob.getId()).thenReturn(77L); + when(_workJobDao.listPendingWorkJobs(VirtualMachine.Type.Instance, vmInstanceVoMockId, commandName, nicUuid)) + .thenReturn(Collections.singletonList(existingJob)); + + AsyncJobExecutionContext execContext = mock(AsyncJobExecutionContext.class); + try (MockedStatic ignored = Mockito.mockStatic(AsyncJobExecutionContext.class)) { + when(AsyncJobExecutionContext.getCurrentExecutionContext()).thenReturn(execContext); + + virtualMachineManagerImpl.removeNicFromVmThroughJobQueue(vmInstanceMock, nic); + + verify(virtualMachineManagerImpl, never()).createWorkJobAndWorkInfo(anyString(), anyLong()); + verify(virtualMachineManagerImpl, never()).setCmdInfoAndSubmitAsyncJob(any(), any(), anyLong()); + verify(execContext, times(1)).joinJob(77L); + } + } + + /** + * More than one pending remove-nic job for the same nic is an inconsistent state and must fail + * fast rather than pick one arbitrarily. Mirrors the guard in addVmToNetworkThroughJobQueue. + */ + @Test(expected = CloudRuntimeException.class) + public void removeNicFromVmThroughJobQueueThrowsWhenMultiplePendingJobsForSameNic() { + String commandName = VmWorkRemoveNicFromVm.class.getName(); + String nicUuid = UUID.randomUUID().toString(); + + Nic nic = mock(Nic.class); + when(nic.getUuid()).thenReturn(nicUuid); + + when(_workJobDao.listPendingWorkJobs(VirtualMachine.Type.Instance, vmInstanceVoMockId, commandName, nicUuid)) + .thenReturn(Arrays.asList(mock(VmWorkJobVO.class), mock(VmWorkJobVO.class))); + + virtualMachineManagerImpl.removeNicFromVmThroughJobQueue(vmInstanceMock, nic); + } + @Test public void testUnmanageHostNotFoundAfterTransaction() { when(vmInstanceMock.getHostId()).thenReturn(hostMockId); diff --git a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java index b4a26c17e2e5..259dfaa6b1fa 100644 --- a/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java +++ b/engine/orchestration/src/test/java/org/apache/cloudstack/engine/orchestration/VolumeOrchestratorTest.java @@ -241,7 +241,7 @@ public void testImportVolume() { volumeOrchestrator.importVolume(volumeType, name, diskOffering, sizeInBytes, null, null, zoneId, hypervisorType, null, null, owner, - deviceId, poolId, Storage.StoragePoolType.NetworkFilesystem, path, chainInfo); + deviceId, poolId, Storage.StoragePoolType.NetworkFilesystem, path, chainInfo, null); VolumeVO volume = volumeVOMockedConstructionConstruction.constructed().get(0); Mockito.verify(volume, Mockito.never()).setInstanceId(Mockito.anyLong()); diff --git a/engine/schema/src/main/java/com/cloud/event/dao/EventDao.java b/engine/schema/src/main/java/com/cloud/event/dao/EventDao.java index c50451b03e4a..03716237cdb0 100644 --- a/engine/schema/src/main/java/com/cloud/event/dao/EventDao.java +++ b/engine/schema/src/main/java/com/cloud/event/dao/EventDao.java @@ -19,6 +19,7 @@ import java.util.Date; import java.util.List; +import com.cloud.event.Event; import com.cloud.event.EventVO; import com.cloud.utils.db.Filter; import com.cloud.utils.db.GenericDao; @@ -31,6 +32,18 @@ public interface EventDao extends GenericDao { EventVO findCompletedEvent(long startId); + /** + * Finds the last non-archived start event matching the specified criteria. + * Events are ordered by ID in descending order, returning the most recent one. + * + * @param type the event type to search for + * @param state the event state to search for (e.g., {@link Event.State#Scheduled}) + * @param resourceId the resource ID associated with the event + * @param resourceType the resource type associated with the event + * @return the most recent EventVO matching the criteria, or null if not found + */ + EventVO findLastEvent(String type, Event.State state, Long resourceId, String resourceType); + public List listToArchiveOrDeleteEvents(List ids, String type, Date startDate, Date endDate, List accountIds); public void archiveEvents(List events); diff --git a/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java b/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java index e748e98900eb..b66da14292ef 100644 --- a/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java @@ -19,7 +19,6 @@ import java.util.Date; import java.util.List; - import org.springframework.stereotype.Component; import com.cloud.event.Event.State; @@ -35,6 +34,7 @@ public class EventDaoImpl extends GenericDaoBase implements EventDao { protected final SearchBuilder CompletedEventSearch; protected final SearchBuilder ToArchiveOrDeleteEventSearch; + protected final SearchBuilder LastStartEventSearch; public EventDaoImpl() { CompletedEventSearch = createSearchBuilder(); @@ -51,6 +51,14 @@ public EventDaoImpl() { ToArchiveOrDeleteEventSearch.and("createdDateL", ToArchiveOrDeleteEventSearch.entity().getCreateDate(), Op.LTEQ); ToArchiveOrDeleteEventSearch.and("archived", ToArchiveOrDeleteEventSearch.entity().getArchived(), Op.EQ); ToArchiveOrDeleteEventSearch.done(); + + LastStartEventSearch = createSearchBuilder(); + LastStartEventSearch.and("type", LastStartEventSearch.entity().getType(), Op.EQ); + LastStartEventSearch.and("state", LastStartEventSearch.entity().getState(), Op.EQ); + LastStartEventSearch.and("resourceId", LastStartEventSearch.entity().getResourceId(), Op.EQ); + LastStartEventSearch.and("resourceType", LastStartEventSearch.entity().getResourceType(), Op.EQ); + LastStartEventSearch.and("archived", LastStartEventSearch.entity().getArchived(), Op.EQ); + LastStartEventSearch.done(); } @Override @@ -77,6 +85,17 @@ public EventVO findCompletedEvent(long startId) { return findOneIncludingRemovedBy(sc); } + @Override + public EventVO findLastEvent(String type, State state, Long resourceId, String resourceType) { + SearchCriteria sc = LastStartEventSearch.create(); + sc.setParameters("type", type); + sc.setParameters("state", state); + sc.setParameters("resourceId", resourceId); + sc.setParameters("resourceType", resourceType); + sc.setParameters("archived", false); + return findLastOneBy(sc); + } + @Override public List listToArchiveOrDeleteEvents(List ids, String type, Date startDate, Date endDate, List accountIds) { SearchCriteria sc = ToArchiveOrDeleteEventSearch.create(); diff --git a/engine/schema/src/main/java/com/cloud/offerings/NetworkOfferingVO.java b/engine/schema/src/main/java/com/cloud/offerings/NetworkOfferingVO.java index 904c8e646eb5..fc22e68985d2 100644 --- a/engine/schema/src/main/java/com/cloud/offerings/NetworkOfferingVO.java +++ b/engine/schema/src/main/java/com/cloud/offerings/NetworkOfferingVO.java @@ -436,7 +436,7 @@ public NetworkOfferingVO(String name, Network.GuestType guestType, boolean speci true, Availability.Optional, null, - Network.GuestType.Isolated, + guestType, true, false, false, diff --git a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java index 3868ca960e06..92de79b83ece 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/DatabaseUpgradeChecker.java @@ -94,6 +94,7 @@ import com.cloud.upgrade.dao.Upgrade42040to42100; import com.cloud.upgrade.dao.Upgrade42100to42200; import com.cloud.upgrade.dao.Upgrade42200to42210; +import com.cloud.upgrade.dao.Upgrade42210to42220; import com.cloud.upgrade.dao.Upgrade420to421; import com.cloud.upgrade.dao.Upgrade421to430; import com.cloud.upgrade.dao.Upgrade430to440; @@ -246,6 +247,7 @@ public DatabaseUpgradeChecker() { .next("4.20.4.0", new Upgrade42040to42100()) .next("4.21.0.0", new Upgrade42100to42200()) .next("4.22.0.0", new Upgrade42200to42210()) + .next("4.22.1.0", new Upgrade42210to42220()) .build(); } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java index 89b71f03289b..3e9355c76036 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/SystemVmTemplateRegistration.java @@ -324,38 +324,33 @@ public void setUpdated(Date updated) { public static final Map NewTemplateMap = new HashMap<>(); - public static final Map RouterTemplateConfigurationNames = new HashMap<>() { - { - put(Hypervisor.HypervisorType.KVM, "router.template.kvm"); - put(Hypervisor.HypervisorType.VMware, "router.template.vmware"); - put(Hypervisor.HypervisorType.XenServer, "router.template.xenserver"); - put(Hypervisor.HypervisorType.Hyperv, "router.template.hyperv"); - put(Hypervisor.HypervisorType.LXC, "router.template.lxc"); - put(Hypervisor.HypervisorType.Ovm3, "router.template.ovm3"); - } - }; - - public static Map hypervisorGuestOsMap = new HashMap<>() { - { - put(Hypervisor.HypervisorType.KVM, LINUX_12_ID); - put(Hypervisor.HypervisorType.XenServer, OTHER_LINUX_ID); - put(Hypervisor.HypervisorType.VMware, OTHER_LINUX_ID); - put(Hypervisor.HypervisorType.Hyperv, LINUX_12_ID); - put(Hypervisor.HypervisorType.LXC, LINUX_12_ID); - put(Hypervisor.HypervisorType.Ovm3, LINUX_12_ID); - } - }; - - public static final Map hypervisorImageFormat = new HashMap() { - { - put(Hypervisor.HypervisorType.KVM, ImageFormat.QCOW2); - put(Hypervisor.HypervisorType.XenServer, ImageFormat.VHD); - put(Hypervisor.HypervisorType.VMware, ImageFormat.OVA); - put(Hypervisor.HypervisorType.Hyperv, ImageFormat.VHD); - put(Hypervisor.HypervisorType.LXC, ImageFormat.QCOW2); - put(Hypervisor.HypervisorType.Ovm3, ImageFormat.RAW); - } - }; + public static final Map RouterTemplateConfigurationNames = Map.of( + Hypervisor.HypervisorType.KVM, "router.template.kvm", + Hypervisor.HypervisorType.VMware, "router.template.vmware", + Hypervisor.HypervisorType.XenServer, "router.template.xenserver", + Hypervisor.HypervisorType.Hyperv, "router.template.hyperv", + Hypervisor.HypervisorType.LXC, "router.template.lxc", + Hypervisor.HypervisorType.Ovm3, "router.template.ovm3" + ); + + public static Map hypervisorGuestOsMap = new HashMap<>(); + static { + hypervisorGuestOsMap.put(Hypervisor.HypervisorType.KVM, LINUX_12_ID); + hypervisorGuestOsMap.put(Hypervisor.HypervisorType.XenServer, OTHER_LINUX_ID); + hypervisorGuestOsMap.put(Hypervisor.HypervisorType.VMware, OTHER_LINUX_ID); + hypervisorGuestOsMap.put(Hypervisor.HypervisorType.Hyperv, LINUX_12_ID); + hypervisorGuestOsMap.put(Hypervisor.HypervisorType.LXC, LINUX_12_ID); + hypervisorGuestOsMap.put(Hypervisor.HypervisorType.Ovm3, LINUX_12_ID); + } + + public static final Map hypervisorImageFormat = Map.of( + Hypervisor.HypervisorType.KVM, ImageFormat.QCOW2, + Hypervisor.HypervisorType.XenServer, ImageFormat.VHD, + Hypervisor.HypervisorType.VMware, ImageFormat.OVA, + Hypervisor.HypervisorType.Hyperv, ImageFormat.VHD, + Hypervisor.HypervisorType.LXC, ImageFormat.QCOW2, + Hypervisor.HypervisorType.Ovm3, ImageFormat.RAW + ); public boolean validateIfSeeded(TemplateDataStoreVO templDataStoreVO, String url, String path, String nfsVersion) { String filePath = null; diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java index 223d7a466376..f5775a56097e 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java @@ -128,7 +128,7 @@ public boolean indexExists(Connection conn, String tableName, String indexName) return true; } } catch (SQLException e) { - logger.debug(String.format("Index %s doesn't exist, ignoring exception:", indexName, e.getMessage())); + logger.debug("Index {} doesn't exist, ignoring exception: {}", indexName, e.getMessage()); } return false; } diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41500to41510.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41500to41510.java index c7295414326d..18d419d29f98 100644 --- a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41500to41510.java +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41500to41510.java @@ -22,7 +22,6 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; -import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -98,49 +97,41 @@ public void updateSystemVmTemplates(final Connection conn) { throw new CloudRuntimeException("updateSystemVmTemplates:Exception while getting hypervisor types from clusters", e); } - final Map NewTemplateNameList = new HashMap() { - { - put(KVM, "systemvm-kvm-4.15.1"); - put(VMware, "systemvm-vmware-4.15.1"); - put(XenServer, "systemvm-xenserver-4.15.1"); - put(Hyperv, "systemvm-hyperv-4.15.1"); - put(LXC, "systemvm-lxc-4.15.1"); - put(Ovm3, "systemvm-ovm3-4.15.1"); - } - }; - - final Map routerTemplateConfigurationNames = new HashMap() { - { - put(KVM, "router.template.kvm"); - put(VMware, "router.template.vmware"); - put(XenServer, "router.template.xenserver"); - put(Hyperv, "router.template.hyperv"); - put(LXC, "router.template.lxc"); - put(Ovm3, "router.template.ovm3"); - } - }; - - final Map newTemplateUrl = new HashMap() { - { - put(KVM, "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.1-kvm.qcow2.bz2"); - put(VMware, "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.1-vmware.ova"); - put(XenServer, "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.1-xen.vhd.bz2"); - put(Hyperv, "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.1-hyperv.vhd.zip"); - put(LXC, "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.1-kvm.qcow2.bz2"); - put(Ovm3, "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.1-ovm.raw.bz2"); - } - }; - - final Map newTemplateChecksum = new HashMap() { - { - put(KVM, "0e9f9a7d0957c3e0a2088e41b2da2cec"); - put(XenServer, "86373992740b1eca8aff8b08ebf3aea5"); - put(VMware, "4006982765846d373eb3719b2fe4d720"); - put(Hyperv, "0b9514e4b6cba1f636fea2125f0f7a5f"); - put(LXC, "0e9f9a7d0957c3e0a2088e41b2da2cec"); - put(Ovm3, "ae3977e696b3e6c81bdcbb792d514d29"); - } - }; + final Map NewTemplateNameList = Map.of( + KVM, "systemvm-kvm-4.15.1", + VMware, "systemvm-vmware-4.15.1", + XenServer, "systemvm-xenserver-4.15.1", + Hyperv, "systemvm-hyperv-4.15.1", + LXC, "systemvm-lxc-4.15.1", + Ovm3, "systemvm-ovm3-4.15.1" + ); + + final Map routerTemplateConfigurationNames = Map.of( + KVM, "router.template.kvm", + VMware, "router.template.vmware", + XenServer, "router.template.xenserver", + Hyperv, "router.template.hyperv", + LXC, "router.template.lxc", + Ovm3, "router.template.ovm3" + ); + + final Map newTemplateUrl = Map.of( + KVM, "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.1-kvm.qcow2.bz2", + VMware, "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.1-vmware.ova", + XenServer, "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.1-xen.vhd.bz2", + Hyperv, "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.1-hyperv.vhd.zip", + LXC, "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.1-kvm.qcow2.bz2", + Ovm3, "https://download.cloudstack.org/systemvm/4.15/systemvmtemplate-4.15.1-ovm.raw.bz2" + ); + + final Map newTemplateChecksum = Map.of( + KVM, "0e9f9a7d0957c3e0a2088e41b2da2cec", + XenServer, "86373992740b1eca8aff8b08ebf3aea5", + VMware, "4006982765846d373eb3719b2fe4d720", + Hyperv, "0b9514e4b6cba1f636fea2125f0f7a5f", + LXC, "0e9f9a7d0957c3e0a2088e41b2da2cec", + Ovm3, "ae3977e696b3e6c81bdcbb792d514d29" + ); for (final Map.Entry hypervisorAndTemplateName : NewTemplateNameList.entrySet()) { logger.debug("Updating " + hypervisorAndTemplateName.getKey() + " System Vms"); diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42210to42220.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42210to42220.java new file mode 100644 index 000000000000..9985705dff34 --- /dev/null +++ b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade42210to42220.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.upgrade.dao; + +import java.sql.Connection; + +public class Upgrade42210to42220 extends DbUpgradeAbstractImpl implements DbUpgrade { + @Override + public String[] getUpgradableVersionRange() { + return new String[] {"4.22.1.0", "4.22.2.0"}; + } + + @Override + public String getUpgradedVersion() { + return "4.22.2.0"; + } + + @Override + public void performDataMigration(Connection conn) { + } +} diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java index 2718127d648a..b140f6b8d02b 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDao.java @@ -105,6 +105,6 @@ public interface NicDao extends GenericDao { NicVO findByIpAddressAndVmType(String ip, VirtualMachine.Type vmType); - List listByNetworkIdAndType(long networkId, VirtualMachine.Type vmType); + List listNonPlaceholderNicsByNetworkIdAndType(long networkId, VirtualMachine.Type vmType); List searchRemovedByVms(List vmIds, Long batchSize); } diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java index f4f3b0d6ad60..84b6ec4f2044 100644 --- a/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java +++ b/engine/schema/src/main/java/com/cloud/vm/dao/NicDaoImpl.java @@ -456,10 +456,11 @@ public NicVO findByIpAddressAndVmType(String ip, VirtualMachine.Type vmType) { } @Override - public List listByNetworkIdAndType(long networkId, VirtualMachine.Type vmType) { + public List listNonPlaceholderNicsByNetworkIdAndType(long networkId, VirtualMachine.Type vmType) { SearchCriteria sc = AllFieldsSearch.create(); sc.setParameters("network", networkId); sc.setParameters("vmType", vmType); + sc.setParameters("strategyNEQ", Nic.ReservationStrategy.PlaceHolder.toString()); return listBy(sc); } diff --git a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java index 0f8a10fb7be6..39a7fd2b2ead 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/backup/BackupVO.java @@ -66,10 +66,10 @@ public class BackupVO implements Backup { private String externalId; @Column(name = "type") - private String backupType; + private String type; @Column(name = "date") - @Temporal(value = TemporalType.DATE) + @Temporal(value = TemporalType.TIMESTAMP) private Date date; @Column(name = GenericDao.REMOVED_COLUMN) @@ -113,7 +113,7 @@ public BackupVO() { @Override public String toString() { return String.format("Backup %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields( - this, "id", "uuid", "vmId", "backupType", "externalId")); + this, "id", "uuid", "vmId", "type", "externalId")); } @Override @@ -145,11 +145,11 @@ public void setExternalId(String externalId) { } public String getType() { - return backupType; + return type; } public void setType(String type) { - this.backupType = type; + this.type = type; } @Override diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java index 57b5f6461c6a..fd85dba1efcc 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDao.java @@ -90,6 +90,22 @@ public interface PrimaryDataStoreDao extends GenericDao { List findDisabledPoolsByScope(long dcId, Long podId, Long clusterId, ScopeType scope); + /** + * Finds disabled storage pools within the specified scope that are associated with the given storage access groups. + * This method is used to locate storage pools with 'Disabled' status that match specific storage access group + * filters. + * + * @param dcId the data center ID. + * @param podId the pod ID. + * @param clusterId the cluster ID. + * @param scope ZONE, CLUSTER OR HOST type scope + * @param storageAccessGroups array of storage access group names to match against. + * Only pools associated with these access groups will be returned. + * If null or empty, returns an empty list. + * @return a list of {@link StoragePoolVO} objects. + */ + List findDisabledPoolsByScopeAndAccessGroups(long dcId, Long podId, Long clusterId, ScopeType scope, String[] storageAccessGroups); + /** * Find pool by UUID. * @@ -167,7 +183,19 @@ Pair, Integer> searchForIdsAndCount(Long storagePoolId, String storag List listByIds(List ids); - List findStoragePoolsByEmptyStorageAccessGroups(Long dcId, Long podId, Long clusterId, ScopeType scope, HypervisorType hypervisorType); + /** + * Finds storage pools that have no storage access groups associated with them within the specified criteria. + * This method identifies storage pools without access group restrictions. + * + * @param dcId the data center ID. Can be null to include all data centers. + * @param podId the pod ID. Can be null to include all pods. + * @param clusterId the cluster ID. Can be null to include all clusters. + * @param scope ZONE, CLUSTER or HOST type scope + * @param hypervisorType the hypervisor type filter. Can be null to include all hypervisor types. + * @param status the storage pool status to filter by. + * @return a list of {@link StoragePoolVO} objects that have no storage access groups associated. + */ + List findStoragePoolsByEmptyStorageAccessGroups(Long dcId, Long podId, Long clusterId, ScopeType scope, HypervisorType hypervisorType, StoragePoolStatus status); List findPoolsByStorageTypeAndZone(Storage.StoragePoolType storageType, Long zoneId); diff --git a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java index b5d6415e3a1a..4d0e581826f4 100644 --- a/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java +++ b/engine/schema/src/main/java/org/apache/cloudstack/storage/datastore/db/PrimaryDataStoreDaoImpl.java @@ -54,6 +54,7 @@ import com.cloud.utils.db.SearchCriteria.Op; import com.cloud.utils.db.TransactionLegacy; import com.cloud.utils.exception.CloudRuntimeException; +import org.apache.commons.lang3.ArrayUtils; @DB() public class PrimaryDataStoreDaoImpl extends GenericDaoBase implements PrimaryDataStoreDao { @@ -86,6 +87,37 @@ public class PrimaryDataStoreDaoImpl extends GenericDaoBase private final String ZoneWideStorageAccessGroupsWithHypervisorTypeSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_and_access_group_map ON storage_pool.id = storage_pool_and_access_group_map.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' and storage_pool.hypervisor = ? and storage_pool.data_center_id = ? and storage_pool.scope = ? and ("; private final String ZoneWideStorageAccessGroupsWithHypervisorTypeSqlSuffix = ") GROUP BY storage_pool_and_access_group_map.pool_id"; + /** + * SQL query prefix to find zone-wide disabled storage pools that are associated with specific storage access groups. + */ + private final String ZoneWideDisabledStorageAccessGroupsSqlPrefix = + "SELECT storage_pool.* FROM storage_pool " + + "LEFT JOIN storage_pool_and_access_group_map ON storage_pool.id = storage_pool_and_access_group_map.pool_id " + + "WHERE storage_pool.removed IS NULL " + + " AND storage_pool.status = 'Disabled' " + + " AND storage_pool.data_center_id = ? " + + " AND storage_pool.scope = ? " + + " AND ("; + + /** + * SQL query prefix to find disabled storage pools with specific storage access groups. The query is used at the HOST scope. + */ + private final String DisabledStorageAccessGroupsForHostConnectionSqlPrefix = + "SELECT storage_pool.* FROM storage_pool " + + "LEFT JOIN storage_pool_and_access_group_map ON storage_pool.id = storage_pool_and_access_group_map.pool_id " + + "WHERE storage_pool.removed IS NULL " + + " AND storage_pool.status = 'Disabled' " + + " AND storage_pool.data_center_id = ? " + + " AND (storage_pool.pod_id = ? OR storage_pool.pod_id IS NULL) " + + " AND storage_pool.scope = ? " + + " AND ("; + + /** + * SQL query suffix for disabled storage access groups queries. This suffix completes the query by grouping + * results by pool_id. + */ + private final String DisabledStorageAccessGroupsSqlSuffix = ") GROUP BY storage_pool_and_access_group_map.pool_id"; + // Storage tags are now separate from storage_pool_details, leaving only details on that table protected final String TagsSqlPrefix = "SELECT storage_pool.* from storage_pool LEFT JOIN storage_pool_tags ON storage_pool.id = storage_pool_tags.pool_id WHERE storage_pool.removed is null and storage_pool.status = 'Up' AND storage_pool_tags.is_tag_a_rule = 0 and storage_pool.data_center_id = ? and (storage_pool.pod_id = ? or storage_pool.pod_id is null) and storage_pool.scope = ? and ("; protected final String TagsSqlSuffix = ") GROUP BY storage_pool_tags.pool_id HAVING COUNT(storage_pool_tags.tag) >= ?"; @@ -568,6 +600,26 @@ public List findDisabledPoolsByScope(long dcId, Long podId, Long return storagePools; } + @Override + public List findDisabledPoolsByScopeAndAccessGroups(long dcId, Long podId, Long clusterId, ScopeType scope, String[] storageAccessGroups) { + if (ArrayUtils.isEmpty(storageAccessGroups)) { + return List.of(); + } + + List storagePools = null; + String sqlValues = getSqlValuesFromStorageAccessGroups(storageAccessGroups); + + if (scope == ScopeType.ZONE) { + String sql = getSqlPreparedStatement(ZoneWideDisabledStorageAccessGroupsSqlPrefix, DisabledStorageAccessGroupsSqlSuffix, sqlValues, null); + storagePools = searchStoragePoolsPreparedStatement(sql, dcId, null, null, scope, null); + } else if ((scope == ScopeType.CLUSTER || scope == ScopeType.HOST) && podId != null && clusterId != null) { + String sql = getSqlPreparedStatement(DisabledStorageAccessGroupsForHostConnectionSqlPrefix, DisabledStorageAccessGroupsSqlSuffix, sqlValues, clusterId); + storagePools = searchStoragePoolsPreparedStatement(sql, dcId, podId, clusterId, scope, null); + } + + return storagePools; + } + @Override public List findLocalStoragePoolsByTags(long dcId, long podId, Long clusterId, String[] tags, boolean validateTagRule) { return findLocalStoragePoolsByTags(dcId, podId, clusterId, tags, validateTagRule, null); @@ -691,7 +743,7 @@ public List findZoneWideStoragePoolsByAccessGroupsAndHypervisorTy } @Override - public List findStoragePoolsByEmptyStorageAccessGroups(Long dcId, Long podId, Long clusterId, ScopeType scope, HypervisorType hypervisorType) { + public List findStoragePoolsByEmptyStorageAccessGroups(Long dcId, Long podId, Long clusterId, ScopeType scope, HypervisorType hypervisorType, StoragePoolStatus status) { SearchBuilder poolSearch = createSearchBuilder(); SearchBuilder storageAccessGroupsPoolSearch = _storagePoolAccessGroupMapDao.createSearchBuilder(); // Set criteria for pools @@ -709,7 +761,6 @@ public List findStoragePoolsByEmptyStorageAccessGroups(Long dcId, SearchCriteria sc = poolSearch.create(); sc.setParameters("scope", scope.toString()); - sc.setParameters("status", Status.Up.toString()); if (dcId != null) { sc.setParameters("datacenterid", dcId); @@ -727,6 +778,10 @@ public List findStoragePoolsByEmptyStorageAccessGroups(Long dcId, sc.setParameters("hypervisortype", hypervisorType); } + if (status != null) { + sc.setParameters("status", status.toString()); + } + return listBy(sc); } diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42220-cleanup.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42220-cleanup.sql new file mode 100644 index 000000000000..85563b6daf03 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42220-cleanup.sql @@ -0,0 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +--; +-- Schema upgrade cleanup from 4.22.1.0 to 4.22.2.0 +--; diff --git a/engine/schema/src/main/resources/META-INF/db/schema-42210to42220.sql b/engine/schema/src/main/resources/META-INF/db/schema-42210to42220.sql new file mode 100644 index 000000000000..7a8ee864eac3 --- /dev/null +++ b/engine/schema/src/main/resources/META-INF/db/schema-42210to42220.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +--; +-- Schema upgrade from 4.22.1.0 to 4.22.2.0 +--; + +-- Widen the unique key on cloud_usage.usage_volume to include vm_id, so the cumulative and +-- per-VM volume usage records introduced in 4.22.1 can coexist. See #13399. +CALL `cloud_usage`.`IDEMPOTENT_DROP_INDEX`('id', 'cloud_usage.usage_volume'); +CALL `cloud_usage`.`IDEMPOTENT_ADD_UNIQUE_INDEX`('cloud_usage.usage_volume', 'id', '(volume_id ASC, created ASC, vm_id ASC)'); + +-- Reclaim VNETs belonging to Netris deleted networks +UPDATE op_dc_vnet_alloc a + JOIN networks n ON n.broadcast_uri = CONCAT('netris://', a.vnet) + SET a.account_id = NULL, + a.reservation_id = NULL, + a.taken = NULL +WHERE n.removed IS NOT NULL; diff --git a/engine/schema/src/test/java/com/cloud/network/as/AutoScaleVmProfileVOTest.java b/engine/schema/src/test/java/com/cloud/network/as/AutoScaleVmProfileVOTest.java index 6813a2091576..4843a97f6d5d 100755 --- a/engine/schema/src/test/java/com/cloud/network/as/AutoScaleVmProfileVOTest.java +++ b/engine/schema/src/test/java/com/cloud/network/as/AutoScaleVmProfileVOTest.java @@ -44,8 +44,15 @@ public void testCounterParamsForUpdate() { AutoScaleVmProfileVO profile = new AutoScaleVmProfileVO(); Map> counterParamList = new LinkedHashMap<>(); - counterParamList.put("0", new LinkedHashMap<>() {{ put("name", "snmpcommunity"); put("value", "public"); }}); - counterParamList.put("1", new LinkedHashMap<>() {{ put("name", "snmpport"); put("value", "161"); }}); + LinkedHashMap param0 = new LinkedHashMap<>(); + param0.put("name", "snmpcommunity"); + param0.put("value", "public"); + counterParamList.put("0", param0); + + LinkedHashMap param1 = new LinkedHashMap<>(); + param1.put("name", "snmpport"); + param1.put("value", "161"); + counterParamList.put("1", param1); profile.setCounterParamsForUpdate(counterParamList); Assert.assertEquals("snmpcommunity=public&snmpport=161", profile.getCounterParamsString()); @@ -63,8 +70,15 @@ public void tstSetOtherDeployParamsForUpdate() { AutoScaleVmProfileVO profile = new AutoScaleVmProfileVO(); Map> otherDeployParamsMap = new HashMap<>(); - otherDeployParamsMap.put("0", new HashMap<>() {{ put("name", "serviceofferingid"); put("value", "a7fb50f6-01d9-11ed-8bc1-77f8f0228926"); }}); - otherDeployParamsMap.put("1", new HashMap<>() {{ put("name", "rootdisksize"); put("value", "10"); }}); + HashMap deployParam0 = new HashMap<>(); + deployParam0.put("name", "serviceofferingid"); + deployParam0.put("value", "a7fb50f6-01d9-11ed-8bc1-77f8f0228926"); + otherDeployParamsMap.put("0", deployParam0); + + HashMap deployParam1 = new HashMap<>(); + deployParam1.put("name", "rootdisksize"); + deployParam1.put("value", "10"); + otherDeployParamsMap.put("1", deployParam1); profile.setOtherDeployParamsForUpdate(otherDeployParamsMap); diff --git a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java index 93be850f558e..7a5e1505c1bb 100644 --- a/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java +++ b/engine/schema/src/test/java/com/cloud/upgrade/SystemVmTemplateRegistrationTest.java @@ -363,6 +363,7 @@ public void testValidateTemplates_fileFailure() { systemVmTemplateRegistration.validateTemplates(list); } + @Test public void testValidateTemplates_downloadableFileNotFound() { CPU.CPUArch arch = SystemVmTemplateRegistration.DOWNLOADABLE_TEMPLATE_ARCH_TYPES.get(0); List> list = new ArrayList<>(); diff --git a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java index 8145158dfa40..c1bcc8ddb9af 100644 --- a/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java +++ b/engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java @@ -472,7 +472,15 @@ private boolean canBypassSecondaryStorage(DataObject srcData, DataObject destDat return true; } + if (Hypervisor.HypervisorType.XenServer.equals(((VolumeInfo) srcData).getHypervisorType())) { + return false; + } + if (destData instanceof VolumeInfo) { + if (Hypervisor.HypervisorType.XenServer.equals(((VolumeInfo) destData).getHypervisorType())) { + return false; + } + Scope srcDataStoreScope = srcData.getDataStore().getScope(); Scope destDataStoreScope = destData.getDataStore().getScope(); logger.info("srcDataStoreScope: {}, srcData pool type: {}; destDataStoreScope: {}, destData pool type: {}", diff --git a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategyTest.java b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategyTest.java index e167cc0a9653..d7ec53702c8c 100755 --- a/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategyTest.java +++ b/engine/storage/datamotion/src/test/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategyTest.java @@ -123,6 +123,7 @@ public void testCanBypassSecondaryStorageForDirectDownload() throws NoSuchMethod @Test public void testCanBypassSecondaryStorageForUnsupportedDataObject() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject()); + Mockito.doReturn(HypervisorType.KVM).when(srcVolumeInfo).getHypervisorType(); TemplateObject destTemplateInfo = Mockito.spy(new TemplateObject()); @@ -136,12 +137,14 @@ public void testCanBypassSecondaryStorageForUnsupportedDataObject() throws NoSuc @Test public void testCanBypassSecondaryStorageForUnsupportedSrcPoolType() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject()); + Mockito.doReturn(HypervisorType.KVM).when(srcVolumeInfo).getHypervisorType(); DataStore srcDataStore = Mockito.mock(DataStore.class); Mockito.doReturn(new ZoneScope(1L)).when(srcDataStore).getScope(); Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore(); Mockito.doReturn(Storage.StoragePoolType.PowerFlex).when(srcVolumeInfo).getStoragePoolType(); VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject()); + Mockito.doReturn(HypervisorType.KVM).when(destVolumeInfo).getHypervisorType(); DataStore destDataStore = Mockito.mock(DataStore.class); Mockito.doReturn(new ZoneScope(1L)).when(destDataStore).getScope(); Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore(); @@ -157,12 +160,14 @@ public void testCanBypassSecondaryStorageForUnsupportedSrcPoolType() throws NoSu @Test public void testCanBypassSecondaryStorageForUnsupportedDestPoolType() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject()); + Mockito.doReturn(HypervisorType.KVM).when(srcVolumeInfo).getHypervisorType(); DataStore srcDataStore = Mockito.mock(DataStore.class); Mockito.doReturn(new ZoneScope(1L)).when(srcDataStore).getScope(); Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore(); Mockito.doReturn(Storage.StoragePoolType.NetworkFilesystem).when(srcVolumeInfo).getStoragePoolType(); VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject()); + Mockito.doReturn(HypervisorType.KVM).when(destVolumeInfo).getHypervisorType(); DataStore destDataStore = Mockito.mock(DataStore.class); Mockito.doReturn(new ZoneScope(1L)).when(destDataStore).getScope(); Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore(); @@ -178,12 +183,14 @@ public void testCanBypassSecondaryStorageForUnsupportedDestPoolType() throws NoS @Test public void testCanBypassSecondaryStorageWithZoneWideNFSPoolsInSameZone() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject()); + Mockito.doReturn(HypervisorType.KVM).when(srcVolumeInfo).getHypervisorType(); DataStore srcDataStore = Mockito.mock(DataStore.class); Mockito.doReturn(new ZoneScope(1L)).when(srcDataStore).getScope(); Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore(); Mockito.doReturn(Storage.StoragePoolType.NetworkFilesystem).when(srcVolumeInfo).getStoragePoolType(); VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject()); + Mockito.doReturn(HypervisorType.KVM).when(destVolumeInfo).getHypervisorType(); DataStore destDataStore = Mockito.mock(DataStore.class); Mockito.doReturn(new ZoneScope(1L)).when(destDataStore).getScope(); Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore(); @@ -199,12 +206,14 @@ public void testCanBypassSecondaryStorageWithZoneWideNFSPoolsInSameZone() throws @Test public void testCanBypassSecondaryStorageWithClusterWideNFSPoolsInSameCluster() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject()); + Mockito.doReturn(HypervisorType.KVM).when(srcVolumeInfo).getHypervisorType(); DataStore srcDataStore = Mockito.mock(DataStore.class); Mockito.doReturn(new ClusterScope(5L, 2L, 1L)).when(srcDataStore).getScope(); Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore(); Mockito.doReturn(Storage.StoragePoolType.NetworkFilesystem).when(srcVolumeInfo).getStoragePoolType(); VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject()); + Mockito.doReturn(HypervisorType.KVM).when(destVolumeInfo).getHypervisorType(); DataStore destDataStore = Mockito.mock(DataStore.class); Mockito.doReturn(new ClusterScope(5L, 2L, 1L)).when(destDataStore).getScope(); Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore(); @@ -220,12 +229,14 @@ public void testCanBypassSecondaryStorageWithClusterWideNFSPoolsInSameCluster() @Test public void testCanBypassSecondaryStorageWithLocalAndClusterWideNFSPoolsInSameCluster() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject()); + Mockito.doReturn(HypervisorType.KVM).when(srcVolumeInfo).getHypervisorType(); DataStore srcDataStore = Mockito.mock(DataStore.class); Mockito.doReturn(new HostScope(1L, 1L, 1L)).when(srcDataStore).getScope(); Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore(); Mockito.doReturn(Storage.StoragePoolType.Filesystem).when(srcVolumeInfo).getStoragePoolType(); VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject()); + Mockito.doReturn(HypervisorType.KVM).when(destVolumeInfo).getHypervisorType(); DataStore destDataStore = Mockito.mock(DataStore.class); Mockito.doReturn(new ClusterScope(1L, 1L, 1L)).when(destDataStore).getScope(); Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore(); @@ -244,12 +255,14 @@ public void testCanBypassSecondaryStorageWithLocalAndClusterWideNFSPoolsInSameCl @Test public void testCanBypassSecondaryStorageWithLocalAndZoneWideNFSPoolsInSameZone() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject()); + Mockito.doReturn(HypervisorType.KVM).when(srcVolumeInfo).getHypervisorType(); DataStore srcDataStore = Mockito.mock(DataStore.class); Mockito.doReturn(new HostScope(1L, 1L, 1L)).when(srcDataStore).getScope(); Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore(); Mockito.doReturn(Storage.StoragePoolType.Filesystem).when(srcVolumeInfo).getStoragePoolType(); VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject()); + Mockito.doReturn(HypervisorType.KVM).when(destVolumeInfo).getHypervisorType(); DataStore destDataStore = Mockito.mock(DataStore.class); Mockito.doReturn(new ZoneScope(1L)).when(destDataStore).getScope(); Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore(); @@ -268,12 +281,14 @@ public void testCanBypassSecondaryStorageWithLocalAndZoneWideNFSPoolsInSameZone( @Test public void testCanBypassSecondaryStorageWithClusterWideNFSAndZoneWideNFSPoolsInSameZone() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException { VolumeObject srcVolumeInfo = Mockito.spy(new VolumeObject()); + Mockito.doReturn(HypervisorType.KVM).when(srcVolumeInfo).getHypervisorType(); DataStore srcDataStore = Mockito.mock(DataStore.class); Mockito.doReturn(new ClusterScope(5L, 2L, 1L)).when(srcDataStore).getScope(); Mockito.doReturn(srcDataStore).when(srcVolumeInfo).getDataStore(); Mockito.doReturn(Storage.StoragePoolType.NetworkFilesystem).when(srcVolumeInfo).getStoragePoolType(); VolumeObject destVolumeInfo = Mockito.spy(new VolumeObject()); + Mockito.doReturn(HypervisorType.KVM).when(destVolumeInfo).getHypervisorType(); DataStore destDataStore = Mockito.mock(DataStore.class); Mockito.doReturn(new ZoneScope(1L)).when(destDataStore).getScope(); Mockito.doReturn(destDataStore).when(destVolumeInfo).getDataStore(); diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/KvmFileBasedStorageVmSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/KvmFileBasedStorageVmSnapshotStrategy.java index 003065e394f5..79e2a9722039 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/KvmFileBasedStorageVmSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/KvmFileBasedStorageVmSnapshotStrategy.java @@ -28,9 +28,14 @@ import com.cloud.agent.api.storage.RevertDiskOnlyVmSnapshotCommand; import com.cloud.agent.api.storage.SnapshotMergeTreeTO; import com.cloud.agent.api.to.DataTO; +import com.cloud.alert.AlertManager; import com.cloud.configuration.Resource; import com.cloud.event.EventTypes; import com.cloud.event.UsageEventUtils; +import com.cloud.host.DetailVO; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDetailsDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.storage.DataStoreRole; import com.cloud.storage.Snapshot; @@ -46,9 +51,11 @@ import com.cloud.utils.fsm.NoTransitionException; import com.cloud.vm.UserVmVO; import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.VMInstanceDetailsDao; import com.cloud.vm.snapshot.VMSnapshot; import com.cloud.vm.snapshot.VMSnapshotDetailsVO; import com.cloud.vm.snapshot.VMSnapshotVO; +import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.backup.BackupOfferingVO; import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine; @@ -59,10 +66,12 @@ import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.snapshot.SnapshotObject; import org.apache.cloudstack.storage.to.SnapshotObjectTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import javax.inject.Inject; import java.util.ArrayList; @@ -76,6 +85,7 @@ public class KvmFileBasedStorageVmSnapshotStrategy extends StorageVMSnapshotStrategy { private static final List supportedStoragePoolTypes = List.of(Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.NetworkFilesystem, Storage.StoragePoolType.SharedMountPoint); + private static final String KVM_FILE_BASED_STORAGE_SNAPSHOT_NVRAM = "kvmFileBasedStorageSnapshotNvram"; @Inject protected SnapshotDataStoreDao snapshotDataStoreDao; @@ -86,6 +96,15 @@ public class KvmFileBasedStorageVmSnapshotStrategy extends StorageVMSnapshotStra @Inject protected BackupOfferingDao backupOfferingDao; + @Inject + protected VMInstanceDetailsDao vmInstanceDetailsDao; + + @Inject + protected HostDetailsDao hostDetailsDao; + + @Inject + protected AlertManager alertManager; + @Override public VMSnapshot takeVMSnapshot(VMSnapshot vmSnapshot) { Map volumeInfoToSnapshotObjectMap = new HashMap<>(); @@ -116,7 +135,8 @@ public boolean deleteVMSnapshot(VMSnapshot vmSnapshot) { logger.info("Starting VM snapshot delete process for snapshot [{}].", vmSnapshot.getUuid()); UserVmVO userVm = userVmDao.findById(vmSnapshot.getVmId()); VMSnapshotVO vmSnapshotBeingDeleted = (VMSnapshotVO) vmSnapshot; - Long hostId = vmSnapshotHelper.pickRunningHost(vmSnapshotBeingDeleted.getVmId()); + Long hostId = pickHostForNvramSidecarCleanup(vmSnapshotBeingDeleted, userVm, "delete"); + validateHostSupportsNvramSidecarCleanup(vmSnapshotBeingDeleted, hostId, "delete"); long virtualSize = 0; boolean isCurrent = vmSnapshotBeingDeleted.getCurrent(); @@ -124,6 +144,7 @@ public boolean deleteVMSnapshot(VMSnapshot vmSnapshot) { List volumeTOs = vmSnapshotHelper.getVolumeTOList(vmSnapshotBeingDeleted.getVmId()); List snapshotChildren = vmSnapshotDao.listByParentAndStateIn(vmSnapshotBeingDeleted.getId(), VMSnapshot.State.Ready, VMSnapshot.State.Hidden); + PrimaryDataStoreTO nvramPrimaryDataStore = getPrimaryDataStoreForNvramCleanup(vmSnapshotBeingDeleted, volumeTOs); long realSize = getVMSnapshotRealSize(vmSnapshotBeingDeleted); int numberOfChildren = snapshotChildren.size(); @@ -157,6 +178,8 @@ public boolean deleteVMSnapshot(VMSnapshot vmSnapshot) { return true; } + deleteNvramSnapshotIfNeeded(vmSnapshotBeingDeleted, hostId, nvramPrimaryDataStore); + transitStateWithoutThrow(vmSnapshotBeingDeleted, VMSnapshot.Event.OperationSucceeded); vmSnapshotDetailsDao.removeDetails(vmSnapshotBeingDeleted.getId()); @@ -175,7 +198,8 @@ public boolean revertVMSnapshot(VMSnapshot vmSnapshot) { } VMSnapshotVO vmSnapshotBeingReverted = (VMSnapshotVO) vmSnapshot; - Long hostId = vmSnapshotHelper.pickRunningHost(vmSnapshotBeingReverted.getVmId()); + Long hostId = pickHostForUefiNvramAwareDiskOnlySnapshot(userVm, "revert"); + validateHostSupportsUefiNvramAwareDiskOnlySnapshots(hostId, userVm, "revert"); transitStateWithoutThrow(vmSnapshotBeingReverted, VMSnapshot.Event.RevertRequested); @@ -184,7 +208,9 @@ public boolean revertVMSnapshot(VMSnapshot vmSnapshot) { .map(snapshot -> (SnapshotObjectTO) snapshotDataFactory.getSnapshot(snapshot.getSnapshotId(), snapshot.getDataStoreId(), DataStoreRole.Primary).getTO()) .collect(Collectors.toList()); - RevertDiskOnlyVmSnapshotCommand revertDiskOnlyVMSnapshotCommand = new RevertDiskOnlyVmSnapshotCommand(volumeSnapshotTos, userVm.getName()); + RevertDiskOnlyVmSnapshotCommand revertDiskOnlyVMSnapshotCommand = + new RevertDiskOnlyVmSnapshotCommand(volumeSnapshotTos, userVm.getName(), userVm.getUuid(), isUefiVm(userVm), + getNvramSnapshotPath(vmSnapshotBeingReverted)); Answer answer = agentMgr.easySend(hostId, revertDiskOnlyVMSnapshotCommand); if (answer == null || !answer.getResult()) { @@ -204,6 +230,13 @@ public boolean revertVMSnapshot(VMSnapshot vmSnapshot) { publishUsageEvent(EventTypes.EVENT_VM_SNAPSHOT_REVERT, vmSnapshotBeingReverted, userVm, volumeObjectTo); } + if (isUefiVm(userVm) && !Objects.equals(userVm.getLastHostId(), hostId)) { + logger.debug("Updating last host of UEFI VM [{}] to [{}] after disk-only snapshot revert because the NVRAM state was restored on that host.", + userVm.getUuid(), hostId); + userVm.setLastHostId(hostId); + userVmDao.update(userVm.getId(), userVm); + } + transitStateWithoutThrow(vmSnapshotBeingReverted, VMSnapshot.Event.OperationSucceeded); VMSnapshotVO currentVmSnapshot = vmSnapshotDao.findCurrentSnapshotByVmId(userVm.getId()); @@ -248,6 +281,8 @@ private void mergeOldSiblingWithOldParentIfOldParentIsDead(VMSnapshotVO oldParen return; } + validateHostSupportsNvramSidecarCleanup(oldParent, hostId, "clean up"); + PrimaryDataStoreTO nvramPrimaryDataStore = getPrimaryDataStoreForNvramCleanup(oldParent, volumeTOs); List snapshotVos; if (oldParent.getCurrent()) { @@ -276,6 +311,8 @@ private void mergeOldSiblingWithOldParentIfOldParentIsDead(VMSnapshotVO oldParen snapshotDao.update(snapshotVO.getId(), snapshotVO); } + deleteNvramSnapshotIfNeeded(oldParent, hostId, nvramPrimaryDataStore); + vmSnapshotDetailsDao.removeDetails(oldParent.getId()); oldParent.setRemoved(DateUtil.now()); @@ -347,12 +384,13 @@ public StrategyPriority canHandle(Long vmId, Long rootPoolId, boolean snapshotMe } private List deleteSnapshot(VMSnapshotVO vmSnapshotVO, Long hostId) { + validateHostSupportsNvramSidecarCleanup(vmSnapshotVO, hostId, "delete"); List volumeSnapshots = getVolumeSnapshotsAssociatedWithVmSnapshot(vmSnapshotVO); List volumeSnapshotTOList = volumeSnapshots.stream() .map(snapshotDataStoreVO -> snapshotDataFactory.getSnapshot(snapshotDataStoreVO.getSnapshotId(), snapshotDataStoreVO.getDataStoreId(), DataStoreRole.Primary).getTO()) .collect(Collectors.toList()); - DeleteDiskOnlyVmSnapshotCommand deleteSnapshotCommand = new DeleteDiskOnlyVmSnapshotCommand(volumeSnapshotTOList); + DeleteDiskOnlyVmSnapshotCommand deleteSnapshotCommand = new DeleteDiskOnlyVmSnapshotCommand(volumeSnapshotTOList, getNvramSnapshotPath(vmSnapshotVO)); Answer answer = agentMgr.easySend(hostId, deleteSnapshotCommand); if (answer == null || !answer.getResult()) { logger.error("Failed to delete VM snapshot [{}] due to {}.", vmSnapshotVO.getUuid(), answer != null ? answer.getDetails() : "Communication failure"); @@ -368,6 +406,20 @@ private List deleteSnapshot(VMSnapshotVO vmSnapshotVO, Long hostId) return snapshotVOList; } + protected void deleteNvramSnapshotIfNeeded(VMSnapshotVO vmSnapshotVO, Long hostId, PrimaryDataStoreTO primaryDataStore) { + String nvramSnapshotPath = getNvramSnapshotPath(vmSnapshotVO); + if (StringUtils.isBlank(nvramSnapshotPath) || primaryDataStore == null) { + return; + } + + DeleteDiskOnlyVmSnapshotCommand deleteSnapshotCommand = new DeleteDiskOnlyVmSnapshotCommand(List.of(), nvramSnapshotPath, primaryDataStore); + Answer answer = agentMgr.easySend(hostId, deleteSnapshotCommand); + if (answer == null || !answer.getResult()) { + logger.warn("Failed to delete the NVRAM sidecar of VM snapshot [{}] due to {}.", vmSnapshotVO.getUuid(), + answer != null ? answer.getDetails() : "communication failure"); + } + } + private List mergeSnapshots(VMSnapshotVO vmSnapshotVO, VMSnapshotVO childSnapshot, UserVmVO userVm, List volumeObjectTOS, Long hostId) { logger.debug("Merging VM snapshot [{}] with its child [{}].", vmSnapshotVO.getUuid(), childSnapshot.getUuid()); @@ -470,12 +522,13 @@ protected VMSnapshot takeVmSnapshotInternal(VMSnapshot vmSnapshot, Map volumeTOs = vmSnapshotHelper.getVolumeTOList(userVm.getId()); - transitStateWithoutThrow(vmSnapshot, VMSnapshot.Event.CreateRequested); - VMSnapshotTO parentSnapshotTo = null; VMSnapshotVO parentSnapshotVo = vmSnapshotDao.findCurrentSnapshotByVmId(userVm.getId()); if (parentSnapshotVo != null) { @@ -493,14 +546,18 @@ protected VMSnapshot takeVmSnapshotInternal(VMSnapshot vmSnapshot, Map volumeTOs) { + return (PrimaryDataStoreTO) volumeTOs.stream() + .filter(volumeObjectTO -> Volume.Type.ROOT.equals(volumeObjectTO.getVolumeType())) + .findFirst() + .orElseThrow(() -> new CloudRuntimeException("Failed to locate the root volume while handling the VM snapshot.")) + .getDataStore(); + } + + protected PrimaryDataStoreTO getRootVolumePrimaryDataStoreForCleanup(VMSnapshotVO vmSnapshot, List volumeTOs) { + try { + return getRootVolumePrimaryDataStore(volumeTOs); + } catch (CloudRuntimeException e) { + logger.warn("Failed to locate the root volume while cleaning up the NVRAM sidecar for VM snapshot [{}].", vmSnapshot.getUuid(), e); + return null; + } + } + + protected PrimaryDataStoreTO getPrimaryDataStoreForNvramCleanup(VMSnapshotVO vmSnapshot, List volumeTOs) { + PrimaryDataStoreTO rootSnapshotPrimaryDataStore = getRootSnapshotPrimaryDataStoreForCleanup(vmSnapshot); + return rootSnapshotPrimaryDataStore != null ? rootSnapshotPrimaryDataStore : getRootVolumePrimaryDataStoreForCleanup(vmSnapshot, volumeTOs); + } + + protected PrimaryDataStoreTO getRootSnapshotPrimaryDataStoreForCleanup(VMSnapshotVO vmSnapshot) { + try { + return (PrimaryDataStoreTO) getVolumeSnapshotsAssociatedWithVmSnapshot(vmSnapshot).stream() + .map(snapshotDataStoreVO -> (SnapshotObjectTO) snapshotDataFactory.getSnapshot(snapshotDataStoreVO.getSnapshotId(), + snapshotDataStoreVO.getDataStoreId(), DataStoreRole.Primary).getTO()) + .filter(snapshotObjectTO -> Volume.Type.ROOT.equals(snapshotObjectTO.getVolume().getVolumeType())) + .findFirst() + .orElseThrow(() -> new CloudRuntimeException("Failed to locate the root volume snapshot while handling the VM snapshot.")) + .getDataStore(); + } catch (CloudRuntimeException e) { + logger.warn("Failed to locate the root volume snapshot while cleaning up the NVRAM sidecar for VM snapshot [{}].", vmSnapshot.getUuid(), e); + return null; + } + } + + protected String getNvramSnapshotPath(VMSnapshotVO vmSnapshot) { + VMSnapshotDetailsVO nvramDetail = vmSnapshotDetailsDao.findDetail(vmSnapshot.getId(), KVM_FILE_BASED_STORAGE_SNAPSHOT_NVRAM); + return nvramDetail != null ? nvramDetail.getValue() : null; + } + + protected Long pickHostForUefiNvramAwareDiskOnlySnapshot(UserVm userVm, String operation) { + Long selectedHostId = vmSnapshotHelper.pickRunningHost(userVm.getId()); + if (!isUefiVm(userVm)) { + return selectedHostId; + } + + boolean isCreate = "create".equals(operation); + if (isCreate) { + validateUefiSnapshotCreateHostOwnsActiveNvram(userVm, selectedHostId); + } + + return pickHostWithRequiredCapabilities(userVm, selectedHostId, operation, !isCreate, + List.of(Host.HOST_UEFI_ENABLE, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM)); + } + + protected void validateUefiSnapshotCreateHostOwnsActiveNvram(UserVm userVm, Long selectedHostId) { + if (VirtualMachine.State.Running.equals(userVm.getState())) { + return; + } + + Long lastHostId = userVm.getLastHostId(); + if (lastHostId == null || !Objects.equals(lastHostId, selectedHostId)) { + throw new CloudRuntimeException(String.format("Cannot create a disk-only snapshot for stopped UEFI VM [%s] on host [%s] because the active NVRAM " + + "state is expected on last host [%s]. Make the last host available or start the VM on a UEFI-capable KVM host before retrying.", + userVm.getUuid(), selectedHostId, lastHostId)); + } + } + + protected Long pickHostForNvramSidecarCleanup(VMSnapshotVO vmSnapshotVO, UserVm userVm, String operation) { + Long selectedHostId = vmSnapshotHelper.pickRunningHost(vmSnapshotVO.getVmId()); + if (StringUtils.isBlank(getNvramSnapshotPath(vmSnapshotVO))) { + return selectedHostId; + } + + return pickHostWithRequiredCapabilities(userVm, selectedHostId, operation, true, List.of(Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM)); + } + + protected Long pickHostWithRequiredCapabilities(UserVm userVm, Long selectedHostId, String operation, boolean canFallbackFromSelectedHost, + List requiredCapabilities) { + if (hostSupportsCapabilities(selectedHostId, requiredCapabilities)) { + return selectedHostId; + } + + if (VirtualMachine.State.Running.equals(userVm.getState()) || !canFallbackFromSelectedHost) { + return selectedHostId; + } + + return listCandidateHostsForVmSnapshot(userVm).stream() + .filter(host -> hostSupportsCapabilities(host.getId(), requiredCapabilities)) + .findFirst() + .map(HostVO::getId) + .orElseThrow(() -> new CloudRuntimeException(String.format("Cannot %s disk-only snapshot state for VM [%s] because no Up and Enabled host in the " + + "VM storage scope advertises [%s].", operation, userVm.getUuid(), String.join(", ", requiredCapabilities)))); + } + + protected List listCandidateHostsForVmSnapshot(UserVm userVm) { + List volumes = volumeDao.findByInstance(userVm.getId()); + if (CollectionUtils.isEmpty(volumes)) { + throw new CloudRuntimeException(String.format("Cannot find a host for VM snapshot operation because VM [%s] has no volumes.", userVm.getUuid())); + } + + VolumeVO volume = volumes.stream() + .filter(volumeVO -> Volume.Type.ROOT.equals(volumeVO.getVolumeType())) + .findFirst() + .orElse(volumes.get(0)); + Long poolId = volume.getPoolId(); + if (poolId == null) { + throw new CloudRuntimeException(String.format("Cannot find a host for VM snapshot operation because volume [%s] has no pool.", volume.getUuid())); + } + + StoragePoolVO storagePoolVO = storagePool.findById(poolId); + if (storagePoolVO == null) { + throw new CloudRuntimeException(String.format("Cannot find a host for VM snapshot operation because storage pool [%s] was not found.", poolId)); + } + + List hosts = hostDao.listAllUpAndEnabledNonHAHosts(Host.Type.Routing, storagePoolVO.getClusterId(), storagePoolVO.getPodId(), + storagePoolVO.getDataCenterId(), null); + if (CollectionUtils.isEmpty(hosts)) { + throw new CloudRuntimeException(String.format("Cannot find a host for VM snapshot operation because no Up and Enabled host was found in storage pool [%s] scope.", + storagePoolVO.getUuid())); + } + return hosts; + } + + protected boolean hostSupportsCapabilities(Long hostId, List requiredCapabilities) { + if (hostId == null || CollectionUtils.isEmpty(requiredCapabilities)) { + return false; + } + return requiredCapabilities.stream().allMatch(capability -> isHostCapabilityEnabled(hostId, capability)); + } + + protected void validateHostSupportsUefiNvramAwareDiskOnlySnapshots(Long hostId, UserVm userVm, String operation) { + if (!isUefiVm(userVm)) { + return; + } + + if (!isHostCapabilityEnabled(hostId, Host.HOST_UEFI_ENABLE)) { + throw new CloudRuntimeException(String.format("Cannot %s a disk-only snapshot for UEFI VM [%s] on host [%s] because the host does not advertise " + + "UEFI support. Ensure the host is configured with UEFI support and retry.", operation, userVm.getUuid(), hostId)); + } + + if (!isHostCapabilityEnabled(hostId, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM)) { + throw new CloudRuntimeException(String.format("Cannot %s a disk-only snapshot for UEFI VM [%s] on host [%s] because the KVM agent does not advertise " + + "NVRAM-aware disk-only snapshot support. Upgrade the host and retry.", operation, userVm.getUuid(), hostId)); + } + } + + protected boolean isHostCapabilityEnabled(Long hostId, String capabilityName) { + DetailVO hostCapability = hostDetailsDao.findDetail(hostId, capabilityName); + return hostCapability != null && Boolean.parseBoolean(hostCapability.getValue()); + } + + protected void validateHostSupportsNvramSidecarCleanup(VMSnapshotVO vmSnapshotVO, Long hostId, String operation) { + if (StringUtils.isBlank(getNvramSnapshotPath(vmSnapshotVO))) { + return; + } + + if (!isHostCapabilityEnabled(hostId, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM)) { + throw new CloudRuntimeException(String.format("Cannot %s VM snapshot [%s] on host [%s] because the KVM agent does not advertise " + + "NVRAM-aware disk-only snapshot support and the snapshot has an NVRAM sidecar that must be cleaned up. Upgrade the host and retry.", + operation, vmSnapshotVO.getUuid(), hostId)); + } + } + + protected void notifyGuestRecoveryIssueIfNeeded(CreateDiskOnlyVmSnapshotAnswer answer, UserVm userVm, VMSnapshotVO vmSnapshot) { + if (StringUtils.isBlank(answer.getDetails())) { + return; + } + + String subject = String.format("Disk-only VM snapshot [%s] completed with guest recovery warnings", vmSnapshot.getUuid()); + String message = String.format("Disk-only VM snapshot [%s] for UEFI VM [%s] completed, but post-snapshot guest recovery reported: %s", + vmSnapshot.getUuid(), userVm.getUuid(), answer.getDetails()); + logger.error(message); + try { + alertManager.sendAlert(AlertManager.AlertType.ALERT_TYPE_VM_SNAPSHOT, userVm.getDataCenterId(), userVm.getPodIdToDeployIn(), subject, message); + } catch (Exception e) { + logger.warn("Failed to send post-snapshot guest recovery alert for VM snapshot [{}].", vmSnapshot.getUuid(), e); + } + } + /** * Given a list of VM snapshots, will remove any that are part of the current direct backing chain (all the direct ancestors of the current vm snapshot). * This is done because, when using virDomainBlockCommit}, Libvirt will maintain diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java index aced750bd320..a18d65748cf1 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/vmsnapshot/ScaleIOVMSnapshotStrategy.java @@ -141,7 +141,7 @@ public StrategyPriority canHandle(Long vmId, Long rootPoolId, boolean snapshotMe for (VolumeObjectTO volumeTO : volumeTOs) { Long poolId = volumeTO.getPoolId(); Storage.StoragePoolType poolType = vmSnapshotHelper.getStoragePoolType(poolId); - if (poolType != Storage.StoragePoolType.PowerFlex || volumeTO.getFormat() != ImageFormat.RAW || poolId != rootPoolId) { + if (poolType != Storage.StoragePoolType.PowerFlex || volumeTO.getFormat() != ImageFormat.RAW || !poolId.equals(rootPoolId)) { return StrategyPriority.CANT_HANDLE; } } diff --git a/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/KvmFileBasedStorageVmSnapshotStrategyTest.java b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/KvmFileBasedStorageVmSnapshotStrategyTest.java new file mode 100644 index 000000000000..35b1f9ff65bf --- /dev/null +++ b/engine/storage/snapshot/src/test/java/org/apache/cloudstack/storage/vmsnapshot/KvmFileBasedStorageVmSnapshotStrategyTest.java @@ -0,0 +1,550 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.vmsnapshot; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.lang.reflect.Method; +import java.util.Collections; +import java.util.List; + +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotDataFactory; +import org.apache.cloudstack.engine.subsystem.api.storage.SnapshotInfo; +import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreDao; +import org.apache.cloudstack.storage.datastore.db.SnapshotDataStoreVO; +import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.api.ApiConstants; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; +import org.mockito.Mockito; + +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.storage.CreateDiskOnlyVmSnapshotAnswer; +import com.cloud.agent.api.storage.DeleteDiskOnlyVmSnapshotCommand; +import com.cloud.agent.api.storage.RevertDiskOnlyVmSnapshotAnswer; +import com.cloud.agent.api.storage.RevertDiskOnlyVmSnapshotCommand; +import com.cloud.alert.AlertManager; +import com.cloud.host.DetailVO; +import com.cloud.host.Host; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.host.dao.HostDetailsDao; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.Volume; +import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.ResourceLimitService; +import com.cloud.uservm.UserVm; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VMInstanceDetailVO; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.dao.VMInstanceDetailsDao; +import com.cloud.vm.dao.UserVmDao; +import com.cloud.vm.snapshot.VMSnapshot; +import com.cloud.vm.snapshot.VMSnapshotDetailsVO; +import com.cloud.vm.snapshot.VMSnapshotVO; +import com.cloud.vm.snapshot.dao.VMSnapshotDao; +import com.cloud.vm.snapshot.dao.VMSnapshotDetailsDao; + +public class KvmFileBasedStorageVmSnapshotStrategyTest { + + private KvmFileBasedStorageVmSnapshotStrategy strategy; + private VMSnapshotDetailsDao vmSnapshotDetailsDao; + private VMSnapshotDao vmSnapshotDao; + private VMSnapshotHelper vmSnapshotHelper; + private AgentManager agentMgr; + private SnapshotDataStoreDao snapshotDataStoreDao; + private HostDetailsDao hostDetailsDao; + private HostDao hostDao; + private PrimaryDataStoreDao storagePoolDao; + + @Before + public void setup() { + strategy = Mockito.spy(new KvmFileBasedStorageVmSnapshotStrategy()); + vmSnapshotDetailsDao = mock(VMSnapshotDetailsDao.class); + vmSnapshotDao = mock(VMSnapshotDao.class); + vmSnapshotHelper = mock(VMSnapshotHelper.class); + agentMgr = mock(AgentManager.class); + snapshotDataStoreDao = mock(SnapshotDataStoreDao.class); + hostDetailsDao = mock(HostDetailsDao.class); + hostDao = mock(HostDao.class); + storagePoolDao = mock(PrimaryDataStoreDao.class); + + strategy.vmSnapshotDetailsDao = vmSnapshotDetailsDao; + strategy.vmSnapshotDao = vmSnapshotDao; + strategy.vmSnapshotHelper = vmSnapshotHelper; + strategy.agentMgr = agentMgr; + strategy.snapshotDataStoreDao = snapshotDataStoreDao; + strategy.hostDao = hostDao; + strategy.storagePool = storagePoolDao; + strategy.resourceLimitManager = mock(ResourceLimitService.class); + strategy.snapshotDataFactory = mock(SnapshotDataFactory.class); + strategy.userVmDao = mock(UserVmDao.class); + strategy.volumeDao = mock(VolumeDao.class); + strategy.vmInstanceDetailsDao = mock(VMInstanceDetailsDao.class); + strategy.hostDetailsDao = hostDetailsDao; + strategy.alertManager = mock(AlertManager.class); + doNothing().when(strategy).publishUsageEvent(anyString(), any(VMSnapshot.class), any(UserVm.class), anyLong(), anyLong()); + doNothing().when(strategy).publishUsageEvent(anyString(), any(VMSnapshot.class), any(UserVm.class), any(VolumeObjectTO.class)); + } + + @Test + public void testProcessCreateVmSnapshotAnswerPersistsNvramPath() throws Exception { + VMSnapshot vmSnapshot = mock(VMSnapshot.class); + CreateDiskOnlyVmSnapshotAnswer answer = mock(CreateDiskOnlyVmSnapshotAnswer.class); + UserVm userVm = mock(UserVm.class); + VMSnapshotVO vmSnapshotVO = mock(VMSnapshotVO.class); + + when(vmSnapshot.getId()).thenReturn(42L); + when(vmSnapshot.getUuid()).thenReturn("vm-snapshot"); + when(answer.getMapVolumeToSnapshotSizeAndNewVolumePath()).thenReturn(Collections.emptyMap()); + when(answer.getNvramSnapshotPath()).thenReturn("nvram/42.fd"); + + Method method = KvmFileBasedStorageVmSnapshotStrategy.class.getDeclaredMethod("processCreateVmSnapshotAnswer", VMSnapshot.class, java.util.Map.class, + CreateDiskOnlyVmSnapshotAnswer.class, UserVm.class, VMSnapshotVO.class, long.class, VMSnapshotVO.class); + method.setAccessible(true); + method.invoke(strategy, vmSnapshot, Collections.emptyMap(), answer, userVm, vmSnapshotVO, 0L, null); + + verify(vmSnapshotDetailsDao).addDetail(42L, "kvmFileBasedStorageSnapshotNvram", "nvram/42.fd", false); + } + + @Test + public void testRevertVMSnapshotPassesNvramPathToAgentCommand() { + long vmId = 10L; + long snapshotId = 20L; + long dataStoreId = 30L; + long hostId = 40L; + + UserVmVO userVm = mock(UserVmVO.class); + VMSnapshotVO vmSnapshot = mock(VMSnapshotVO.class); + VMSnapshotVO currentVmSnapshot = mock(VMSnapshotVO.class); + SnapshotDataStoreVO snapshotDataStoreVO = mock(SnapshotDataStoreVO.class); + SnapshotInfo snapshotInfo = mock(SnapshotInfo.class); + SnapshotObjectTO snapshotObjectTO = mock(SnapshotObjectTO.class); + VMSnapshotDetailsVO volumeSnapshotDetail = new VMSnapshotDetailsVO(snapshotId, "kvmFileBasedStorageSnapshot", String.valueOf(snapshotId), true); + VMSnapshotDetailsVO nvramDetail = new VMSnapshotDetailsVO(snapshotId, "kvmFileBasedStorageSnapshotNvram", "nvram/42.fd", false); + + when(vmSnapshot.getVmId()).thenReturn(vmId); + when(vmSnapshot.getId()).thenReturn(snapshotId); + when(vmSnapshot.getUuid()).thenReturn("vm-snapshot"); + when(userVm.getState()).thenReturn(VirtualMachine.State.Stopped); + when(userVm.getName()).thenReturn("i-10-VM"); + when(userVm.getUuid()).thenReturn("vm-uuid"); + when(userVm.getId()).thenReturn(vmId); + when(userVm.getLastHostId()).thenReturn(39L); + when(strategy.userVmDao.findById(vmId)).thenReturn(userVm); + when(strategy.vmInstanceDetailsDao.findDetail(vmId, ApiConstants.BootType.UEFI.toString())) + .thenReturn(new VMInstanceDetailVO(vmId, ApiConstants.BootType.UEFI.toString(), "SECURE", true)); + when(vmSnapshotHelper.pickRunningHost(vmId)).thenReturn(hostId); + when(hostDetailsDao.findDetail(hostId, Host.HOST_UEFI_ENABLE)) + .thenReturn(new DetailVO(hostId, Host.HOST_UEFI_ENABLE, Boolean.TRUE.toString())); + when(hostDetailsDao.findDetail(hostId, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM)) + .thenReturn(new DetailVO(hostId, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM, Boolean.TRUE.toString())); + when(vmSnapshotDetailsDao.findDetails(snapshotId, "kvmFileBasedStorageSnapshot")).thenReturn(List.of(volumeSnapshotDetail)); + when(vmSnapshotDetailsDao.findDetail(snapshotId, "kvmFileBasedStorageSnapshotNvram")).thenReturn(nvramDetail); + when(snapshotDataStoreDao.findOneBySnapshotAndDatastoreRole(snapshotId, DataStoreRole.Primary)).thenReturn(snapshotDataStoreVO); + when(snapshotDataStoreVO.getSnapshotId()).thenReturn(snapshotId); + when(snapshotDataStoreVO.getDataStoreId()).thenReturn(dataStoreId); + when(strategy.snapshotDataFactory.getSnapshot(snapshotId, dataStoreId, DataStoreRole.Primary)).thenReturn(snapshotInfo); + when(snapshotInfo.getTO()).thenReturn(snapshotObjectTO); + when(vmSnapshotDao.findCurrentSnapshotByVmId(vmId)).thenReturn(currentVmSnapshot); + when(agentMgr.easySend(eq(hostId), any())).thenAnswer(invocation -> + new RevertDiskOnlyVmSnapshotAnswer((RevertDiskOnlyVmSnapshotCommand) invocation.getArgument(1), Collections.emptyList())); + + ArgumentCaptor commandCaptor = ArgumentCaptor.forClass(RevertDiskOnlyVmSnapshotCommand.class); + + strategy.revertVMSnapshot(vmSnapshot); + + verify(agentMgr).easySend(eq(hostId), commandCaptor.capture()); + verify(userVm).setLastHostId(hostId); + verify(strategy.userVmDao).update(vmId, userVm); + assertEquals("vm-uuid", commandCaptor.getValue().getVmUuid()); + assertEquals(true, commandCaptor.getValue().isUefiEnabled()); + assertEquals("nvram/42.fd", commandCaptor.getValue().getNvramSnapshotPath()); + } + + @Test + public void testPickHostForUefiNvramAwareDiskOnlySnapshotUsesCapableCandidateForRevertWhenDefaultHostLacksSupport() { + long vmId = 10L; + long defaultHostId = 40L; + long capableHostId = 41L; + long poolId = 50L; + long clusterId = 60L; + long podId = 70L; + long dataCenterId = 80L; + + UserVmVO userVm = mock(UserVmVO.class); + VolumeVO rootVolume = mock(VolumeVO.class); + StoragePoolVO storagePool = mock(StoragePoolVO.class); + HostVO defaultHost = mock(HostVO.class); + HostVO capableHost = mock(HostVO.class); + + when(userVm.getId()).thenReturn(vmId); + when(userVm.getUuid()).thenReturn("vm-uuid"); + when(userVm.getState()).thenReturn(VirtualMachine.State.Stopped); + when(strategy.vmInstanceDetailsDao.findDetail(vmId, ApiConstants.BootType.UEFI.toString())) + .thenReturn(new VMInstanceDetailVO(vmId, ApiConstants.BootType.UEFI.toString(), "SECURE", true)); + when(vmSnapshotHelper.pickRunningHost(vmId)).thenReturn(defaultHostId); + when(rootVolume.getVolumeType()).thenReturn(Volume.Type.ROOT); + when(rootVolume.getPoolId()).thenReturn(poolId); + when(strategy.volumeDao.findByInstance(vmId)).thenReturn(List.of(rootVolume)); + when(storagePoolDao.findById(poolId)).thenReturn(storagePool); + when(storagePool.getClusterId()).thenReturn(clusterId); + when(storagePool.getPodId()).thenReturn(podId); + when(storagePool.getDataCenterId()).thenReturn(dataCenterId); + when(defaultHost.getId()).thenReturn(defaultHostId); + when(capableHost.getId()).thenReturn(capableHostId); + when(hostDao.listAllUpAndEnabledNonHAHosts(Host.Type.Routing, clusterId, podId, dataCenterId, null)) + .thenReturn(List.of(defaultHost, capableHost)); + when(hostDetailsDao.findDetail(defaultHostId, Host.HOST_UEFI_ENABLE)).thenReturn(null); + when(hostDetailsDao.findDetail(defaultHostId, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM)).thenReturn(null); + when(hostDetailsDao.findDetail(capableHostId, Host.HOST_UEFI_ENABLE)) + .thenReturn(new DetailVO(capableHostId, Host.HOST_UEFI_ENABLE, Boolean.TRUE.toString())); + when(hostDetailsDao.findDetail(capableHostId, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM)) + .thenReturn(new DetailVO(capableHostId, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM, Boolean.TRUE.toString())); + + Long hostId = strategy.pickHostForUefiNvramAwareDiskOnlySnapshot(userVm, "revert"); + + assertEquals(Long.valueOf(capableHostId), hostId); + } + + @Test(expected = CloudRuntimeException.class) + public void testPickHostForUefiNvramAwareDiskOnlySnapshotFailsCreateWhenSelectedHostIsNotLastHost() { + long vmId = 10L; + long lastHostId = 39L; + long selectedHostId = 40L; + + UserVmVO userVm = mock(UserVmVO.class); + + when(userVm.getId()).thenReturn(vmId); + when(userVm.getUuid()).thenReturn("vm-uuid"); + when(userVm.getState()).thenReturn(VirtualMachine.State.Stopped); + when(userVm.getLastHostId()).thenReturn(lastHostId); + when(strategy.vmInstanceDetailsDao.findDetail(vmId, ApiConstants.BootType.UEFI.toString())) + .thenReturn(new VMInstanceDetailVO(vmId, ApiConstants.BootType.UEFI.toString(), "SECURE", true)); + when(vmSnapshotHelper.pickRunningHost(vmId)).thenReturn(selectedHostId); + when(hostDetailsDao.findDetail(selectedHostId, Host.HOST_UEFI_ENABLE)) + .thenReturn(new DetailVO(selectedHostId, Host.HOST_UEFI_ENABLE, Boolean.TRUE.toString())); + when(hostDetailsDao.findDetail(selectedHostId, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM)) + .thenReturn(new DetailVO(selectedHostId, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM, Boolean.TRUE.toString())); + + strategy.pickHostForUefiNvramAwareDiskOnlySnapshot(userVm, "create"); + } + + @Test + public void testDeleteNvramSnapshotIfNeededPassesPrimaryDataStoreToAgentCommand() { + long hostId = 40L; + + VMSnapshotVO vmSnapshot = mock(VMSnapshotVO.class); + VMSnapshotDetailsVO nvramDetail = new VMSnapshotDetailsVO(20L, "kvmFileBasedStorageSnapshotNvram", "nvram/42.fd", false); + PrimaryDataStoreTO primaryDataStore = mock(PrimaryDataStoreTO.class); + + when(vmSnapshot.getId()).thenReturn(20L); + when(vmSnapshot.getUuid()).thenReturn("vm-snapshot"); + when(vmSnapshotDetailsDao.findDetail(20L, "kvmFileBasedStorageSnapshotNvram")).thenReturn(nvramDetail); + when(hostDetailsDao.findDetail(hostId, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM)) + .thenReturn(new DetailVO(hostId, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM, Boolean.TRUE.toString())); + when(agentMgr.easySend(eq(hostId), any())).thenReturn(new Answer(null, true, null)); + + strategy.deleteNvramSnapshotIfNeeded(vmSnapshot, hostId, primaryDataStore); + + verify(agentMgr).easySend(eq(hostId), argThat(command -> { + if (!(command instanceof DeleteDiskOnlyVmSnapshotCommand)) { + return false; + } + + DeleteDiskOnlyVmSnapshotCommand deleteCommand = (DeleteDiskOnlyVmSnapshotCommand) command; + return deleteCommand.getSnapshots().isEmpty() + && "nvram/42.fd".equals(deleteCommand.getNvramSnapshotPath()) + && deleteCommand.getPrimaryDataStore() == primaryDataStore; + })); + } + + @Test(expected = CloudRuntimeException.class) + public void testValidateHostSupportsNvramSidecarCleanupFailsWhenHostLacksNvramAwareCleanupCapability() { + long hostId = 40L; + + VMSnapshotVO vmSnapshot = mock(VMSnapshotVO.class); + VMSnapshotDetailsVO nvramDetail = new VMSnapshotDetailsVO(20L, "kvmFileBasedStorageSnapshotNvram", "nvram/42.fd", false); + + when(vmSnapshot.getId()).thenReturn(20L); + when(vmSnapshot.getUuid()).thenReturn("vm-snapshot"); + when(vmSnapshotDetailsDao.findDetail(20L, "kvmFileBasedStorageSnapshotNvram")).thenReturn(nvramDetail); + when(hostDetailsDao.findDetail(hostId, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM)).thenReturn(null); + + try { + strategy.validateHostSupportsNvramSidecarCleanup(vmSnapshot, hostId, "delete"); + } finally { + verify(agentMgr, never()).easySend(eq(hostId), any()); + } + } + + @Test + public void testGetRootVolumePrimaryDataStoreForCleanupReturnsNullWhenRootVolumeIsMissing() { + VMSnapshotVO vmSnapshot = mock(VMSnapshotVO.class); + VolumeObjectTO dataVolume = mock(VolumeObjectTO.class); + + when(vmSnapshot.getUuid()).thenReturn("vm-snapshot"); + when(dataVolume.getVolumeType()).thenReturn(Volume.Type.DATADISK); + + PrimaryDataStoreTO primaryDataStore = strategy.getRootVolumePrimaryDataStoreForCleanup(vmSnapshot, List.of(dataVolume)); + + assertNull(primaryDataStore); + } + + @Test + public void testGetPrimaryDataStoreForNvramCleanupPrefersRootSnapshotPrimaryDataStore() { + long vmSnapshotId = 20L; + long rootSnapshotId = 30L; + long dataStoreId = 40L; + + VMSnapshotVO vmSnapshot = mock(VMSnapshotVO.class); + VolumeObjectTO rootVolume = mock(VolumeObjectTO.class); + PrimaryDataStoreTO liveRootVolumePrimaryDataStore = mock(PrimaryDataStoreTO.class); + PrimaryDataStoreTO rootSnapshotPrimaryDataStore = mock(PrimaryDataStoreTO.class); + SnapshotDataStoreVO rootSnapshotDataStore = mock(SnapshotDataStoreVO.class); + SnapshotInfo rootSnapshotInfo = mock(SnapshotInfo.class); + SnapshotObjectTO rootSnapshotObjectTo = mock(SnapshotObjectTO.class); + VolumeObjectTO rootSnapshotVolume = mock(VolumeObjectTO.class); + VMSnapshotDetailsVO volumeSnapshotDetail = new VMSnapshotDetailsVO(vmSnapshotId, "kvmFileBasedStorageSnapshot", String.valueOf(rootSnapshotId), true); + + when(vmSnapshot.getId()).thenReturn(vmSnapshotId); + when(vmSnapshot.getUuid()).thenReturn("vm-snapshot"); + when(vmSnapshotDetailsDao.findDetails(vmSnapshotId, "kvmFileBasedStorageSnapshot")).thenReturn(List.of(volumeSnapshotDetail)); + when(snapshotDataStoreDao.findOneBySnapshotAndDatastoreRole(rootSnapshotId, DataStoreRole.Primary)).thenReturn(rootSnapshotDataStore); + when(rootSnapshotDataStore.getSnapshotId()).thenReturn(rootSnapshotId); + when(rootSnapshotDataStore.getDataStoreId()).thenReturn(dataStoreId); + when(strategy.snapshotDataFactory.getSnapshot(rootSnapshotId, dataStoreId, DataStoreRole.Primary)).thenReturn(rootSnapshotInfo); + when(rootSnapshotInfo.getTO()).thenReturn(rootSnapshotObjectTo); + when(rootSnapshotObjectTo.getVolume()).thenReturn(rootSnapshotVolume); + when(rootSnapshotVolume.getVolumeType()).thenReturn(Volume.Type.ROOT); + when(rootSnapshotObjectTo.getDataStore()).thenReturn(rootSnapshotPrimaryDataStore); + when(rootVolume.getVolumeType()).thenReturn(Volume.Type.ROOT); + when(rootVolume.getDataStore()).thenReturn(liveRootVolumePrimaryDataStore); + + PrimaryDataStoreTO primaryDataStore = strategy.getPrimaryDataStoreForNvramCleanup(vmSnapshot, List.of(rootVolume)); + + assertSame(rootSnapshotPrimaryDataStore, primaryDataStore); + } + + @Test(expected = CloudRuntimeException.class) + public void testTakeVmSnapshotInternalFailsWhenHostLacksNvramAwareSnapshotCapabilityForUefiVm() throws Exception { + long vmId = 10L; + long hostId = 40L; + + UserVmVO userVm = mock(UserVmVO.class); + VMSnapshotVO vmSnapshot = mock(VMSnapshotVO.class); + + when(vmSnapshot.getVmId()).thenReturn(vmId); + when(userVm.getId()).thenReturn(vmId); + when(userVm.getUuid()).thenReturn("vm-uuid"); + when(userVm.getState()).thenReturn(VirtualMachine.State.Running); + when(strategy.userVmDao.findById(vmId)).thenReturn(userVm); + when(vmSnapshotHelper.pickRunningHost(vmId)).thenReturn(hostId); + when(strategy.vmInstanceDetailsDao.findDetail(vmId, ApiConstants.BootType.UEFI.toString())) + .thenReturn(new VMInstanceDetailVO(vmId, ApiConstants.BootType.UEFI.toString(), "SECURE", true)); + when(hostDetailsDao.findDetail(hostId, Host.HOST_UEFI_ENABLE)) + .thenReturn(new DetailVO(hostId, Host.HOST_UEFI_ENABLE, Boolean.TRUE.toString())); + when(hostDetailsDao.findDetail(hostId, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM)).thenReturn(null); + + strategy.takeVmSnapshotInternal(vmSnapshot, Collections.emptyMap()); + } + + @Test(expected = CloudRuntimeException.class) + public void testTakeVMSnapshotMarksSnapshotFailedWhenHostCapabilityValidationFails() throws Exception { + long vmId = 10L; + long hostId = 40L; + + UserVmVO userVm = mock(UserVmVO.class); + VMSnapshotVO vmSnapshot = mock(VMSnapshotVO.class); + + when(vmSnapshot.getVmId()).thenReturn(vmId); + when(userVm.getId()).thenReturn(vmId); + when(userVm.getUuid()).thenReturn("vm-uuid"); + when(userVm.getState()).thenReturn(VirtualMachine.State.Running); + when(strategy.userVmDao.findById(vmId)).thenReturn(userVm); + when(vmSnapshotHelper.pickRunningHost(vmId)).thenReturn(hostId); + when(strategy.vmInstanceDetailsDao.findDetail(vmId, ApiConstants.BootType.UEFI.toString())) + .thenReturn(new VMInstanceDetailVO(vmId, ApiConstants.BootType.UEFI.toString(), "SECURE", true)); + when(hostDetailsDao.findDetail(hostId, Host.HOST_UEFI_ENABLE)) + .thenReturn(new DetailVO(hostId, Host.HOST_UEFI_ENABLE, Boolean.TRUE.toString())); + when(hostDetailsDao.findDetail(hostId, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM)).thenReturn(null); + + try { + strategy.takeVMSnapshot(vmSnapshot); + } finally { + InOrder inOrder = inOrder(vmSnapshotHelper); + inOrder.verify(vmSnapshotHelper).vmSnapshotStateTransitTo(vmSnapshot, VMSnapshot.Event.CreateRequested); + inOrder.verify(vmSnapshotHelper).vmSnapshotStateTransitTo(vmSnapshot, VMSnapshot.Event.OperationFailed); + verify(agentMgr, never()).easySend(eq(hostId), any()); + } + } + + @Test(expected = CloudRuntimeException.class) + public void testDeleteVMSnapshotFailsWhenHostLacksNvramAwareCleanupCapabilityForSidecarSnapshot() { + long vmId = 10L; + long vmSnapshotId = 20L; + long hostId = 40L; + + UserVmVO userVm = mock(UserVmVO.class); + VMSnapshotVO vmSnapshot = mock(VMSnapshotVO.class); + VMSnapshotDetailsVO nvramDetail = new VMSnapshotDetailsVO(vmSnapshotId, "kvmFileBasedStorageSnapshotNvram", "nvram/42.fd", false); + + when(vmSnapshot.getVmId()).thenReturn(vmId); + when(vmSnapshot.getId()).thenReturn(vmSnapshotId); + when(vmSnapshot.getUuid()).thenReturn("vm-snapshot"); + when(userVm.getState()).thenReturn(VirtualMachine.State.Running); + when(strategy.userVmDao.findById(vmId)).thenReturn(userVm); + when(vmSnapshotHelper.pickRunningHost(vmId)).thenReturn(hostId); + when(vmSnapshotDetailsDao.findDetail(vmSnapshotId, "kvmFileBasedStorageSnapshotNvram")).thenReturn(nvramDetail); + when(hostDetailsDao.findDetail(hostId, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM)).thenReturn(null); + + strategy.deleteVMSnapshot(vmSnapshot); + } + + @Test(expected = CloudRuntimeException.class) + public void testTakeVmSnapshotInternalFailsWhenHostLacksUefiCapabilityForUefiVm() throws Exception { + long vmId = 10L; + long hostId = 40L; + + UserVmVO userVm = mock(UserVmVO.class); + VMSnapshotVO vmSnapshot = mock(VMSnapshotVO.class); + + when(vmSnapshot.getVmId()).thenReturn(vmId); + when(userVm.getId()).thenReturn(vmId); + when(userVm.getUuid()).thenReturn("vm-uuid"); + when(userVm.getState()).thenReturn(VirtualMachine.State.Running); + when(strategy.userVmDao.findById(vmId)).thenReturn(userVm); + when(vmSnapshotHelper.pickRunningHost(vmId)).thenReturn(hostId); + when(strategy.vmInstanceDetailsDao.findDetail(vmId, ApiConstants.BootType.UEFI.toString())) + .thenReturn(new VMInstanceDetailVO(vmId, ApiConstants.BootType.UEFI.toString(), "SECURE", true)); + when(hostDetailsDao.findDetail(hostId, Host.HOST_UEFI_ENABLE)).thenReturn(null); + when(hostDetailsDao.findDetail(hostId, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM)) + .thenReturn(new DetailVO(hostId, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM, Boolean.TRUE.toString())); + + strategy.takeVmSnapshotInternal(vmSnapshot, Collections.emptyMap()); + } + + @Test(expected = CloudRuntimeException.class) + public void testRevertVMSnapshotFailsWhenHostLacksNvramAwareSnapshotCapabilityForUefiVm() { + long vmId = 10L; + long hostId = 40L; + + UserVmVO userVm = mock(UserVmVO.class); + VMSnapshotVO vmSnapshot = mock(VMSnapshotVO.class); + + when(vmSnapshot.getVmId()).thenReturn(vmId); + when(userVm.getId()).thenReturn(vmId); + when(userVm.getUuid()).thenReturn("vm-uuid"); + when(userVm.getState()).thenReturn(VirtualMachine.State.Stopped); + when(strategy.userVmDao.findById(vmId)).thenReturn(userVm); + when(vmSnapshotHelper.pickRunningHost(vmId)).thenReturn(hostId); + mockCandidateHostScope(vmId, hostId); + when(strategy.vmInstanceDetailsDao.findDetail(vmId, ApiConstants.BootType.UEFI.toString())) + .thenReturn(new VMInstanceDetailVO(vmId, ApiConstants.BootType.UEFI.toString(), "SECURE", true)); + when(hostDetailsDao.findDetail(hostId, Host.HOST_UEFI_ENABLE)) + .thenReturn(new DetailVO(hostId, Host.HOST_UEFI_ENABLE, Boolean.TRUE.toString())); + when(hostDetailsDao.findDetail(hostId, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM)).thenReturn(null); + + strategy.revertVMSnapshot(vmSnapshot); + } + + @Test(expected = CloudRuntimeException.class) + public void testRevertVMSnapshotFailsWhenHostLacksUefiCapabilityForUefiVm() { + long vmId = 10L; + long hostId = 40L; + + UserVmVO userVm = mock(UserVmVO.class); + VMSnapshotVO vmSnapshot = mock(VMSnapshotVO.class); + + when(vmSnapshot.getVmId()).thenReturn(vmId); + when(userVm.getId()).thenReturn(vmId); + when(userVm.getUuid()).thenReturn("vm-uuid"); + when(userVm.getState()).thenReturn(VirtualMachine.State.Stopped); + when(strategy.userVmDao.findById(vmId)).thenReturn(userVm); + when(vmSnapshotHelper.pickRunningHost(vmId)).thenReturn(hostId); + mockCandidateHostScope(vmId, hostId); + when(strategy.vmInstanceDetailsDao.findDetail(vmId, ApiConstants.BootType.UEFI.toString())) + .thenReturn(new VMInstanceDetailVO(vmId, ApiConstants.BootType.UEFI.toString(), "SECURE", true)); + when(hostDetailsDao.findDetail(hostId, Host.HOST_UEFI_ENABLE)).thenReturn(null); + when(hostDetailsDao.findDetail(hostId, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM)) + .thenReturn(new DetailVO(hostId, Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM, Boolean.TRUE.toString())); + + strategy.revertVMSnapshot(vmSnapshot); + } + + @Test + public void testNotifyGuestRecoveryIssueIfNeededSendsAlert() { + CreateDiskOnlyVmSnapshotAnswer answer = mock(CreateDiskOnlyVmSnapshotAnswer.class); + UserVm userVm = mock(UserVm.class); + VMSnapshotVO vmSnapshot = mock(VMSnapshotVO.class); + + when(answer.getDetails()).thenReturn("VM could not be thawed"); + when(userVm.getUuid()).thenReturn("vm-uuid"); + when(userVm.getDataCenterId()).thenReturn(1L); + when(userVm.getPodIdToDeployIn()).thenReturn(2L); + when(vmSnapshot.getUuid()).thenReturn("snapshot-uuid"); + + strategy.notifyGuestRecoveryIssueIfNeeded(answer, userVm, vmSnapshot); + + verify(strategy.alertManager).sendAlert(eq(AlertManager.AlertType.ALERT_TYPE_VM_SNAPSHOT), eq(1L), eq(2L), anyString(), anyString()); + } + + private void mockCandidateHostScope(long vmId, long hostId) { + long poolId = 50L; + long clusterId = 60L; + long podId = 70L; + long dataCenterId = 80L; + + VolumeVO rootVolume = mock(VolumeVO.class); + StoragePoolVO storagePool = mock(StoragePoolVO.class); + HostVO host = mock(HostVO.class); + + when(rootVolume.getVolumeType()).thenReturn(Volume.Type.ROOT); + when(rootVolume.getPoolId()).thenReturn(poolId); + when(strategy.volumeDao.findByInstance(vmId)).thenReturn(List.of(rootVolume)); + when(storagePoolDao.findById(poolId)).thenReturn(storagePool); + when(storagePool.getClusterId()).thenReturn(clusterId); + when(storagePool.getPodId()).thenReturn(podId); + when(storagePool.getDataCenterId()).thenReturn(dataCenterId); + when(host.getId()).thenReturn(hostId); + when(hostDao.listAllUpAndEnabledNonHAHosts(Host.Type.Routing, clusterId, podId, dataCenterId, null)).thenReturn(List.of(host)); + } +} diff --git a/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java b/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java index 061d18dc3769..c80267a4d36e 100644 --- a/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java +++ b/engine/storage/src/main/java/org/apache/cloudstack/storage/endpoint/DefaultEndPointSelector.java @@ -25,7 +25,6 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; import java.util.List; import javax.inject.Inject; @@ -198,12 +197,7 @@ private void moveDedicatedHostsToLowerPriority(StringBuilder sbuilder, List, Integer> hostIds = dedicatedResourceDao.searchDedicatedHosts(null, null, account.getId(), null, null); List accountDedicatedHosts = hostIds.first(); for (DedicatedResourceVO accountDedicatedResource: accountDedicatedHosts){ - Iterator dedicatedHostsIterator = dedicatedHosts.iterator(); - while (dedicatedHostsIterator.hasNext()) { - if (dedicatedHostsIterator.next() == accountDedicatedResource.getHostId()) { - dedicatedHostsIterator.remove(); - } - } + dedicatedHosts.removeIf(hostId -> hostId.equals(accountDedicatedResource.getHostId())); } } } diff --git a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java index 426d157b13a6..58807bdc6a65 100644 --- a/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java +++ b/engine/storage/volume/src/main/java/org/apache/cloudstack/storage/volume/VolumeServiceImpl.java @@ -329,7 +329,7 @@ protected Void createVolumeCallback(AsyncCallbackDispatcher subGroup = key.subGroup(); ConfigurationSubGroupVO subGroupVO = _configSubGroupDao.findByNameAndGroup(subGroup.first(), groupId); if (subGroupVO == null) { - subGroupVO = new ConfigurationSubGroupVO(); + subGroupVO = new ConfigurationSubGroupVO(subGroup.first(), null, subGroup.second()); + subGroupVO.setGroupId(groupId); subGroupVO = _configSubGroupDao.persist(subGroupVO); } subGroupId = subGroupVO.getId(); diff --git a/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java b/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java index f4012b43fa8d..97f9b1765a7b 100644 --- a/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java +++ b/framework/config/src/test/java/org/apache/cloudstack/framework/config/impl/ConfigDepotImplTest.java @@ -19,6 +19,7 @@ package org.apache.cloudstack.framework.config.impl; import java.util.Collections; +import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -31,6 +32,7 @@ import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; @@ -47,9 +49,32 @@ public class ConfigDepotImplTest { @Mock ConfigurationSubGroupDao configSubGroupDao; + @Mock + ConfigurationSubGroupDao _configSubGroupDao; + @InjectMocks private ConfigDepotImpl configDepotImpl = new ConfigDepotImpl(); + @Test + public void createConfigObjectPersistsSubGroupWithNameAndGroupId() { + ConfigKey key = Mockito.mock(ConfigKey.class); + Mockito.when(key.group()).thenReturn(null); + Mockito.when(key.subGroup()).thenReturn(new Pair<>("ConsoleProxy VM", 5L)); + Mockito.when(key.key()).thenReturn("consoleproxy.capacity.standby"); + Mockito.when(_configSubGroupDao.findByNameAndGroup("ConsoleProxy VM", 1L)).thenReturn(null); + Mockito.when(_configSubGroupDao.persist(Mockito.any(ConfigurationSubGroupVO.class))) + .thenAnswer(invocation -> invocation.getArgument(0)); + Mockito.when(_configDao.findById("consoleproxy.capacity.standby")).thenReturn(Mockito.mock(ConfigurationVO.class)); + + ArgumentCaptor captor = ArgumentCaptor.forClass(ConfigurationSubGroupVO.class); + ReflectionTestUtils.invokeMethod(configDepotImpl, "createOrupdateConfigObject", + new Date(), "components", key, "someValue"); + + Mockito.verify(_configSubGroupDao).persist(captor.capture()); + Assert.assertEquals("ConsoleProxy VM", captor.getValue().getName()); + Assert.assertEquals(Long.valueOf(1L), captor.getValue().getGroupId()); + } + @Test public void createEmptyScopeLevelMappingsTest() { configDepotImpl.createEmptyScopeLevelMappings(); diff --git a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java index dcd863465d1b..8fc10cf37ab2 100644 --- a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java +++ b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java @@ -34,6 +34,9 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; +import java.sql.Time; +import java.sql.Timestamp; +import java.sql.Types; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; @@ -67,7 +70,6 @@ import org.apache.commons.lang3.exception.ExceptionUtils; import com.amazonaws.util.CollectionUtils; -import com.cloud.utils.DateUtil; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; import com.cloud.utils.Ternary; @@ -129,6 +131,28 @@ public abstract class GenericDaoBase extends Compone protected final static TimeZone s_gmtTimeZone = TimeZone.getTimeZone("GMT"); + /** + * Returns a fresh GMT {@link Calendar} for a single JDBC get/set timestamp call. Calendar is + * mutable and JDBC drivers may mutate the instance passed to them, so a new one is used per call + * rather than sharing a single instance across concurrent DAO operations. + */ + protected static Calendar gmtCalendar() { + return Calendar.getInstance(s_gmtTimeZone); + } + + /** + * Returns the SQL type ({@link Types}) matching the temporal flag of the given attribute, so a + * null date/time/timestamp column is bound with the correct type instead of always TIMESTAMP. + */ + protected static int temporalSqlType(Attribute attr) { + if (attr.is(Attribute.Flag.Date)) { + return Types.DATE; + } else if (attr.is(Attribute.Flag.Time)) { + return Types.TIME; + } + return Types.TIMESTAMP; + } + protected final static Map, GenericDao> s_daoMaps = new ConcurrentHashMap, GenericDao>(71); private final ConversionSupport _conversionSupport; @@ -598,20 +622,16 @@ protected void setField(Object entity, Field field, ResultSet rs, int index) thr field.set(entity, rs.getInt(index)); } } else if (type == Date.class) { - final Object data = rs.getDate(index); - if (data == null) { - field.set(entity, null); - return; - } - field.set(entity, DateUtil.parseDateString(s_gmtTimeZone, rs.getString(index))); + final Timestamp ts = rs.getTimestamp(index, gmtCalendar()); + field.set(entity, ts == null ? null : new Date(ts.getTime())); } else if (type == Calendar.class) { - final Object data = rs.getDate(index); + final Timestamp data = rs.getTimestamp(index, gmtCalendar()); if (data == null) { field.set(entity, null); return; } - final Calendar cal = Calendar.getInstance(); - cal.setTime(DateUtil.parseDateString(s_gmtTimeZone, rs.getString(index))); + final Calendar cal = Calendar.getInstance(s_gmtTimeZone); + cal.setTime(data); field.set(entity, cal); } else if (type == boolean.class) { field.setBoolean(entity, rs.getBoolean(index)); @@ -732,11 +752,11 @@ protected static M getObject(Class type, ResultSet rs, int index) throws return (M) (Long) rs.getLong(index); } } else if (type == Date.class) { - final Object data = rs.getDate(index); - if (data == null) { + final Timestamp ts = rs.getTimestamp(index, gmtCalendar()); + if (ts == null) { return null; } else { - return (M)DateUtil.parseDateString(s_gmtTimeZone, rs.getString(index)); + return (M) new Date(ts.getTime()); } } else if (type == short.class) { return (M) (Short) rs.getShort(index); @@ -779,12 +799,12 @@ protected static M getObject(Class type, ResultSet rs, int index) throws return (M) (Byte) rs.getByte(index); } } else if (type == Calendar.class) { - final Object data = rs.getDate(index); + final Timestamp data = rs.getTimestamp(index, gmtCalendar()); if (data == null) { return null; } else { - final Calendar cal = Calendar.getInstance(); - cal.setTime(DateUtil.parseDateString(s_gmtTimeZone, rs.getString(index))); + final Calendar cal = Calendar.getInstance(s_gmtTimeZone); + cal.setTime(data); return (M)cal; } } else if (type == byte[].class) { @@ -929,7 +949,7 @@ public Class getEntityBeanType() { protected T findOneIncludingRemovedBy(final SearchCriteria sc) { Filter filter = new Filter(1, true); List results = searchIncludingRemoved(sc, filter, null, false); - assert results.size() <= 1 : "Didn't the limiting worked?"; + assert results.size() <= 1 : "Didn't the limiting work?"; return results.size() == 0 ? null : results.get(0); } @@ -949,6 +969,15 @@ public T findOneBy(SearchCriteria sc, final Filter filter) { return results.isEmpty() ? null : results.get(0); } + @DB() + protected T findLastOneBy(SearchCriteria sc) { + sc = checkAndSetRemovedIsNull(sc); + Filter filter = new Filter(_entityBeanType, "id", Boolean.FALSE, 0L, 1L); + List results = searchIncludingRemoved(sc, filter, null, false); + assert results.size() <= 1 : "Didn't the limiting work?"; + return results.size() == 0 ? null : results.get(0); + } + @DB() public List listBy(SearchCriteria sc, final Filter filter) { sc = checkAndSetRemovedIsNull(sc); @@ -1707,7 +1736,12 @@ protected void insertElementCollection(T entity, Attribute idAttribute, ID id, M while (en.hasMoreElements()) { pstmt = txn.prepareAutoCloseStatement(ec.insertSql); if (ec.targetClass == Date.class) { - pstmt.setString(1, DateUtil.getDateDisplayString(s_gmtTimeZone, (Date)en.nextElement())); + Date d = (Date) en.nextElement(); + if (d == null) { + pstmt.setNull(1, Types.TIMESTAMP); + } else { + pstmt.setTimestamp(1, new Timestamp(d.getTime()), gmtCalendar()); + } } else { pstmt.setObject(1, en.nextElement()); } @@ -1811,28 +1845,28 @@ protected void prepareAttribute(final int j, final PreparedStatement pstmt, fina } else if (attr.field.getType() == Date.class) { final Date date = (Date)value; if (date == null || date.equals(DATE_TO_NULL)) { - pstmt.setObject(j, null); + pstmt.setNull(j, temporalSqlType(attr)); return; } if (attr.is(Attribute.Flag.Date)) { - pstmt.setString(j, DateUtil.getDateDisplayString(s_gmtTimeZone, date)); + pstmt.setDate(j, new java.sql.Date(date.getTime()), gmtCalendar()); } else if (attr.is(Attribute.Flag.TimeStamp)) { - pstmt.setString(j, DateUtil.getDateDisplayString(s_gmtTimeZone, date)); + pstmt.setTimestamp(j, new Timestamp(date.getTime()), gmtCalendar()); } else if (attr.is(Attribute.Flag.Time)) { - pstmt.setString(j, DateUtil.getDateDisplayString(s_gmtTimeZone, date)); + pstmt.setTime(j, new java.sql.Time(date.getTime()), gmtCalendar()); } } else if (attr.field.getType() == Calendar.class) { final Calendar cal = (Calendar)value; if (cal == null) { - pstmt.setObject(j, null); + pstmt.setNull(j, temporalSqlType(attr)); return; } if (attr.is(Attribute.Flag.Date)) { - pstmt.setString(j, DateUtil.getDateDisplayString(s_gmtTimeZone, cal.getTime())); + pstmt.setDate(j, new java.sql.Date(cal.getTimeInMillis()), gmtCalendar()); } else if (attr.is(Attribute.Flag.TimeStamp)) { - pstmt.setString(j, DateUtil.getDateDisplayString(s_gmtTimeZone, cal.getTime())); + pstmt.setTimestamp(j, new Timestamp(cal.getTimeInMillis()), gmtCalendar()); } else if (attr.is(Attribute.Flag.Time)) { - pstmt.setString(j, DateUtil.getDateDisplayString(s_gmtTimeZone, cal.getTime())); + pstmt.setTime(j, new Time(cal.getTimeInMillis()), gmtCalendar()); } } else if (attr.field.getType().isEnum()) { final Enumerated enumerated = attr.field.getAnnotation(Enumerated.class); @@ -1966,7 +2000,8 @@ protected void loadCollection(T entity, Attribute attr) { } } else if (ec.targetClass == Date.class) { while (rs.next()) { - lst.add(DateUtil.parseDateString(s_gmtTimeZone, rs.getString(1))); + final Timestamp ts = rs.getTimestamp(1, gmtCalendar()); + lst.add(ts == null ? null : new Date(ts.getTime())); } } else if (ec.targetClass == Boolean.class) { while (rs.next()) { @@ -2072,8 +2107,8 @@ protected void setField(final Object entity, final ResultSet rs, ResultSetMetaDa } } if(attr == null) { - logger.warn(String.format("Failed to find attribute in the entity %s to map column %s.%s (%s)", - ClassUtils.getUserClass(entity).getSimpleName(), tableName, columnName)); + logger.warn("Failed to find attribute in the entity {} to map column {}.{}", + ClassUtils.getUserClass(entity).getSimpleName(), tableName, columnName); } else { setField(entity, attr.field, rs, index); } diff --git a/framework/db/src/test/java/com/cloud/utils/db/GenericDaoBaseTest.java b/framework/db/src/test/java/com/cloud/utils/db/GenericDaoBaseTest.java index ebf514f532f7..41a7be4ab867 100644 --- a/framework/db/src/test/java/com/cloud/utils/db/GenericDaoBaseTest.java +++ b/framework/db/src/test/java/com/cloud/utils/db/GenericDaoBaseTest.java @@ -18,9 +18,14 @@ import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; import java.util.ArrayList; +import java.util.Calendar; import java.util.Collection; +import java.util.Date; import java.util.List; +import java.util.TimeZone; import org.junit.Assert; import org.junit.Before; @@ -331,4 +336,74 @@ public void testLockOneRandomRowReturnsFirstElement() { Assert.assertNotNull(result); Assert.assertEquals(expectedResult, result); } + + @Test + public void gmtCalendarUsesGmtTimeZone() { + Calendar calendar = GenericDaoBase.gmtCalendar(); + + Assert.assertEquals(TimeZone.getTimeZone("GMT"), calendar.getTimeZone()); + } + + @Test + public void gmtCalendarReturnsFreshInstancePerCall() { + Assert.assertNotSame(GenericDaoBase.gmtCalendar(), GenericDaoBase.gmtCalendar()); + } + + @Test + public void temporalSqlTypeDate() { + Attribute attr = new Attribute("table", "column"); + attr.flags = Attribute.Flag.Date.setTrue(attr.flags); + + Assert.assertEquals(Types.DATE, GenericDaoBase.temporalSqlType(attr)); + } + + @Test + public void temporalSqlTypeTime() { + Attribute attr = new Attribute("table", "column"); + attr.flags = Attribute.Flag.Time.setTrue(attr.flags); + + Assert.assertEquals(Types.TIME, GenericDaoBase.temporalSqlType(attr)); + } + + @Test + public void temporalSqlTypeDefaultsToTimestamp() { + Attribute attr = new Attribute("table", "column"); + attr.flags = Attribute.Flag.TimeStamp.setTrue(attr.flags); + + Assert.assertEquals(Types.TIMESTAMP, GenericDaoBase.temporalSqlType(attr)); + } + + @Test + public void getObjectDateReadsViaGmtTimestamp() throws SQLException { + Timestamp ts = new Timestamp(1_700_000_000_000L); + Mockito.when(resultSet.getTimestamp(Mockito.eq(2), Mockito.any(Calendar.class))).thenReturn(ts); + + Date result = GenericDaoBase.getObject(Date.class, resultSet, 2); + + Assert.assertEquals(ts.getTime(), result.getTime()); + } + + @Test + public void getObjectDateNullTimestampReturnsNull() throws SQLException { + Mockito.when(resultSet.getTimestamp(Mockito.eq(3), Mockito.any(Calendar.class))).thenReturn(null); + + Assert.assertNull(GenericDaoBase.getObject(Date.class, resultSet, 3)); + } + + @Test + public void getObjectCalendarReadsViaGmtTimestamp() throws SQLException { + Timestamp ts = new Timestamp(1_700_000_000_000L); + Mockito.when(resultSet.getTimestamp(Mockito.eq(4), Mockito.any(Calendar.class))).thenReturn(ts); + + Calendar result = GenericDaoBase.getObject(Calendar.class, resultSet, 4); + + Assert.assertEquals(ts.getTime(), result.getTimeInMillis()); + } + + @Test + public void getObjectCalendarNullTimestampReturnsNull() throws SQLException { + Mockito.when(resultSet.getTimestamp(Mockito.eq(5), Mockito.any(Calendar.class))).thenReturn(null); + + Assert.assertNull(GenericDaoBase.getObject(Calendar.class, resultSet, 5)); + } } diff --git a/framework/ipc/src/main/java/org/apache/cloudstack/framework/serializer/OnwireClassRegistry.java b/framework/ipc/src/main/java/org/apache/cloudstack/framework/serializer/OnwireClassRegistry.java index 3cc643956e93..93f5e0f9b5c0 100644 --- a/framework/ipc/src/main/java/org/apache/cloudstack/framework/serializer/OnwireClassRegistry.java +++ b/framework/ipc/src/main/java/org/apache/cloudstack/framework/serializer/OnwireClassRegistry.java @@ -55,7 +55,7 @@ public OnwireClassRegistry(String packageName) { } public OnwireClassRegistry(List packages) { - packages.addAll(packages); + this.packages.addAll(packages); } public List getPackages() { diff --git a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index 79075e5c4b64..1572adc50085 100644 --- a/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/main/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -125,6 +125,10 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, Integer.class, "vm.job.lock.timeout", "1800", "Time in seconds to wait in acquiring lock to submit a vm worker job", false); private static final ConfigKey HidePassword = new ConfigKey("Advanced", Boolean.class, "log.hide.password", "true", "If set to true, the password is hidden", true, ConfigKey.Scope.Global); + public static final ConfigKey ApiJobPoolSize = new ConfigKey<>("Advanced", Integer.class, "api.job.pool.size", "50", + "Minimum size of the API job executor thread pool. Actual size is the max of this value and db.cloud.maxActive / 2.", false, ConfigKey.Scope.Global); + public static final ConfigKey WorkJobPoolSize = new ConfigKey<>("Advanced", Integer.class, "work.job.pool.size", "50", + "Minimum size of the Worker job executor thread pool. Actual size is the max of this value and db.cloud.maxActive * 2 / 3.", false, ConfigKey.Scope.Global); private static final int ACQUIRE_GLOBAL_LOCK_TIMEOUT_FOR_COOPERATION = 3; // 3 seconds @@ -198,7 +202,7 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[] {JobExpireMinutes, JobCancelThresholdMinutes, VmJobLockTimeout, HidePassword}; + return new ConfigKey[] {JobExpireMinutes, JobCancelThresholdMinutes, VmJobLockTimeout, HidePassword, ApiJobPoolSize, WorkJobPoolSize}; } @Override @@ -995,6 +999,9 @@ public void reallyRun() { } logger.trace("End cleanup expired async-jobs"); + + cleanupNetworksStuckInImplementing(); + } catch (Throwable e) { logger.error("Unexpected exception when trying to execute queue item, ", e); } @@ -1002,6 +1009,15 @@ public void reallyRun() { }; } + private void cleanupNetworksStuckInImplementing() { + // Cleanup orphaned networks stuck in Implementing state without async jobs + try { + cleanupOrphanedNetworks(); + } catch (Throwable e) { + logger.error("Unexpected exception when trying to cleanup orphaned networks", e); + } + } + @DB protected void expungeAsyncJob(final AsyncJobVO job) { Transaction.execute(new TransactionCallbackNoReturn() { @@ -1088,13 +1104,18 @@ public boolean configure(String name, Map params) throws Configu final Properties dbProps = DbProperties.getDbProperties(); final int cloudMaxActive = Integer.parseInt(dbProps.getProperty("db.cloud.maxActive")); - int apiPoolSize = cloudMaxActive / 2; - int workPoolSize = (cloudMaxActive * 2) / 3; + int defaultApiPoolSize = cloudMaxActive / 2; + int defaultWorkPoolSize = (cloudMaxActive * 2) / 3; + + int apiPoolSize = Math.max(ApiJobPoolSize.value(), defaultApiPoolSize); + int workPoolSize = Math.max(WorkJobPoolSize.value(), defaultWorkPoolSize); - logger.info("Start AsyncJobManager API executor thread pool in size " + apiPoolSize); + logger.info("Start AsyncJobManager API executor thread pool in size " + apiPoolSize + + " (configured=" + ApiJobPoolSize.value() + ", db.derived=" + defaultApiPoolSize + ", db.cloud.maxActive=" + cloudMaxActive + ")"); _apiJobExecutor = Executors.newFixedThreadPool(apiPoolSize, new NamedThreadFactory(AsyncJobManager.API_JOB_POOL_THREAD_PREFIX)); - logger.info("Start AsyncJobManager Work executor thread pool in size " + workPoolSize); + logger.info("Start AsyncJobManager Work executor thread pool in size " + workPoolSize + + " (configured=" + WorkJobPoolSize.value() + ", db.derived=" + defaultWorkPoolSize + ", db.cloud.maxActive=" + cloudMaxActive + ")"); _workerJobExecutor = Executors.newFixedThreadPool(workPoolSize, new NamedThreadFactory(AsyncJobManager.WORK_JOB_POOL_THREAD_PREFIX)); } catch (final Exception e) { throw new ConfigurationException("Unable to load db.properties to configure AsyncJobManagerImpl"); @@ -1339,6 +1360,74 @@ private void cleanupFailedSnapshotsCreatedWithDefaultStrategy(final long msid) { } } + /** + * Cleanup networks that are stuck in Implementing state without associated async jobs. + * This only processes networks that have been stuck for longer than the job expiration threshold. + */ + private void cleanupOrphanedNetworks() { + try { + SearchCriteria sc = networkDao.createSearchCriteria(); + sc.addAnd("state", SearchCriteria.Op.EQ, Network.State.Implementing); + sc.addAnd("removed", SearchCriteria.Op.NULL); + List implementingNetworks = networkDao.search(sc, null); + + if (implementingNetworks == null || implementingNetworks.isEmpty()) { + return; + } + + logger.debug("Found {} networks in Implementing state, checking for orphaned networks", implementingNetworks.size()); + + final long expireMinutes = JobExpireMinutes.value(); + final Date cutoffTime = new Date(System.currentTimeMillis() - (expireMinutes * 60 * 1000)); + + for (NetworkVO network : implementingNetworks) { + if (network.getCreated().after(cutoffTime)) { + logger.trace("Network {} in Implementing state is only {} minutes old (threshold: {} minutes), skipping cleanup", + network.getId(), + (System.currentTimeMillis() - network.getCreated().getTime()) / 60000, + expireMinutes); + continue; + } + + List jobs = _jobDao.findInstancePendingAsyncJobs("Network", network.getAccountId()); + boolean hasActiveJob = false; + for (AsyncJobVO job : jobs) { + if (job.getInstanceId() != null && job.getInstanceId().equals(network.getId())) { + hasActiveJob = true; + break; + } + } + + if (hasActiveJob) { + logger.debug("Network {} in Implementing state has active async job, skipping cleanup", network.getId()); + continue; + } + + logger.warn("Found orphaned network {} in Implementing state without async job. " + + "Network created: {}, age: {} minutes, expiration threshold: {} minutes. Transitioning to Shutdown state.", + network.getId(), network.getCreated(), + (System.currentTimeMillis() - network.getCreated().getTime()) / 60000, + expireMinutes); + updateNetworkState(network); + + } + } catch (Exception e) { + logger.error("Error while cleaning up orphaned networks", e); + } + } + + private void updateNetworkState(NetworkVO network) { + try { + networkOrchestrationService.stateTransitTo(network, Network.Event.OperationFailed); + logger.info("Successfully transitioned orphaned network {} to Shutdown state using state machine", network.getId()); + } catch (final NoTransitionException e) { + logger.debug("State transition failed for orphaned network {}, forcing state update", network.getId()); + network.setState(Network.State.Shutdown); + networkDao.update(network.getId(), network); + logger.info("Successfully forced orphaned network {} to Shutdown state", network.getId()); + } + } + @Override public void onManagementNodeJoined(List nodeList, long selfNodeId) { } diff --git a/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImplTest.java b/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImplTest.java index 0be5dbc01cbd..08d4c2afd431 100644 --- a/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImplTest.java +++ b/framework/jobs/src/test/java/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImplTest.java @@ -17,19 +17,30 @@ package org.apache.cloudstack.framework.jobs.impl; +import java.lang.reflect.Field; +import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadPoolExecutor; + import com.cloud.network.Network; import com.cloud.network.dao.NetworkDao; import com.cloud.network.dao.NetworkVO; import com.cloud.storage.Volume; +import com.cloud.utils.db.DbProperties; import com.cloud.utils.fsm.NoTransitionException; import com.cloud.vm.VMInstanceVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.VirtualMachineManager; import com.cloud.vm.dao.VMInstanceDao; + import org.apache.cloudstack.api.ApiCommandResourceType; import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeDataFactory; import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; @@ -42,6 +53,12 @@ @RunWith(MockitoJUnitRunner.class) public class AsyncJobManagerImplTest { + + // db.cloud.maxActive value preloaded into DbProperties for every test in this class. + // Chosen so the db-derived defaults (4000/2 = 2000 for api, 4000*2/3 = 2666 for worker) + // are non-trivial and easy to reason about vs. the configured minimums under test. + private static final int DB_CLOUD_MAX_ACTIVE = 4000; + @Spy @InjectMocks AsyncJobManagerImpl asyncJobManager; @@ -56,6 +73,30 @@ public class AsyncJobManagerImplTest { @Mock NetworkOrchestrationService networkOrchestrationService; + @Before + public void preloadDbProperties() throws Exception { + Properties props = new Properties(); + props.setProperty("db.cloud.maxActive", String.valueOf(DB_CLOUD_MAX_ACTIVE)); + // Bypass DbProperties' internal "loaded" guard so configure() reads our + // properties instead of trying to load a real db.properties file. + setDbPropertiesStatics(props, true); + } + + @After + public void tearDown() throws Exception { + try { + shutdownIfPresent("_apiJobExecutor"); + shutdownIfPresent("_workerJobExecutor"); + } finally { + // Reset static state even if executor shutdown throws, so we do not + // pollute later test classes with our stubbed ConfigKey values or + // preloaded DbProperties. + resetConfigValue(AsyncJobManagerImpl.ApiJobPoolSize); + resetConfigValue(AsyncJobManagerImpl.WorkJobPoolSize); + setDbPropertiesStatics(new Properties(), false); + } + } + @Test public void testCleanupVolumeResource() { AsyncJobVO job = new AsyncJobVO(); @@ -93,4 +134,109 @@ public void testCleanupNetworkResource() throws NoTransitionException { Mockito.verify(networkOrchestrationService, Mockito.times(1)).stateTransitTo(networkVO, Network.Event.OperationFailed); } + + @Test + public void configureApiPoolUsesConfiguredValueWhenLargerThanDbDerived() throws Exception { + overrideConfigValue(AsyncJobManagerImpl.ApiJobPoolSize, 3000); + overrideConfigValue(AsyncJobManagerImpl.WorkJobPoolSize, 50); + invokeConfigureAndSwallowPostSetupNpe(); + // apiPoolSize = max(3000, 4000/2) = 3000 (configured wins) + Assert.assertEquals(3000, apiPoolCoreSize()); + } + + @Test + public void configureApiPoolUsesDbDerivedWhenLargerThanConfigured() throws Exception { + overrideConfigValue(AsyncJobManagerImpl.ApiJobPoolSize, 50); + overrideConfigValue(AsyncJobManagerImpl.WorkJobPoolSize, 50); + invokeConfigureAndSwallowPostSetupNpe(); + // apiPoolSize = max(50, 4000/2) = 2000 (db-derived wins) + Assert.assertEquals(DB_CLOUD_MAX_ACTIVE / 2, apiPoolCoreSize()); + } + + @Test + public void configureWorkerPoolUsesTwoThirdsFormula() throws Exception { + overrideConfigValue(AsyncJobManagerImpl.ApiJobPoolSize, 50); + overrideConfigValue(AsyncJobManagerImpl.WorkJobPoolSize, 50); + invokeConfigureAndSwallowPostSetupNpe(); + // workPoolSize = max(50, 4000*2/3) = 2666 (db-derived wins) + Assert.assertEquals((DB_CLOUD_MAX_ACTIVE * 2) / 3, workerPoolCoreSize()); + } + + @Test + public void configureWorkerPoolUsesConfiguredValueWhenLargerThanDbDerived() throws Exception { + overrideConfigValue(AsyncJobManagerImpl.ApiJobPoolSize, 50); + overrideConfigValue(AsyncJobManagerImpl.WorkJobPoolSize, 5000); + invokeConfigureAndSwallowPostSetupNpe(); + // workPoolSize = max(5000, 4000*2/3) = 5000 (configured wins) + Assert.assertEquals(5000, workerPoolCoreSize()); + } + + /** + * Invoke configure() and swallow the NPE it throws when wiring SearchBuilders + * on the un-mocked DAOs after the pool-sizing block. The executors are created + * inside configure()'s try/catch (before the DAO wiring), so pool sizes can + * still be inspected after the failure. + */ + private void invokeConfigureAndSwallowPostSetupNpe() { + try { + asyncJobManager.configure(null, null); + } catch (Exception ignored) { + // expected: DAO wiring after pool-sizing NPEs because we intentionally + // did not mock the DAOs (they are not part of what we are testing). + } + } + + private int apiPoolCoreSize() throws Exception { + Object executor = readField("_apiJobExecutor"); + Assert.assertNotNull("configure() did not create _apiJobExecutor - has DAO wiring been moved before pool sizing?", executor); + return ((ThreadPoolExecutor) executor).getCorePoolSize(); + } + + private int workerPoolCoreSize() throws Exception { + Object executor = readField("_workerJobExecutor"); + Assert.assertNotNull("configure() did not create _workerJobExecutor - has DAO wiring been moved before pool sizing?", executor); + return ((ThreadPoolExecutor) executor).getCorePoolSize(); + } + + private Object readField(String name) throws Exception { + Field f = AsyncJobManagerImpl.class.getDeclaredField(name); + f.setAccessible(true); + return f.get(asyncJobManager); + } + + private void shutdownIfPresent(String executorFieldName) { + try { + Object executor = readField(executorFieldName); + if (executor instanceof ExecutorService) { + ((ExecutorService) executor).shutdownNow(); + } + } catch (Exception ignored) { + // never populated by this test - nothing to clean up + } + } + + private void overrideConfigValue(final ConfigKey configKey, final Object value) { + try { + Field f = ConfigKey.class.getDeclaredField("_value"); + f.setAccessible(true); + f.set(configKey, value); + } catch (IllegalAccessException | NoSuchFieldException e) { + Assert.fail(e.getMessage()); + } + } + + private void resetConfigValue(final ConfigKey configKey) throws Exception { + Field f = ConfigKey.class.getDeclaredField("_value"); + f.setAccessible(true); + f.set(configKey, null); + } + + private void setDbPropertiesStatics(Properties props, boolean loaded) throws Exception { + Field propsField = DbProperties.class.getDeclaredField("properties"); + Field loadedField = DbProperties.class.getDeclaredField("loaded"); + propsField.setAccessible(true); + loadedField.setAccessible(true); + propsField.set(null, props); + loadedField.set(null, loaded); + } } diff --git a/framework/spring/module/src/test/java/org/apache/cloudstack/spring/module/factory/ModuleBasedContextFactoryTest.java b/framework/spring/module/src/test/java/org/apache/cloudstack/spring/module/factory/ModuleBasedContextFactoryTest.java index 884665efed2d..b6f55f6b9e37 100644 --- a/framework/spring/module/src/test/java/org/apache/cloudstack/spring/module/factory/ModuleBasedContextFactoryTest.java +++ b/framework/spring/module/src/test/java/org/apache/cloudstack/spring/module/factory/ModuleBasedContextFactoryTest.java @@ -164,11 +164,12 @@ protected void testBeansInContext(ModuleDefinitionSet set, String name, int orde public static class InstantiationCounter { public static Integer count = 0; + private static final Object countLock = new Object(); int myCount; public InstantiationCounter() { - synchronized (count) { + synchronized (countLock) { myCount = count + 1; count = myCount; } diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupChainKeys.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupChainKeys.java new file mode 100644 index 000000000000..511f0ccb7114 --- /dev/null +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupChainKeys.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.backup; + +/** + * Keys used by the NAS backup provider when storing incremental-chain metadata + * in the existing {@code backup_details} key/value table. Stored here (not on + * the {@code backups} table) so other providers do not need a schema change to + * support their own incremental implementations. + */ +public final class NASBackupChainKeys { + + /** UUID of the parent backup (full or previous incremental). Empty for full backups. */ + public static final String PARENT_BACKUP_ID = "nas.parent_backup_id"; + + /** QEMU dirty-bitmap name created by this backup, used as the {@code } reference for the next one. */ + public static final String BITMAP_NAME = "nas.bitmap_name"; + + /** Identifier shared by every backup in the same chain (the full anchors a chain; its incrementals inherit the id). */ + public static final String CHAIN_ID = "nas.chain_id"; + + /** Position within the chain: 0 for the full, 1 for the first incremental, and so on. */ + public static final String CHAIN_POSITION = "nas.chain_position"; + + /** + * In-memory chain-mode sentinels used by {@code ChainDecision.mode}. The persisted + * full-vs-incremental backup type lives on the {@code backup.type} column (set in + * {@code takeBackup}) — single source of truth. Not duplicated into backup_details. + */ + public static final String TYPE_FULL = "full"; + public static final String TYPE_INCREMENTAL = "incremental"; + // Feature disabled: behave exactly like the pre-incremental full-only backup — no QEMU + // bitmap/checkpoint is created and no chain metadata is persisted. Matches nasbackup.sh's + // "legacy-full" mode token (which sets make_checkpoint=0). + public static final String TYPE_LEGACY_FULL = "legacy-full"; + + /** + * VM-scoped detail (stored in {@code vm_instance_details}) holding the QEMU dirty-bitmap + * name that currently exists on the running VM and is therefore the only valid parent + * for the next incremental backup. Written by {@link #BITMAP_NAME} on each successful + * backup; cleared on restore (the restored disk image has no bitmap, so the next backup + * must be a fresh full). When the VM has no detail, {@code decideChain} forces full. + */ + public static final String VM_ACTIVE_CHECKPOINT_ID = "nas.active_checkpoint_id"; + + private NASBackupChainKeys() { + } +} diff --git a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java index fb1b78eb963b..60c9983f016e 100644 --- a/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java +++ b/plugins/backup/nas/src/main/java/org/apache/cloudstack/backup/NASBackupProvider.java @@ -17,8 +17,10 @@ package org.apache.cloudstack.backup; import com.cloud.agent.AgentManager; +import com.cloud.alert.AlertManager; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.OperationTimedoutException; +import com.cloud.configuration.Resource; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; @@ -37,9 +39,13 @@ import com.cloud.storage.dao.VolumeDao; import com.cloud.utils.Pair; import com.cloud.utils.component.AdapterBase; +import com.cloud.utils.db.GlobalLock; import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.user.ResourceLimitService; +import com.cloud.vm.VMInstanceDetailVO; import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; +import com.cloud.vm.dao.VMInstanceDetailsDao; import com.cloud.vm.snapshot.VMSnapshot; import com.cloud.vm.snapshot.VMSnapshotDetailsVO; import com.cloud.vm.snapshot.VMSnapshotVO; @@ -48,6 +54,7 @@ import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupDetailsDao; import org.apache.cloudstack.backup.dao.BackupRepositoryDao; import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; @@ -57,6 +64,7 @@ import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; @@ -78,6 +86,8 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Configurable { private static final Logger LOG = LogManager.getLogger(NASBackupProvider.class); + private static final int CHAIN_LOCK_TIMEOUT_SECONDS = 300; + ConfigKey NASBackupRestoreMountTimeout = new ConfigKey<>("Advanced", Integer.class, "nas.backup.restore.mount.timeout", "30", @@ -85,6 +95,38 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co true, BackupFrameworkEnabled.key()); + ConfigKey NASBackupIncrementalEnabled = new ConfigKey<>("Advanced", Boolean.class, + "nas.backup.incremental.enabled", + "false", + "Master switch for NAS incremental backups. Defaults to false so existing zones keep the " + + "legacy full-only behavior on upgrade; opt in per-zone when ready to use chains. " + + "When false, every NAS backup is taken as a full regardless of nas.backup.full.every. " + + "Toggling this is safe at any time: switching off forces the next backup to be a fresh " + + "full anchor (existing chains stay restorable), switching back on resumes incrementals " + + "on the next full + incremental cycle.", + true, + ConfigKey.Scope.Zone, + BackupFrameworkEnabled.key()); + + ConfigKey NASBackupFullEvery = new ConfigKey<>("Advanced", Integer.class, + "nas.backup.full.every", + "10", + "Take a full NAS backup every Nth backup; remaining backups in between are incremental. " + + "Counts backups, not days, so it works for hourly, daily, and ad-hoc schedules. " + + "Set to 1 to disable incrementals (every backup is full).", + true, + ConfigKey.Scope.Zone, + NASBackupIncrementalEnabled.key()); + + ConfigKey NASBackupQuiesceAgentTimeout = new ConfigKey<>("Advanced", Integer.class, + "nas.backup.quiesce.agent.timeout", + "30", + "Timeout in seconds for the guest agent filesystem freeze and thaw commands of a quiesced NAS backup. " + + "Set to 0 to use the libvirt default (5 seconds unless changed on the host).", + true, + ConfigKey.Scope.Zone, + BackupFrameworkEnabled.key()); + @Inject private BackupDao backupDao; @@ -106,6 +148,9 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Inject private VMInstanceDao vmInstanceDao; + @Inject + private VMInstanceDetailsDao vmInstanceDetailsDao; + @Inject private PrimaryDataStoreDao primaryDataStoreDao; @@ -115,6 +160,9 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Inject private AgentManager agentManager; + @Inject + private ResourceLimitService resourceLimitMgr; + @Inject private VMSnapshotDao vmSnapshotDao; @@ -130,6 +178,12 @@ public class NASBackupProvider extends AdapterBase implements BackupProvider, Co @Inject private DiskOfferingDao diskOfferingDao; + @Inject + private BackupDetailsDao backupDetailsDao; + + @Inject + private AlertManager alertManager; + private Long getClusterIdFromRootVolume(VirtualMachine vm) { VolumeVO rootVolume = volumeDao.getInstanceRootVolume(vm.getId()); StoragePoolVO rootDiskPool = primaryDataStoreDao.findById(rootVolume.getPoolId()); @@ -168,6 +222,334 @@ protected Host getVMHypervisorHost(VirtualMachine vm) { return resourceManager.findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, vm.getDataCenterId()); } + /** + * Returned by {@link #decideChain(VirtualMachine)} to describe the next backup's place in + * the chain: full vs incremental, the bitmap name to create, and (for incrementals) the + * parent bitmap and parent file path. + */ + static final class ChainDecision { + final String mode; // "full" or "incremental" + final String bitmapNew; + final String bitmapParent; // null for full + // Per-volume parent backup file paths, one per current VM volume in deviceId order. + // null/empty for full. Each volume needs its own parent file because backup files + // are named after each volume's own UUID (root..qcow2 / datadisk..qcow2). + final List parentPaths; + final String chainId; // chain identifier this backup belongs to + final int chainPosition; // 0 for full, N for the Nth incremental in the chain + + private ChainDecision(String mode, String bitmapNew, String bitmapParent, List parentPaths, + String chainId, int chainPosition) { + this.mode = mode; + this.bitmapNew = bitmapNew; + this.bitmapParent = bitmapParent; + this.parentPaths = parentPaths; + this.chainId = chainId; + this.chainPosition = chainPosition; + } + + static ChainDecision fullStart(String bitmapName) { + return new ChainDecision(NASBackupChainKeys.TYPE_FULL, bitmapName, null, null, + UUID.randomUUID().toString(), 0); + } + + /** + * Decision used when the incremental feature is disabled: a plain full backup that + * creates no bitmap and carries no chain identity, so nothing chain/checkpoint-related + * is sent to the agent or persisted. Keeps the feature-off path byte-for-byte legacy. + */ + static ChainDecision legacyFull() { + return new ChainDecision(NASBackupChainKeys.TYPE_LEGACY_FULL, null, null, null, null, 0); + } + + static ChainDecision incremental(String bitmapNew, String bitmapParent, List parentPaths, + String chainId, int chainPosition) { + return new ChainDecision(NASBackupChainKeys.TYPE_INCREMENTAL, bitmapNew, bitmapParent, + parentPaths, chainId, chainPosition); + } + + boolean isIncremental() { + return NASBackupChainKeys.TYPE_INCREMENTAL.equals(mode); + } + + boolean isLegacyFull() { + return NASBackupChainKeys.TYPE_LEGACY_FULL.equals(mode); + } + } + + /** + * Decides whether the next backup for {@code vm} should be a fresh full or an incremental + * appended to the existing chain. Stopped VMs are always full (libvirt {@code backup-begin} + * requires a running QEMU process). The {@code nas.backup.full.every} ConfigKey controls + * how many backups (full + incrementals) form one chain before a new full is forced. + * + *

The decision is anchored on the VM's {@code nas.active_checkpoint_id} detail, which + * records the bitmap that currently exists on the running QEMU. After a restore that + * detail is cleared, so the next backup is automatically full — even though there may be + * a more recent "last backup taken" row in the database. The decision deliberately avoids + * relying on "last backup taken", because that row is misleading after a restore.

+ */ + protected ChainDecision decideChain(VirtualMachine vm) { + // Master switch — when the operator disables incrementals at the zone level the backup + // behaves exactly like the pre-incremental full-only path: no bitmap is generated and no + // chain/checkpoint metadata is created, sent to the agent, or persisted (legacy-full). + Boolean incrementalEnabled = NASBackupIncrementalEnabled.valueIn(vm.getDataCenterId()); + if (incrementalEnabled == null || !incrementalEnabled) { + return ChainDecision.legacyFull(); + } + + // Incremental backups rely on QEMU dirty bitmaps / libvirt checkpoints, which only exist + // on file-based qcow2 storage. Storage such as Ceph-RBD and Linstor cannot carry per-disk + // checkpoints, so a VM with any volume on such a pool must stay on the full-only (legacy) + // path — otherwise an incremental attempt would fail or regress those storages. + if (!allVolumesOnCheckpointCapableStorage(vm)) { + return ChainDecision.legacyFull(); + } + + final String newBitmap = "backup-" + System.currentTimeMillis() / 1000L; + + // Stopped VMs cannot do incrementals — script will also fall back, but we make the + // decision here so we register the right type up-front. + if (VirtualMachine.State.Stopped.equals(vm.getState())) { + return ChainDecision.fullStart(newBitmap); + } + + Integer fullEvery = NASBackupFullEvery.valueIn(vm.getDataCenterId()); + if (fullEvery == null || fullEvery <= 1) { + // Disabled or every-backup-is-full mode. + return ChainDecision.fullStart(newBitmap); + } + + // 1. If the VM has no active_checkpoint_id, there is no bitmap on the host to use as + // a parent. This is the case after restore (we clear it), after VM was just assigned + // to the offering, or on the very first backup. + String activeCheckpoint = readVmActiveCheckpoint(vm.getId()); + if (activeCheckpoint == null) { + return ChainDecision.fullStart(newBitmap); + } + + // 2. The most-recent BackedUp backup is the only safe parent — after restore the + // next backup is always a fresh full, so anything older has a rotated-out bitmap. + // If the latest backup's bitmap doesn't match the VM's active_checkpoint_id, the + // chain is broken: force a full. + Backup parent = findLatestBackedUpBackup(vm.getId()); + if (parent == null || !activeCheckpoint.equals(readDetail(parent, NASBackupChainKeys.BITMAP_NAME))) { + LOG.debug("VM {} latest backup does not match active_checkpoint_id={} — forcing full", + vm.getInstanceName(), activeCheckpoint); + return ChainDecision.fullStart(newBitmap); + } + + String parentChainId = readDetail(parent, NASBackupChainKeys.CHAIN_ID); + int parentChainPosition = chainPosition(parent); + if (parentChainId == null || parentChainPosition == Integer.MAX_VALUE) { + return ChainDecision.fullStart(newBitmap); + } + + // Force a fresh full when the chain has reached the configured length. + if (parentChainPosition + 1 >= fullEvery) { + return ChainDecision.fullStart(newBitmap); + } + + // The script needs the parent backup's on-NAS file path PER VOLUME so it can rebase + // each new qcow2 onto the matching parent. The paths are stored relative to the NAS + // mount root — the script resolves them inside its mount session. When alignment + // fails (volume count changed, etc.) compose returns null and we fall back to full + // so we don't risk corrupting the chain. + List parentPaths = composeParentBackupPaths(parent, vm.getId()); + if (parentPaths == null) { + LOG.debug("VM {} parent backup {} volume layout no longer matches current VM — forcing full", + vm.getInstanceName(), parent.getUuid()); + return ChainDecision.fullStart(newBitmap); + } + return ChainDecision.incremental(newBitmap, activeCheckpoint, parentPaths, + parentChainId, parentChainPosition + 1); + } + + /** + * Incremental backups require QEMU dirty bitmaps / libvirt checkpoints, which are only + * possible on file-based qcow2 storage. Returns {@code true} only when EVERY volume of the + * VM sits on HOST-scope local, {@code SharedMountPoint}, or {@code NetworkFilesystem} (NFS) + * storage. Ceph-RBD, Linstor, and any other pool that cannot carry a per-disk checkpoint + * make this return {@code false} so the caller falls back to the legacy full-only path. A + * volume whose pool can no longer be resolved is treated as incapable (safe default). + */ + protected boolean allVolumesOnCheckpointCapableStorage(VirtualMachine vm) { + List volumes = volumeDao.findByInstance(vm.getId()); + if (volumes == null) { + return true; + } + for (VolumeVO volume : volumes) { + StoragePoolVO pool = primaryDataStoreDao.findById(volume.getPoolId()); + if (pool == null) { + LOG.debug("VM {} volume {} has no resolvable storage pool — forcing legacy full", + vm.getInstanceName(), volume.getUuid()); + return false; + } + boolean checkpointCapable = ScopeType.HOST.equals(pool.getScope()) + || Storage.StoragePoolType.SharedMountPoint.equals(pool.getPoolType()) + || Storage.StoragePoolType.NetworkFilesystem.equals(pool.getPoolType()); + if (!checkpointCapable) { + LOG.debug("VM {} volume {} is on {} (scope {}) which cannot carry checkpoints — forcing legacy full", + vm.getInstanceName(), volume.getUuid(), pool.getPoolType(), pool.getScope()); + return false; + } + } + return true; + } + + /** + * Read the {@code nas.active_checkpoint_id} VM detail. Returns {@code null} when no detail + * exists (post-restore, first backup, or after explicit reset). + */ + private String readVmActiveCheckpoint(long vmId) { + VMInstanceDetailVO d = vmInstanceDetailsDao.findDetail(vmId, NASBackupChainKeys.VM_ACTIVE_CHECKPOINT_ID); + if (d == null) { + return null; + } + String v = d.getValue(); + return StringUtils.isBlank(v) ? null : v; + } + + /** + * Locate the most-recent {@code BackedUp} backup for {@code vmId}. The chain invariant + * guarantees the latest backup is the only valid incremental parent — after restore the + * next backup is always a fresh full, and {@link #decideChain} checks the bitmap matches. + */ + private Backup findLatestBackedUpBackup(long vmId) { + List history = backupDao.listByVmId(null, vmId); + if (CollectionUtils.isEmpty(history)) { + return null; + } + return history.stream() + .filter(b -> Backup.Status.BackedUp.equals(b.getStatus())) + .max(Comparator.comparing(Backup::getDate)) + .orElse(null); + } + + private String readDetail(Backup backup, String key) { + BackupDetailVO d = backupDetailsDao.findDetail(backup.getId(), key); + return d == null ? null : d.getValue(); + } + + /** + * Compose the on-NAS path of EVERY parent backup file (one per VM volume) in the same + * order the script will iterate the current VM's disks (deviceId asc). Relative to the + * NAS mount, matches the layout written by {@code nasbackup.sh}: + * first disk -> {@code /root..qcow2} + * others -> {@code /datadisk..qcow2} + * + * Returns {@code null} if the parent's stored volume count doesn't match the current VM's + * volume count. Volume attach/detach is blocked while a VM is assigned to a backup offering; + * if the offering was removed and re-assigned the active checkpoint is cleared in + * {@link #removeVMFromBackupOffering}, so this method doesn't need to revalidate volume + * identities — a count mismatch is the only way to reach this branch with a non-null + * active_checkpoint_id. + */ + private List composeParentBackupPaths(Backup parent, long vmId) { + // backupPath is stored as externalId by createBackupObject — e.g. + // "i-2-1234-VM/2026.04.27.13.45.00". + String dir = parent.getExternalId(); + if (StringUtils.isBlank(dir)) { + LOG.debug("Parent backup {} has no backup path recorded; cannot compose parent backup paths", parent.getUuid()); + return null; + } + + List parentVols = parent.getBackedUpVolumes(); + if (CollectionUtils.isEmpty(parentVols)) { + LOG.debug("Parent backup {} has no backed up volumes recorded; cannot compose parent backup paths", parent.getUuid()); + return null; + } + + List currentVols = volumeDao.findByInstance(vmId); + if (CollectionUtils.isEmpty(currentVols) || currentVols.size() != parentVols.size()) { + LOG.debug("VM id={} currently has {} volume(s) but parent backup {} recorded {}; cannot compose parent backup paths", + vmId, currentVols == null ? 0 : currentVols.size(), parent.getUuid(), parentVols.size()); + return null; + } + + // parentVols is in deviceId order at the time the parent was taken. The script names the + // per-disk files from the volume PATH basename (root..qcow2 / datadisk..qcow2, + // see nasbackup.sh: volUuid="${fullpath##*/}"). Use getPath(), NOT getUuid(): after a + // volume migration the uuid and the on-disk path diverge, and the backup file is named by + // path — a uuid-based parent path then fails to resolve for the incremental (test 13). + List paths = new ArrayList<>(parentVols.size()); + for (int i = 0; i < parentVols.size(); i++) { + String volPath = parentVols.get(i).getPath(); + String prefix = (i == 0) ? "root" : "datadisk"; + paths.add(dir + "/" + prefix + "." + volPath + ".qcow2"); + } + return paths; + } + + /** + * Persist chain metadata under backup_details. Stored here (not on the backups table) so + * other providers can implement their own chain semantics without schema changes. + */ + private void persistChainMetadata(Backup backup, ChainDecision decision, String bitmapFromAgent) { + // Only persist nas.bitmap_name when the agent confirmed the bitmap exists on the host. + // The agent wrapper sets bitmapFromAgent=null when nasbackup.sh exits + // EXIT_BITMAP_NOT_SEEDED (=22) — currently only the stopped-VM path where qemu-img + // bitmap --add failed on every source disk. Anchoring the next incremental on a + // bitmap that doesn't exist would force a non-recoverable failure, so we leave the + // detail empty and let the next backup start a fresh full chain. + if (bitmapFromAgent != null && !bitmapFromAgent.isEmpty()) { + backupDetailsDao.persist(new BackupDetailVO(backup.getId(), NASBackupChainKeys.BITMAP_NAME, bitmapFromAgent, true)); + } + backupDetailsDao.persist(new BackupDetailVO(backup.getId(), NASBackupChainKeys.CHAIN_ID, decision.chainId, true)); + backupDetailsDao.persist(new BackupDetailVO(backup.getId(), NASBackupChainKeys.CHAIN_POSITION, + String.valueOf(decision.chainPosition), true)); + // Backup full-vs-incremental type lives on backup.type (set by takeBackup) — single + // source of truth. Not duplicated into backup_details. + if (decision.isIncremental()) { + // Resolve the parent backup's UUID so restore can walk the chain by id, not by path. + String parentUuid = lookupParentBackupUuid(backup.getVmId(), decision.bitmapParent); + if (parentUuid != null) { + backupDetailsDao.persist(new BackupDetailVO(backup.getId(), NASBackupChainKeys.PARENT_BACKUP_ID, parentUuid, true)); + } + } + } + + /** + * Upsert the VM's {@code nas.active_checkpoint_id} detail to {@code bitmapName}. Called + * after every successful backup so the next backup's parent-bitmap decision is anchored + * on what actually exists on QEMU, not on "last backup taken". + */ + private void upsertVmActiveCheckpoint(long vmId, String bitmapName) { + VMInstanceDetailVO existing = vmInstanceDetailsDao.findDetail(vmId, NASBackupChainKeys.VM_ACTIVE_CHECKPOINT_ID); + if (existing == null) { + vmInstanceDetailsDao.addDetail(vmId, NASBackupChainKeys.VM_ACTIVE_CHECKPOINT_ID, bitmapName, false); + return; + } + existing.setValue(bitmapName); + vmInstanceDetailsDao.update(existing.getId(), existing); + } + + /** + * Remove the VM's {@code nas.active_checkpoint_id} detail. Called from the restore paths: + * after restore the disk image has no QEMU bitmap attached, so any future incremental + * would be based on stale state. Clearing forces the next backup to be a fresh full. + */ + private void clearVmActiveCheckpoint(long vmId) { + VMInstanceDetailVO existing = vmInstanceDetailsDao.findDetail(vmId, NASBackupChainKeys.VM_ACTIVE_CHECKPOINT_ID); + if (existing != null) { + vmInstanceDetailsDao.removeDetail(vmId, NASBackupChainKeys.VM_ACTIVE_CHECKPOINT_ID); + LOG.debug("Cleared nas.active_checkpoint_id for VM id={} (was {})", vmId, existing.getValue()); + } + } + + private String lookupParentBackupUuid(long vmId, String parentBitmap) { + if (parentBitmap == null) { + return null; + } + for (Backup b : backupDao.listByVmId(null, vmId)) { + String bm = readDetail(b, NASBackupChainKeys.BITMAP_NAME); + if (parentBitmap.equals(bm)) { + return b.getUuid(); + } + } + return null; + } + protected Host getVMHypervisorHostForBackup(VirtualMachine vm) { Long hostId = vm.getHostId(); if (hostId == null && VirtualMachine.State.Running.equals(vm.getState())) { @@ -199,12 +581,21 @@ public Pair takeBackup(final VirtualMachine vm, Boolean quiesce final String backupPath = String.format("%s/%s", vm.getInstanceName(), new SimpleDateFormat("yyyy.MM.dd.HH.mm.ss").format(creationDate)); - BackupVO backupVO = createBackupObject(vm, backupPath); + // Decide full vs incremental for this backup. Stopped VMs are always full + // (libvirt backup-begin requires a running QEMU process). + ChainDecision decision = decideChain(vm); + + BackupVO backupVO = createBackupObject(vm, backupPath, decision.isIncremental() ? "INCREMENTAL" : "FULL"); TakeBackupCommand command = new TakeBackupCommand(vm.getInstanceName(), backupPath); command.setBackupRepoType(backupRepository.getType()); command.setBackupRepoAddress(backupRepository.getAddress()); command.setMountOptions(backupRepository.getMountOptions()); command.setQuiesce(quiesceVM); + command.setQuiesceTimeout(NASBackupQuiesceAgentTimeout.valueIn(vm.getDataCenterId())); + command.setMode(decision.mode); + command.setBitmapNew(decision.bitmapNew); + command.setBitmapParent(decision.bitmapParent); + command.setParentPaths(decision.parentPaths); if (VirtualMachine.State.Stopped.equals(vm.getState())) { List vmVolumes = volumeDao.findByInstance(vm.getId()); @@ -233,32 +624,68 @@ public Pair takeBackup(final VirtualMachine vm, Boolean quiesce backupVO.setDate(new Date()); backupVO.setSize(answer.getSize()); backupVO.setStatus(Backup.Status.BackedUp); + // If the agent fell back to full (stopped VM mid-incremental cycle), record this + // backup as a full and start a new chain. + ChainDecision effective = decision; + if (answer.getIncrementalFallback()) { + effective = ChainDecision.fullStart(decision.bitmapNew); + backupVO.setType("FULL"); + } List volumes = new ArrayList<>(volumeDao.findByInstance(vm.getId())); backupVO.setBackedUpVolumes(backupManager.createVolumeInfoFromVolumes(volumes)); if (backupDao.update(backupVO.getId(), backupVO)) { + // Legacy-full (incremental feature disabled): persist no chain/checkpoint metadata + // and do not touch the VM's active_checkpoint_id — keep the feature-off path legacy. + if (!decision.isLegacyFull()) { + persistChainMetadata(backupVO, effective, answer.getBitmapCreated()); + // Pin the VM's active_checkpoint_id to whichever bitmap the agent actually + // created — the only valid parent for the next incremental (see decideChain). + // If the agent reports no bitmap (bitmapCreated=null), clear any stale detail + // so the next backup starts a fresh full. + String confirmedBitmap = answer.getBitmapCreated(); + if (confirmedBitmap != null) { + upsertVmActiveCheckpoint(vm.getId(), confirmedBitmap); + } else { + clearVmActiveCheckpoint(vm.getId()); + } + } return new Pair<>(true, backupVO); } else { throw new CloudRuntimeException("Failed to update backup"); } } else { logger.error("Failed to take backup for VM {}: {}", vm.getInstanceName(), answer != null ? answer.getDetails() : "No answer received"); - if (answer.getNeedsCleanup()) { - logger.error("Backup cleanup failed for VM {}. Leaving the backup in Error state.", vm.getInstanceName()); + if (answer != null && answer.getNeedsCleanup()) { + logger.error("Backup cleanup failed for VM {}. Leaving the backup in Error state. Backup should be manually deleted to free up the space", vm.getInstanceName()); + sendBackupCleanupFailedAlert(vm, host, answer.getDetails()); backupVO.setStatus(Backup.Status.Error); backupDao.update(backupVO.getId(), backupVO); - } else { - backupVO.setStatus(Backup.Status.Failed); - backupDao.remove(backupVO.getId()); + // Return the row to the caller to record its schedule - e.g. Scheduled or Manual. + return new Pair<>(false, backupVO); } + backupVO.setStatus(Backup.Status.Failed); + backupDao.remove(backupVO.getId()); return new Pair<>(false, null); } } - private BackupVO createBackupObject(VirtualMachine vm, String backupPath) { + /** + * Alerts when nasbackup.sh could not clean up after a failed backup. The script's output + * names the backup repository mount point it left on the host. + */ + private void sendBackupCleanupFailedAlert(VirtualMachine vm, Host host, String details) { + String subject = String.format("NAS backup cleanup failed for VM %s on host %s", vm.getInstanceName(), host.getName()); + String body = String.format("A failed backup of VM %s could not be cleaned up on host %s (%s). " + + "The backup repository may still be mounted on that host. Script output: %s", + vm.getInstanceName(), host.getName(), host.getPrivateIpAddress(), details); + alertManager.sendAlert(AlertManager.AlertType.ALERT_TYPE_BACKUP_STORAGE, vm.getDataCenterId(), host.getPodId(), subject, body); + } + + private BackupVO createBackupObject(VirtualMachine vm, String backupPath, String type) { BackupVO backup = new BackupVO(); backup.setVmId(vm.getId()); backup.setExternalId(backupPath); - backup.setType("FULL"); + backup.setType(type); backup.setDate(new Date()); long virtualSize = 0L; for (final Volume volume: volumeDao.findByInstance(vm.getId())) { @@ -290,15 +717,21 @@ public boolean restoreVMFromBackup(VirtualMachine vm, Backup backup) { } private Pair restoreVMBackup(VirtualMachine vm, Backup backup) { - List backedVolumes = backup.getBackedUpVolumes(); - List backedVolumesUUIDs = backedVolumes.stream() + // Sort once and derive every per-volume list from that same ordering. The UUID list and the + // backup file list are consumed index-by-index on the agent side, so they have to agree; + // sorting only one of them leaves the two out of step whenever the stored order of the + // backed up volumes differs from their device id order. + List backedVolumes = backup.getBackedUpVolumes().stream() .sorted(Comparator.comparingLong(Backup.VolumeInfo::getDeviceId)) + .collect(Collectors.toList()); + List backedVolumesUUIDs = backedVolumes.stream() .map(Backup.VolumeInfo::getUuid) .collect(Collectors.toList()); - List restoreVolumes = volumeDao.findByInstance(vm.getId()).stream() + List vmVolumes = volumeDao.findByInstance(vm.getId()).stream() .sorted(Comparator.comparingLong(VolumeVO::getDeviceId)) .collect(Collectors.toList()); + List restoreVolumes = getVolumesToRestoreInto(backedVolumes, vmVolumes); LOG.debug("Restoring vm {} from backup {} on the NAS Backup Provider", vm, backup); BackupRepository backupRepository = getBackupRepository(backup); @@ -327,9 +760,53 @@ private Pair restoreVMBackup(VirtualMachine vm, Backup backup) } catch (OperationTimedoutException e) { throw new CloudRuntimeException("Operation to restore backup timed out, please try again"); } + if (answer == null) { + throw new CloudRuntimeException(String.format("No answer received from the backend control plane while restoring VM %s from backup %s", vm.getInstanceName(), backup.getUuid())); + } + // After a restore the QEMU dirty-bitmap chain is gone, so clear active_checkpoint_id so + // the next backup is taken as a fresh full and starts a new chain. See decideChain. + if (answer.getResult()) { + clearVmActiveCheckpoint(vm.getId()); + } return new Pair<>(answer.getResult(), answer.getDetails()); } + /** + * Returns, for each backed up volume in the given order, the instance's volume its backup has to be + * written into. + *

+ * A backed up volume goes into the instance's volume with the same uuid. The uuid is compared, not the + * volume's path: the path is the volume's name on the storage, and it differs from the uuid once the + * volume was migrated to another pool, or when it was created from a snapshot or an upload. + *

+ * Creating an instance from a backup gives it brand new volumes, so none of the uuids can match. Only then + * are the instance's volumes taken in device id order. When some of them match, a missing one is a volume + * that is no longer attached, and the restore is refused before anything is written, instead of writing + * that backup into another volume. + */ + protected List getVolumesToRestoreInto(List backedVolumes, List vmVolumes) { + Map vmVolumeByUuid = vmVolumes.stream().collect(Collectors.toMap(VolumeVO::getUuid, volume -> volume)); + boolean restoringIntoNewVolumes = backedVolumes.stream().noneMatch(backedVolume -> vmVolumeByUuid.containsKey(backedVolume.getUuid())); + if (restoringIntoNewVolumes) { + if (backedVolumes.size() != vmVolumes.size()) { + throw new CloudRuntimeException(String.format("Unable to restore backup: it holds %d volumes but the instance has %d.", + backedVolumes.size(), vmVolumes.size())); + } + LOG.debug("None of the backed up volumes belong to this instance; restoring into its volumes in device id order."); + return vmVolumes; + } + List restoreVolumes = new ArrayList<>(); + for (Backup.VolumeInfo backedVolume : backedVolumes) { + VolumeVO volume = vmVolumeByUuid.get(backedVolume.getUuid()); + if (volume == null) { + throw new CloudRuntimeException(String.format("Unable to restore backup: volume [%s] recorded in the backup" + + " is not attached to the instance any more.", backedVolume.getUuid())); + } + restoreVolumes.add(volume); + } + return restoreVolumes; + } + private List getBackupFiles(List backedVolumes) { List backupFiles = new ArrayList<>(); for (Backup.VolumeInfo backedVolume : backedVolumes) { @@ -451,6 +928,13 @@ public Pair restoreBackedUpVolume(Backup backup, Backup.VolumeI } catch (Exception e) { throw new CloudRuntimeException("Unable to create restored volume due to: " + e); } + // The restored volume is attached to this VM with no QEMU bitmap on its image, so the + // VM's tracked checkpoint is now stale; clear it to force the next backup to be a full + // (mirrors the full restore paths restoreVMFromBackup/restoreBackupToVM). + VirtualMachine restoreTargetVm = vmInstanceDao.findVMByInstanceName(vmNameAndState.first()); + if (restoreTargetVm != null) { + clearVmActiveCheckpoint(restoreTargetVm.getId()); + } } return new Pair<>(answer.getResult(), answer.getDetails()); @@ -471,6 +955,14 @@ private Backup.VolumeInfo getBackedUpVolumeInfo(List backedUp .orElse(null); } + @Override + public boolean handlesChainDeleteResourceAccounting() { + // This provider deletes whole incremental chains (leaf + swept delete-pending ancestors) + // and decrements resource count/usage once per physically-removed backup itself, so the + // manager must not also decrement or remove the DB row. + return true; + } + @Override public boolean deleteBackup(Backup backup, boolean forced) { final BackupRepository backupRepository = backupRepositoryDao.findByBackupOfferingId(backup.getBackupOfferingId()); @@ -489,9 +981,74 @@ public boolean deleteBackup(Backup backup, boolean forced) { throw new CloudRuntimeException(String.format("Unable to find a running KVM host in zone %d to delete backup %s", backup.getZoneId(), backup.getUuid())); } - DeleteBackupCommand command = new DeleteBackupCommand(backup.getExternalId(), backupRepository.getType(), - backupRepository.getAddress(), backupRepository.getMountOptions()); + // Backups outside any tracked chain (legacy or non-incremental providers) are + // deleted straight away — no children semantics apply. + if (readDetail(backup, NASBackupChainKeys.CHAIN_ID) == null) { + return deleteBackupFileAndRow(backup, backupRepository, host); + } + + // Concurrent deletes can mutate the same chain concurrently resulting in inconsistent states. + // take a per-VM lock before modifying the chain. + final GlobalLock chainLock = acquireChainDeleteLock(backup.getVmId()); + try { + Backup current = backupDao.findById(backup.getId()); + if (current == null) { + LOG.debug("Backup {} was already removed by a concurrent chain delete", backup.getUuid()); + return true; + } + backup = current; + + // Snapshot-style cascade: defer the on-NAS rm + DB row while there are live children, + // mark this backup as delete-pending, and let the leaf's deletion sweep it up later. + // See DefaultSnapshotStrategy#deleteSnapshotChain for the same pattern on incremental + // snapshots. forced=true means the caller wants the entire subtree gone right now. + if (forced) { + return cascadeDeleteSubtree(backup, backupRepository, host); + } + + if (hasLiveChildren(backup)) { + markDeletePending(backup); + LOG.debug("Backup {} has live descendants in its chain; setting status as Hidden. " + + "The on-NAS file and DB row will be removed once the last descendant is gone, " + + "or pass forced=true.", + backup.getUuid()); + return true; + } + // No live children — physically delete this backup, then walk up the chain and + // collect any ancestors that were left in Hidden state. + return deleteLeafBackupAndSweepPendingAncestors(backup, backupRepository, host); + } finally { + releaseChainDeleteLock(chainLock); + } + } + + /** + * Take the per-VM lock that serializes chain-mutating backup deletes. + */ + protected GlobalLock acquireChainDeleteLock(long vmId) { + GlobalLock lock = GlobalLock.getInternLock("nas.backup.chain.vm." + vmId); + if (!lock.lock(CHAIN_LOCK_TIMEOUT_SECONDS)) { + lock.releaseRef(); + throw new CloudRuntimeException(String.format( + "Timed out waiting for concurrent backup chain operations on VM %d to finish, please try again", vmId)); + } + return lock; + } + + protected void releaseChainDeleteLock(GlobalLock lock) { + lock.unlock(); + lock.releaseRef(); + } + + /** + * The single physical-delete step: rm the on-NAS directory, then remove the DB row. + * Returns {@code false} (and leaves both intact) if the agent reports failure, so the + * caller's recursion stops cleanly. + */ + private boolean deleteBackupFileAndRow(Backup backup, BackupRepository repo, Host host) { + DeleteBackupCommand command = new DeleteBackupCommand(backup.getExternalId(), repo.getType(), + repo.getAddress(), repo.getMountOptions()); BackupAnswer answer; try { answer = (BackupAnswer) agentManager.send(host.getId(), command); @@ -500,15 +1057,157 @@ public boolean deleteBackup(Backup backup, boolean forced) { } catch (OperationTimedoutException e) { throw new CloudRuntimeException("Operation to delete backup timed out, please try again"); } + if (answer == null || !answer.getResult()) { + logger.error("Failed to delete backup file for {} ({}); leaving DB row intact", + backup.getUuid(), backup.getExternalId()); + return false; + } + // Capture the deleted backup's bitmap before the row (and its backup_details) are removed. + String deletedBitmap = readDetail(backup, NASBackupChainKeys.BITMAP_NAME); + backupDao.remove(backup.getId()); + // If this backup owned the bitmap the VM's active_checkpoint_id points to, that on-host QEMU + // dirty-bitmap is gone with it — clear active_checkpoint_id so the next backup starts a fresh + // full chain instead of anchoring an incremental on a deleted checkpoint (test: delete last backup). + if (deletedBitmap != null && deletedBitmap.equals(readVmActiveCheckpoint(backup.getVmId()))) { + clearVmActiveCheckpoint(backup.getVmId()); + } + // Exactly-once resource accounting: decrement at the single point a backup row + file are + // physically removed. This runs for the leaf and for every swept delete-pending ancestor, + // so a chain delete decrements once per actually-removed backup. The manager skips its own + // accounting for this provider (see handlesChainDeleteResourceAccounting()). + long size = backup.getSize() != null ? backup.getSize() : 0L; + resourceLimitMgr.decrementResourceCount(backup.getAccountId(), Resource.ResourceType.backup); + resourceLimitMgr.decrementResourceCount(backup.getAccountId(), Resource.ResourceType.backup_storage, size); + return true; + } - if (answer != null && answer.getResult()) { - return backupDao.remove(backup.getId()); + /** + * Tombstone {@code backup} by moving it to {@link Backup.Status#Hidden}. Idempotent. + * The row stays in the DB so the chain GC can sweep it once its last descendant is deleted + * ({@code listByVmId} is status-agnostic), but it disappears from the user-facing list + * ({@link BackupManagerImpl#listBackups} filters Hidden) and all backup operations refuse it + * (they require {@code BackedUp}). Replaces the previous {@code nas.delete_pending} detail. + */ + private void markDeletePending(Backup backup) { + if (Backup.Status.Hidden.equals(backup.getStatus())) { + return; + } + BackupVO vo = backupDao.findById(backup.getId()); + if (vo != null) { + vo.setStatus(Backup.Status.Hidden); + backupDao.update(vo.getId(), vo); } + } + + /** + * @return true if this backup is a tombstone (Hidden) awaiting chain sweep. + */ + private boolean isDeletePending(Backup backup) { + return Backup.Status.Hidden.equals(backup.getStatus()); + } - logger.debug("There was an error removing the backup with id {}", backup.getId()); + /** + * Whether there are any live (not tombstoned/Hidden, not Removed) descendants of {@code backup} within the + * same chain. This uses {@code CHAIN_POSITION} to detect dependents deeper in the chain. + */ + private boolean hasLiveChildren(Backup backup) { + String chainId = readDetail(backup, NASBackupChainKeys.CHAIN_ID); + if (chainId == null) { + return false; + } + int position = chainPosition(backup); + for (Backup b : backupDao.listByVmId(null, backup.getVmId())) { + if (b.getId() == backup.getId() || !chainId.equals(readDetail(b, NASBackupChainKeys.CHAIN_ID))) { + continue; + } + if (!isDeletePending(b) && chainPosition(b) > position) { + return true; + } + } return false; } + /** + * Return the chain containing {@code member}, ordered leaf-first + * (highest {@code CHAIN_POSITION} → root). + * + *

Materialises the chain via a single {@link BackupDao#listByVmId} call. Callers that + * previously walked the chain by repeatedly calling {@link #findChainParent} were doing + * O(N) {@code listByVmId} calls (one per ancestor); this collapses that to one. + * + *

If {@code member} has no {@code CHAIN_ID} metadata it is returned as a one-element + * list (it is its own degenerate chain). + */ + private List getChainOrderedLeafToRoot(Backup member) { + String chainId = readDetail(member, NASBackupChainKeys.CHAIN_ID); + if (chainId == null) { + return Collections.singletonList(member); + } + List chain = new ArrayList<>(); + for (Backup b : backupDao.listByVmId(null, member.getVmId())) { + if (chainId.equals(readDetail(b, NASBackupChainKeys.CHAIN_ID))) { + chain.add(b); + } + } + // Descending CHAIN_POSITION = leaf-first (highest position = furthest from root). + chain.sort((a, b) -> Integer.compare(chainPosition(b), chainPosition(a))); + return chain; + } + + /** + * Physically delete the leaf {@code backup}, then walk up the chain while each ancestor + * is in Hidden state. Mirrors the snapshot subsystem pattern: once a leaf is + * gone, garbage-collect any tombstoned parents. + * + *

Caller must guarantee {@code backup} is a leaf (no live children). Each tombstoned + * ancestor is by definition childless once its sole child is deleted here, so no extra + * live-children check is needed inside the loop. + */ + private boolean deleteLeafBackupAndSweepPendingAncestors(Backup backup, BackupRepository repo, Host host) { + // Snapshot the chain BEFORE the leaf delete — deleteBackupFileAndRow removes the row, + // after which the in-memory list still resolves but the DB no longer would. + List chain = getChainOrderedLeafToRoot(backup); + if (!deleteBackupFileAndRow(backup, repo, host)) { + return false; + } + for (Backup member : chain) { + if (member.getId() == backup.getId()) { + continue; + } + if (!isDeletePending(member)) { + break; + } + deleteBackupFileAndRow(member, repo, host); + } + return true; + } + + /** + * Forced delete of {@code root}'s entire chain, leaf-first. NAS backups form a linear + * chain (full → inc → inc → …), not a tree, so we just walk the ordered chain and + * delete each member without re-querying parents. + */ + private boolean cascadeDeleteSubtree(Backup root, BackupRepository repo, Host host) { + for (Backup b : getChainOrderedLeafToRoot(root)) { + if (!deleteBackupFileAndRow(b, repo, host)) { + return false; + } + } + return true; + } + + private int chainPosition(Backup b) { + String s = readDetail(b, NASBackupChainKeys.CHAIN_POSITION); + if (s == null) { + return Integer.MAX_VALUE; // no metadata => sort to end + } + try { + return Integer.parseInt(s); + } catch (NumberFormatException e) { + return Integer.MAX_VALUE; + } + } + public void syncBackupMetrics(Long zoneId) { } @@ -537,6 +1236,11 @@ public boolean assignVMToBackupOffering(VirtualMachine vm, BackupOffering backup @Override public boolean removeVMFromBackupOffering(VirtualMachine vm) { + // Clear the VM's active checkpoint so any future re-assignment to a backup offering + // starts a fresh chain. Without this, a detach-volume + attach-different-volume cycle + // while the offering is unassigned would lead to the next backup trying to rebase + // onto a stale parent (different volume identity, same VM id). + clearVmActiveCheckpoint(vm.getId()); return true; } @@ -618,7 +1322,10 @@ public Boolean crossZoneInstanceCreationEnabled(BackupOffering backupOffering) { @Override public ConfigKey[] getConfigKeys() { return new ConfigKey[]{ - NASBackupRestoreMountTimeout + NASBackupRestoreMountTimeout, + NASBackupFullEvery, + NASBackupIncrementalEnabled, + NASBackupQuiesceAgentTimeout }; } diff --git a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java index 7540cabbbf52..fbd681e726f8 100644 --- a/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java +++ b/plugins/backup/nas/src/test/java/org/apache/cloudstack/backup/NASBackupProviderTest.java @@ -27,6 +27,7 @@ import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.Mockito; @@ -35,26 +36,45 @@ import org.springframework.test.util.ReflectionTestUtils; import com.cloud.agent.AgentManager; +import com.cloud.alert.AlertManager; import com.cloud.exception.AgentUnavailableException; import com.cloud.exception.OperationTimedoutException; +import com.cloud.configuration.Resource; import com.cloud.host.Host; import com.cloud.host.HostVO; import com.cloud.host.Status; import com.cloud.host.dao.HostDao; import com.cloud.hypervisor.Hypervisor; import com.cloud.resource.ResourceManager; +import com.cloud.storage.DiskOfferingVO; +import com.cloud.storage.ScopeType; +import com.cloud.storage.Storage; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; +import com.cloud.storage.dao.DiskOfferingDao; import com.cloud.storage.dao.VolumeDao; +import com.cloud.user.ResourceLimitService; import com.cloud.utils.Pair; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.vm.VMInstanceDetailVO; import com.cloud.vm.VMInstanceVO; +import com.cloud.vm.VirtualMachine; import com.cloud.vm.dao.VMInstanceDao; +import com.cloud.vm.dao.VMInstanceDetailsDao; + +import com.google.gson.Gson; import org.apache.cloudstack.backup.dao.BackupDao; +import org.apache.cloudstack.backup.dao.BackupDetailsDao; import org.apache.cloudstack.backup.dao.BackupRepositoryDao; import org.apache.cloudstack.backup.dao.BackupOfferingDao; import org.apache.cloudstack.storage.datastore.db.PrimaryDataStoreDao; import org.apache.cloudstack.storage.datastore.db.StoragePoolVO; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreManager; +import org.apache.cloudstack.engine.subsystem.api.storage.DataStore; +import com.cloud.storage.DataStoreRole; +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; @RunWith(MockitoJUnitRunner.class) public class NASBackupProviderTest { @@ -92,6 +112,24 @@ public class NASBackupProviderTest { @Mock private PrimaryDataStoreDao storagePoolDao; + @Mock + private BackupDetailsDao backupDetailsDao; + + @Mock + private VMInstanceDetailsDao vmInstanceDetailsDao; + + @Mock + private DiskOfferingDao diskOfferingDao; + + @Mock + private ResourceLimitService resourceLimitMgr; + + @Mock + private DataStoreManager dataStoreMgr; + + @Mock + private AlertManager alertManager; + @Test public void testDeleteBackup() throws OperationTimedoutException, AgentUnavailableException { Long hostId = 1L; @@ -169,16 +207,7 @@ public void testGetBackupStorageStats() { Assert.assertEquals(Long.valueOf(3000L), result.second()); } - @Test - public void takeBackupSuccessfully() throws AgentUnavailableException, OperationTimedoutException { - Long vmId = 1L; - Long hostId = 2L; - Long backupOfferingId = 3L; - Long accountId = 4L; - Long domainId = 5L; - Long zoneId = 6L; - Long backupId = 7L; - + private VMInstanceVO mockRunningVmForBackup(Long vmId, Long hostId, Long backupOfferingId, Long accountId, Long domainId, Long zoneId) { VMInstanceVO vm = mock(VMInstanceVO.class); Mockito.when(vm.getId()).thenReturn(vmId); Mockito.when(vm.getHostId()).thenReturn(hostId); @@ -208,6 +237,16 @@ public void takeBackupSuccessfully() throws AgentUnavailableException, Operation Mockito.when(volume2.getState()).thenReturn(Volume.State.Ready); Mockito.when(volume2.getSize()).thenReturn(200L); Mockito.when(volumeDao.findByInstance(vmId)).thenReturn(List.of(volume1, volume2)); + return vm; + } + + @Test + public void takeBackupSuccessfully() throws AgentUnavailableException, OperationTimedoutException { + Long backupOfferingId = 3L; + Long accountId = 4L; + Long domainId = 5L; + Long zoneId = 6L; + VMInstanceVO vm = mockRunningVmForBackup(1L, 2L, backupOfferingId, accountId, domainId, zoneId); BackupAnswer answer = mock(BackupAnswer.class); Mockito.when(answer.getResult()).thenReturn(true); @@ -236,6 +275,60 @@ public void takeBackupSuccessfully() throws AgentUnavailableException, Operation Mockito.verify(agentManager).send(anyLong(), Mockito.any(TakeBackupCommand.class)); } + @Test + public void takeBackupFailureNeedingCleanupReturnsErrorBackup() throws AgentUnavailableException, OperationTimedoutException { + VMInstanceVO vm = mockRunningVmForBackup(1L, 2L, 3L, 4L, 5L, 6L); + HostVO host = hostDao.findById(2L); + Mockito.when(host.getName()).thenReturn("kvm01"); + Mockito.when(host.getPodId()).thenReturn(7L); + + BackupAnswer answer = mock(BackupAnswer.class); + Mockito.when(answer.getResult()).thenReturn(false); + Mockito.when(answer.getNeedsCleanup()).thenReturn(true); + Mockito.when(answer.getDetails()).thenReturn("leaving /tmp/csbackup.abcde/i-2-3-VM mounted at /tmp/csbackup.abcde"); + ArgumentCaptor commandCaptor = ArgumentCaptor.forClass(TakeBackupCommand.class); + Mockito.when(agentManager.send(anyLong(), commandCaptor.capture())).thenReturn(answer); + + Mockito.when(backupDao.persist(Mockito.any(BackupVO.class))).thenAnswer(invocation -> invocation.getArgument(0)); + + Pair result = nasBackupProvider.takeBackup(vm, true); + + Assert.assertFalse(result.first()); + Assert.assertNotNull(result.second()); + Assert.assertEquals(Backup.Status.Error, result.second().getStatus()); + Mockito.verify(backupDao, Mockito.never()).remove(Mockito.anyLong()); + + TakeBackupCommand command = commandCaptor.getValue(); + Assert.assertTrue(command.getQuiesce()); + Assert.assertEquals(Integer.valueOf(30), command.getQuiesceTimeout()); + + ArgumentCaptor subjectCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor bodyCaptor = ArgumentCaptor.forClass(String.class); + Mockito.verify(alertManager).sendAlert(Mockito.eq(AlertManager.AlertType.ALERT_TYPE_BACKUP_STORAGE), Mockito.eq(6L), Mockito.eq(7L), + subjectCaptor.capture(), bodyCaptor.capture()); + Assert.assertTrue(subjectCaptor.getValue().contains("kvm01")); + Assert.assertTrue(bodyCaptor.getValue().contains("/tmp/csbackup.abcde")); + } + + @Test + public void takeBackupFailureWithoutCleanupRemovesBackup() throws AgentUnavailableException, OperationTimedoutException { + VMInstanceVO vm = mockRunningVmForBackup(1L, 2L, 3L, 4L, 5L, 6L); + + BackupAnswer answer = mock(BackupAnswer.class); + Mockito.when(answer.getResult()).thenReturn(false); + Mockito.when(answer.getNeedsCleanup()).thenReturn(false); + Mockito.when(agentManager.send(anyLong(), Mockito.any(TakeBackupCommand.class))).thenReturn(answer); + + Mockito.when(backupDao.persist(Mockito.any(BackupVO.class))).thenAnswer(invocation -> invocation.getArgument(0)); + + Pair result = nasBackupProvider.takeBackup(vm, false); + + Assert.assertFalse(result.first()); + Assert.assertNull(result.second()); + Mockito.verify(backupDao).remove(Mockito.anyLong()); + Mockito.verifyNoInteractions(alertManager); + } + @Test public void testGetVMHypervisorHost() { Long hostId = 1L; @@ -349,4 +442,727 @@ public void testGetVMHypervisorHostFallbackToZoneWideKVMHost() { Mockito.verify(hostDao).findHypervisorHostInCluster(clusterId); Mockito.verify(resourceManager).findOneRandomRunningHostByHypervisor(Hypervisor.HypervisorType.KVM, zoneId); } + + private VolumeVO volume(String uuid, String path, long deviceId) { + VolumeVO volume = mock(VolumeVO.class); + Mockito.lenient().when(volume.getUuid()).thenReturn(uuid); + Mockito.lenient().when(volume.getPath()).thenReturn(path); + Mockito.lenient().when(volume.getDeviceId()).thenReturn(deviceId); + return volume; + } + + private Backup.VolumeInfo backedVolume(String uuid, long deviceId) { + return new Backup.VolumeInfo(uuid, uuid, deviceId == 0 ? Volume.Type.ROOT : Volume.Type.DATADISK, 1024L, deviceId, null, null, null); + } + + @Test + public void getVolumesToRestoreIntoMatchesByUuidNotByPathOrDeviceId() { + // data-a was migrated (its path is no longer its uuid) and the data disks swapped device ids + VolumeVO root = volume("root", "root", 0L); + VolumeVO dataB = volume("data-b", "data-b", 1L); + VolumeVO dataA = volume("data-a", "migrated-path-of-a", 2L); + List backed = List.of(backedVolume("root", 0L), backedVolume("data-a", 1L), backedVolume("data-b", 2L)); + + List result = nasBackupProvider.getVolumesToRestoreInto(backed, List.of(root, dataB, dataA)); + + Assert.assertEquals(List.of(root, dataA, dataB), result); + } + + @Test + public void getVolumesToRestoreIntoRefusesWhenABackedUpVolumeIsNoLongerAttached() { + List backed = List.of(backedVolume("root", 0L), backedVolume("data-z", 1L)); + List vmVolumes = List.of(volume("root", "root", 0L), volume("data-a", "data-a", 1L)); + + CloudRuntimeException e = Assert.assertThrows(CloudRuntimeException.class, + () -> nasBackupProvider.getVolumesToRestoreInto(backed, vmVolumes)); + Assert.assertTrue(e.getMessage().contains("data-z")); + } + + @Test + public void getVolumesToRestoreIntoUsesDeviceIdOrderForBrandNewVolumes() { + // an instance created from the backup: none of its volumes has a uuid recorded in the backup + VolumeVO newRoot = volume("new-root", "new-root", 0L); + VolumeVO newData = volume("new-data", "new-data", 1L); + List backed = List.of(backedVolume("root", 0L), backedVolume("data-a", 1L)); + + Assert.assertEquals(List.of(newRoot, newData), nasBackupProvider.getVolumesToRestoreInto(backed, List.of(newRoot, newData))); + } + + @Test(expected = CloudRuntimeException.class) + public void getVolumesToRestoreIntoRefusesBrandNewVolumesOfADifferentCount() { + nasBackupProvider.getVolumesToRestoreInto(List.of(backedVolume("root", 0L), backedVolume("data-a", 1L)), + List.of(volume("new-root", "new-root", 0L))); + } + + @Test + public void restoreVMFromBackupSendsNothingWhenABackedUpVolumeIsNoLongerAttached() throws AgentUnavailableException, OperationTimedoutException { + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(1L); + BackupVO backup = mock(BackupVO.class); + Mockito.when(backup.getBackedUpVolumes()).thenReturn(List.of(backedVolume("root", 0L), backedVolume("data-z", 1L))); + VolumeVO root = volume("root", "root", 0L); + VolumeVO dataA = volume("data-a", "data-a", 1L); + Mockito.when(volumeDao.findByInstance(1L)).thenReturn(List.of(root, dataA)); + + Assert.assertThrows(CloudRuntimeException.class, () -> nasBackupProvider.restoreVMFromBackup(vm, backup)); + Mockito.verify(agentManager, Mockito.never()).send(anyLong(), Mockito.any(com.cloud.agent.api.Command.class)); + } + + // -- nas.backup.incremental.enabled master switch ------------------------------------ + + /** + * When the operator sets nas.backup.incremental.enabled=false at the zone level, every + * backup must be a fresh full anchor, regardless of VM state or nas.backup.full.every. + * This is a single toggle the + * operator can flip without having to count remaining backups in a chain. + */ + @Test + public void decideChainReturnsLegacyFullWhenIncrementalDisabled() { + Long zoneId = 1L; + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.lenient().when(vm.getDataCenterId()).thenReturn(zoneId); + + // Stub the master switch to false. ConfigKey.valueIn delegates to the framework's + // ConfigDepot at runtime; for the unit test we override the in-memory value via the + // ConfigKey's local override (set by ReflectionTestUtils on the spy provider). + ReflectionTestUtils.setField(nasBackupProvider, "NASBackupIncrementalEnabled", + new org.apache.cloudstack.framework.config.ConfigKey<>("Advanced", Boolean.class, + "nas.backup.incremental.enabled", "false", + "test override — disabled", true, + org.apache.cloudstack.framework.config.ConfigKey.Scope.Zone)); + + NASBackupProvider.ChainDecision decision = nasBackupProvider.decideChain(vm); + Assert.assertNotNull(decision); + Assert.assertEquals(NASBackupChainKeys.TYPE_LEGACY_FULL, decision.mode); + Assert.assertNull("legacy-full must not carry a bitmap", decision.bitmapNew); + Assert.assertNull(decision.bitmapParent); + Assert.assertNull("legacy-full must not start a chain", decision.chainId); + Assert.assertEquals(0, decision.chainPosition); + } + + // -- decideChain anchored on VM's active_checkpoint_id ------------------------------- + + /** + * No active_checkpoint_id on the VM (post-restore, first-ever backup, or detail purged) => + * decideChain must return a fresh full. Relying on the last backup taken as the parent + * breaks after a restore, so the decision is anchored on the active checkpoint instead. + */ + @Test + public void decideChainReturnsFullWhenVmHasNoActiveCheckpoint() { + Long zoneId = 1L; + Long vmId = 42L; + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(vmId); + Mockito.when(vm.getDataCenterId()).thenReturn(zoneId); + Mockito.when(vm.getState()).thenReturn(VMInstanceVO.State.Running); + + // Master switch defaults to false (opt-in by zone) — explicitly enable it for this + // test so we exercise the "no active_checkpoint_id" branch rather than short-circuit + // at the master-switch gate. + ReflectionTestUtils.setField(nasBackupProvider, "NASBackupIncrementalEnabled", + new org.apache.cloudstack.framework.config.ConfigKey<>("Advanced", Boolean.class, + "nas.backup.incremental.enabled", "true", + "test override — enabled", true, + org.apache.cloudstack.framework.config.ConfigKey.Scope.Zone)); + + Mockito.when(vmInstanceDetailsDao.findDetail(vmId, NASBackupChainKeys.VM_ACTIVE_CHECKPOINT_ID)).thenReturn(null); + + NASBackupProvider.ChainDecision decision = nasBackupProvider.decideChain(vm); + Assert.assertNotNull(decision); + Assert.assertEquals(NASBackupChainKeys.TYPE_FULL, decision.mode); + Assert.assertNull(decision.bitmapParent); + Assert.assertEquals(0, decision.chainPosition); + } + + // -- incremental storage-capability guard (Ceph-RBD / Linstor stay on legacy full) ---- + + /** + * Incremental checkpoints are only possible on file-based qcow2 storage. A VM whose every + * volume sits on NFS / HOST-scope local / SharedMountPoint is checkpoint-capable. + */ + @Test + public void allVolumesOnCheckpointCapableStorageTrueForNfsHostAndSharedMount() { + Long vmId = 55L; + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(vmId); + + VolumeVO nfsVol = mock(VolumeVO.class); + Mockito.when(nfsVol.getPoolId()).thenReturn(1L); + VolumeVO hostVol = mock(VolumeVO.class); + Mockito.when(hostVol.getPoolId()).thenReturn(2L); + VolumeVO smpVol = mock(VolumeVO.class); + Mockito.when(smpVol.getPoolId()).thenReturn(3L); + Mockito.when(volumeDao.findByInstance(vmId)).thenReturn(List.of(nfsVol, hostVol, smpVol)); + + StoragePoolVO nfs = mock(StoragePoolVO.class); + Mockito.when(nfs.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + StoragePoolVO host = mock(StoragePoolVO.class); + Mockito.when(host.getScope()).thenReturn(ScopeType.HOST); + StoragePoolVO smp = mock(StoragePoolVO.class); + Mockito.when(smp.getPoolType()).thenReturn(Storage.StoragePoolType.SharedMountPoint); + Mockito.when(storagePoolDao.findById(1L)).thenReturn(nfs); + Mockito.when(storagePoolDao.findById(2L)).thenReturn(host); + Mockito.when(storagePoolDao.findById(3L)).thenReturn(smp); + + Assert.assertTrue(nasBackupProvider.allVolumesOnCheckpointCapableStorage(vm)); + } + + /** + * A single volume on Ceph-RBD (or any pool that cannot carry a per-disk checkpoint) forces + * the whole VM onto the legacy full-only path — avoids regressing RBD/Linstor storages. + */ + @Test + public void allVolumesOnCheckpointCapableStorageFalseWhenAnyVolumeOnRbd() { + Long vmId = 56L; + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(vmId); + + VolumeVO nfsVol = mock(VolumeVO.class); + Mockito.when(nfsVol.getPoolId()).thenReturn(1L); + VolumeVO rbdVol = mock(VolumeVO.class); + Mockito.when(rbdVol.getPoolId()).thenReturn(9L); + Mockito.when(volumeDao.findByInstance(vmId)).thenReturn(List.of(nfsVol, rbdVol)); + + StoragePoolVO nfs = mock(StoragePoolVO.class); + Mockito.when(nfs.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + StoragePoolVO rbd = mock(StoragePoolVO.class); + Mockito.when(rbd.getPoolType()).thenReturn(Storage.StoragePoolType.RBD); + Mockito.when(storagePoolDao.findById(1L)).thenReturn(nfs); + Mockito.when(storagePoolDao.findById(9L)).thenReturn(rbd); + + Assert.assertFalse(nasBackupProvider.allVolumesOnCheckpointCapableStorage(vm)); + } + + /** A volume whose storage pool can no longer be resolved is treated as incapable (safe). */ + @Test + public void allVolumesOnCheckpointCapableStorageFalseWhenPoolUnresolvable() { + Long vmId = 57L; + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(vmId); + VolumeVO vol = mock(VolumeVO.class); + Mockito.when(vol.getPoolId()).thenReturn(1L); + Mockito.when(volumeDao.findByInstance(vmId)).thenReturn(List.of(vol)); + Mockito.when(storagePoolDao.findById(1L)).thenReturn(null); + + Assert.assertFalse(nasBackupProvider.allVolumesOnCheckpointCapableStorage(vm)); + } + + // -- restore clears active_checkpoint_id --------------------------------------------- + + /** + * After a successful restoreVMFromBackup, decideChain on the next backup must produce + * a full. We verify this end-to-end by checking that vmInstanceDetailsDao.removeDetail + * is called with the active_checkpoint_id key. + */ + @Test + public void restoreClearsActiveCheckpointDetail() throws AgentUnavailableException, OperationTimedoutException { + Long vmId = 7L; + Long hostId = 8L; + Long backupOfferingId = 9L; + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(vmId); + Mockito.when(vm.getLastHostId()).thenReturn(hostId); + Mockito.when(vm.getRemoved()).thenReturn(null); + Mockito.when(vm.getName()).thenReturn("vm7"); + + HostVO host = mock(HostVO.class); + Mockito.when(host.getStatus()).thenReturn(Status.Up); + Mockito.when(host.getId()).thenReturn(hostId); + Mockito.when(hostDao.findById(hostId)).thenReturn(host); + + BackupVO backup = new BackupVO(); + backup.setVmId(vmId); + backup.setBackupOfferingId(backupOfferingId); + backup.setExternalId("i-2-7-VM/2026.05.16.10.00.00"); + ReflectionTestUtils.setField(backup, "id", 100L); + // backedUpVolumes defaults to null => BackupVO.getBackedUpVolumes returns emptyList(). + + BackupRepositoryVO repo = new BackupRepositoryVO(1L, "nas", "test-repo", + "nfs", "address", "sync", 1024L, null); + Mockito.when(backupRepositoryDao.findByBackupOfferingId(backupOfferingId)).thenReturn(repo); + + Mockito.when(volumeDao.findByInstance(vmId)).thenReturn(Collections.emptyList()); + + BackupAnswer answer = mock(BackupAnswer.class); + Mockito.when(answer.getResult()).thenReturn(true); + Mockito.when(agentManager.send(Mockito.anyLong(), Mockito.any(RestoreBackupCommand.class))).thenReturn(answer); + + // Pre-existing checkpoint detail so removeDetail has something to "clear". + VMInstanceDetailVO existing = mock(VMInstanceDetailVO.class); + Mockito.when(existing.getValue()).thenReturn("backup-1715000000"); + Mockito.when(vmInstanceDetailsDao.findDetail(vmId, NASBackupChainKeys.VM_ACTIVE_CHECKPOINT_ID)).thenReturn(existing); + + boolean ok = nasBackupProvider.restoreVMFromBackup(vm, backup); + Assert.assertTrue(ok); + Mockito.verify(vmInstanceDetailsDao).removeDetail(vmId, NASBackupChainKeys.VM_ACTIVE_CHECKPOINT_ID); + } + + /** + * Single-volume restore (restoreBackedUpVolume) must also clear the target VM's + * active_checkpoint_id, so the next backup of that VM is a fresh full — the restored + * volume's image carries no QEMU bitmap. + */ + @Test + public void restoreBackedUpVolumeClearsTargetVmActiveCheckpoint() + throws AgentUnavailableException, OperationTimedoutException { + Long targetVmId = 42L; + Long backupOfferingId = 9L; + String targetVmName = "i-2-42-VM"; + String volUuid = "vol-uuid-1"; + String hostIp = "10.0.0.5"; + String dsUuid = "ds-uuid-1"; + + VolumeVO srcVolume = mock(VolumeVO.class); + Mockito.when(srcVolume.getUuid()).thenReturn(volUuid); + Mockito.when(srcVolume.getName()).thenReturn("data1"); + Mockito.when(volumeDao.findByUuid(volUuid)).thenReturn(srcVolume); + + DiskOfferingVO diskOffering = mock(DiskOfferingVO.class); + Mockito.when(diskOffering.getId()).thenReturn(5L); + Mockito.when(diskOffering.getProvisioningType()).thenReturn(Storage.ProvisioningType.THIN); + Mockito.when(diskOfferingDao.findByUuid(Mockito.anyString())).thenReturn(diskOffering); + + StoragePoolVO pool = mock(StoragePoolVO.class); + Mockito.when(pool.getId()).thenReturn(11L); + Mockito.when(pool.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + Mockito.when(pool.getUuid()).thenReturn(dsUuid); + Mockito.when(storagePoolDao.findByUuid(dsUuid)).thenReturn(pool); + + HostVO host = mock(HostVO.class); + Mockito.when(host.getId()).thenReturn(8L); + Mockito.when(hostDao.findByIp(hostIp)).thenReturn(host); + + DataStore dataStore = mock(DataStore.class); + PrimaryDataStoreTO dataStoreTO = mock(PrimaryDataStoreTO.class); + Mockito.when(dataStore.getTO()).thenReturn(dataStoreTO); + Mockito.when(dataStoreMgr.getDataStore(11L, DataStoreRole.Primary)).thenReturn(dataStore); + + Backup.VolumeInfo backedUp = new Backup.VolumeInfo(volUuid, "i-2-99-VM/2026/data1.qcow2", + Volume.Type.DATADISK, 1024L, 1L, "disk-offering-uuid", null, null); + + BackupVO backup = new BackupVO(); + backup.setVmId(99L); + backup.setBackupOfferingId(backupOfferingId); + backup.setExternalId("i-2-99-VM/2026.06.22.10.00.00"); + backup.setSize(1024L); + backup.setBackedUpVolumes(new Gson().toJson(Collections.singletonList(backedUp))); + ReflectionTestUtils.setField(backup, "id", 200L); + + BackupRepositoryVO repo = new BackupRepositoryVO(1L, "nas", "test-repo", + "nfs", "address", "sync", 1024L, null); + Mockito.when(backupRepositoryDao.findByBackupOfferingId(backupOfferingId)).thenReturn(repo); + + BackupAnswer answer = mock(BackupAnswer.class); + Mockito.when(answer.getResult()).thenReturn(true); + Mockito.when(answer.getDetails()).thenReturn("success"); + Mockito.when(agentManager.send(Mockito.anyLong(), Mockito.any(RestoreBackupCommand.class))).thenReturn(answer); + + VMInstanceVO targetVm = mock(VMInstanceVO.class); + Mockito.when(targetVm.getId()).thenReturn(targetVmId); + Mockito.when(vmInstanceDao.findVMByInstanceName(targetVmName)).thenReturn(targetVm); + + VMInstanceDetailVO existing = mock(VMInstanceDetailVO.class); + Mockito.when(existing.getValue()).thenReturn("backup-1718000000"); + Mockito.when(vmInstanceDetailsDao.findDetail(targetVmId, NASBackupChainKeys.VM_ACTIVE_CHECKPOINT_ID)).thenReturn(existing); + + Pair result = nasBackupProvider.restoreBackedUpVolume( + backup, backedUp, hostIp, dsUuid, new Pair<>(targetVmName, VirtualMachine.State.Stopped)); + + Assert.assertTrue(result.first()); + Mockito.verify(vmInstanceDetailsDao).removeDetail(targetVmId, NASBackupChainKeys.VM_ACTIVE_CHECKPOINT_ID); + } + + // -- delete-pending cascade ---------------------------------------------------------- + + /** + * Deleting an incremental that has a live child must mark the incremental as + * delete-pending in backup_details and NOT touch the on-NAS file or the backups row. + * A parent with live children is soft-deleted (delete-pending) rather than removed. + */ + @Test + public void deleteWithLiveChildMarksDeletePendingAndPreservesFile() + throws AgentUnavailableException, OperationTimedoutException { + Long zoneId = 1L; + Long vmId = 2L; + Long hostId = 3L; + Long offeringId = 4L; + + BackupVO parent = new BackupVO(); + parent.setVmId(vmId); + parent.setBackupOfferingId(offeringId); + parent.setExternalId("i-2-2-VM/2026.05.10.10.00.00"); + parent.setZoneId(zoneId); + ReflectionTestUtils.setField(parent, "id", 50L); + ReflectionTestUtils.setField(parent, "uuid", "parent-uuid"); + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getLastHostId()).thenReturn(hostId); + HostVO host = mock(HostVO.class); + Mockito.when(host.getStatus()).thenReturn(Status.Up); + // Note: host.getId() is intentionally not stubbed — the live-child path never + // contacts the agent (verified below), so the stub would be unnecessary. + Mockito.when(hostDao.findById(hostId)).thenReturn(host); + + BackupRepositoryVO repo = new BackupRepositoryVO(1L, "nas", "test-repo", + "nfs", "address", "sync", 1024L, null); + Mockito.when(backupRepositoryDao.findByBackupOfferingId(offeringId)).thenReturn(repo); + Mockito.when(vmInstanceDao.findByIdIncludingRemoved(vmId)).thenReturn(vm); + + // CHAIN_ID on the parent => not the no-chain fast path. + BackupDetailVO chainIdDetail = new BackupDetailVO(50L, NASBackupChainKeys.CHAIN_ID, "chain-1", true); + BackupDetailVO chainPosDetail = new BackupDetailVO(50L, NASBackupChainKeys.CHAIN_POSITION, "0", true); + Mockito.when(backupDetailsDao.findDetail(50L, NASBackupChainKeys.CHAIN_ID)).thenReturn(chainIdDetail); + Mockito.when(backupDetailsDao.findDetail(50L, NASBackupChainKeys.CHAIN_POSITION)).thenReturn(chainPosDetail); + + // A live child sits deeper in the chain (higher CHAIN_POSITION). + BackupVO child = new BackupVO(); + child.setVmId(vmId); + child.setBackupOfferingId(offeringId); + child.setExternalId("i-2-2-VM/2026.05.10.10.30.00"); + child.setZoneId(zoneId); + child.setStatus(Backup.Status.BackedUp); + ReflectionTestUtils.setField(child, "id", 51L); + ReflectionTestUtils.setField(child, "uuid", "child-uuid"); + + BackupDetailVO childChainId = new BackupDetailVO(51L, NASBackupChainKeys.CHAIN_ID, "chain-1", true); + BackupDetailVO childChainPos = new BackupDetailVO(51L, NASBackupChainKeys.CHAIN_POSITION, "1", true); + Mockito.when(backupDetailsDao.findDetail(51L, NASBackupChainKeys.CHAIN_ID)).thenReturn(childChainId); + Mockito.when(backupDetailsDao.findDetail(51L, NASBackupChainKeys.CHAIN_POSITION)).thenReturn(childChainPos); + + Mockito.when(backupDao.listByVmId(null, vmId)).thenReturn(List.of(parent, child)); + // Re-read under the chain lock + markDeletePending both load the row by id. + Mockito.when(backupDao.findById(50L)).thenReturn(parent); + Mockito.doReturn(mock(GlobalLock.class)).when(nasBackupProvider).acquireChainDeleteLock(vmId); + + boolean result = nasBackupProvider.deleteBackup(parent, false); + Assert.assertTrue(result); + + // No agent traffic — the on-NAS file must be preserved while children are alive. + Mockito.verify(agentManager, Mockito.never()).send(Mockito.anyLong(), Mockito.any(DeleteBackupCommand.class)); + // No DB row removal — the row is the tombstone marker. + Mockito.verify(backupDao, Mockito.never()).remove(50L); + // A tombstoned backup is NOT decremented — its space is still occupied until swept. + Mockito.verify(resourceLimitMgr, Mockito.never()).decrementResourceCount(Mockito.anyLong(), Mockito.eq(Resource.ResourceType.backup)); + // The tombstoned backup is moved to Status.Hidden (replaces the old DELETE_PENDING detail). + ArgumentCaptor captor = ArgumentCaptor.forClass(BackupVO.class); + Mockito.verify(backupDao).update(Mockito.eq(50L), captor.capture()); + Assert.assertEquals(Backup.Status.Hidden, captor.getValue().getStatus()); + Mockito.verify(backupDetailsDao, Mockito.never()).persist(Mockito.any(BackupDetailVO.class)); + } + + /** + * Deleting a leaf incremental whose parent is delete-pending must (a) delete the leaf and + * then (b) sweep up the tombstoned parent. Mirrors DefaultSnapshotStrategy's + * "delete leaf, then walk up while parent is destroying-and-childless". + */ + @Test + public void deletingLeafSweepsUpDeletePendingParent() + throws AgentUnavailableException, OperationTimedoutException { + Long zoneId = 1L; + Long vmId = 2L; + Long hostId = 3L; + Long offeringId = 4L; + + BackupVO leaf = new BackupVO(); + leaf.setVmId(vmId); + leaf.setBackupOfferingId(offeringId); + leaf.setExternalId("i-2-2-VM/2026.05.10.11.00.00"); + leaf.setZoneId(zoneId); + ReflectionTestUtils.setField(leaf, "id", 51L); + ReflectionTestUtils.setField(leaf, "uuid", "leaf-uuid"); + + BackupVO parent = new BackupVO(); + parent.setVmId(vmId); + parent.setBackupOfferingId(offeringId); + parent.setExternalId("i-2-2-VM/2026.05.10.10.30.00"); + parent.setZoneId(zoneId); + ReflectionTestUtils.setField(parent, "id", 50L); + ReflectionTestUtils.setField(parent, "uuid", "parent-uuid"); + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getLastHostId()).thenReturn(hostId); + HostVO host = mock(HostVO.class); + Mockito.when(host.getStatus()).thenReturn(Status.Up); + Mockito.when(host.getId()).thenReturn(hostId); + Mockito.when(hostDao.findById(hostId)).thenReturn(host); + + BackupRepositoryVO repo = new BackupRepositoryVO(1L, "nas", "test-repo", + "nfs", "address", "sync", 1024L, null); + Mockito.when(backupRepositoryDao.findByBackupOfferingId(offeringId)).thenReturn(repo); + Mockito.when(vmInstanceDao.findByIdIncludingRemoved(vmId)).thenReturn(vm); + + // Leaf details. CHAIN_POSITION=1 puts the leaf after the full anchor in the + // ordered chain — getChainOrderedLeafToRoot sorts by CHAIN_POSITION descending. + BackupDetailVO leafChainId = new BackupDetailVO(51L, NASBackupChainKeys.CHAIN_ID, "chain-1", true); + BackupDetailVO leafChainPos = new BackupDetailVO(51L, NASBackupChainKeys.CHAIN_POSITION, "1", true); + Mockito.when(backupDetailsDao.findDetail(51L, NASBackupChainKeys.CHAIN_ID)).thenReturn(leafChainId); + Mockito.when(backupDetailsDao.findDetail(51L, NASBackupChainKeys.CHAIN_POSITION)).thenReturn(leafChainPos); + + // Parent is the tombstoned full anchor (CHAIN_POSITION=0). + BackupDetailVO parentChainId = new BackupDetailVO(50L, NASBackupChainKeys.CHAIN_ID, "chain-1", true); + BackupDetailVO parentChainPos = new BackupDetailVO(50L, NASBackupChainKeys.CHAIN_POSITION, "0", true); + // The parent is the tombstone — now represented by Status.Hidden (was the DELETE_PENDING detail). + parent.setStatus(Backup.Status.Hidden); + Mockito.when(backupDetailsDao.findDetail(50L, NASBackupChainKeys.CHAIN_ID)).thenReturn(parentChainId); + Mockito.when(backupDetailsDao.findDetail(50L, NASBackupChainKeys.CHAIN_POSITION)).thenReturn(parentChainPos); + + // We still use a mutable list + remove() answer so the DAO contract is realistic. + java.util.List liveBackups = new java.util.ArrayList<>(List.of(parent, leaf)); + Mockito.when(backupDao.listByVmId(null, vmId)).thenAnswer(inv -> new java.util.ArrayList<>(liveBackups)); + // The target row is re-read under the chain lock before any chain decision. + Mockito.when(backupDao.findById(51L)).thenReturn(leaf); + Mockito.doReturn(mock(GlobalLock.class)).when(nasBackupProvider).acquireChainDeleteLock(vmId); + + // Agent acknowledges every delete. + Mockito.when(agentManager.send(Mockito.anyLong(), Mockito.any(DeleteBackupCommand.class))) + .thenReturn(new BackupAnswer(new DeleteBackupCommand(null, null, null, null), true, "ok")); + // backupDao.remove(id) drops the corresponding row from the live list so the next + // listByVmId call reflects post-delete state — mirrors the real DAO contract. + Mockito.when(backupDao.remove(Mockito.anyLong())).thenAnswer(inv -> { + Long id = inv.getArgument(0); + liveBackups.removeIf(b -> b.getId() == id); + return true; + }); + + boolean result = nasBackupProvider.deleteBackup(leaf, false); + Assert.assertTrue(result); + + // Both backups must be physically deleted (leaf first, then tombstoned parent). + Mockito.verify(agentManager, Mockito.times(2)) + .send(Mockito.anyLong(), Mockito.any(DeleteBackupCommand.class)); + Mockito.verify(backupDao).remove(51L); + Mockito.verify(backupDao).remove(50L); + // Exactly-once resource accounting: decremented for BOTH physically-removed backups + // (leaf + swept ancestor), not just one. + Mockito.verify(resourceLimitMgr, Mockito.times(2)) + .decrementResourceCount(Mockito.anyLong(), Mockito.eq(Resource.ResourceType.backup)); + Mockito.verify(resourceLimitMgr, Mockito.times(2)) + .decrementResourceCount(Mockito.anyLong(), Mockito.eq(Resource.ResourceType.backup_storage), Mockito.any()); + } + + @Test + public void deletingLastLiveMemberCollectsDeeperOrphanTombstones() + throws AgentUnavailableException, OperationTimedoutException { + Long zoneId = 1L; + Long vmId = 2L; + Long hostId = 3L; + Long offeringId = 4L; + + BackupVO full = new BackupVO(); + full.setVmId(vmId); + full.setBackupOfferingId(offeringId); + full.setExternalId("i-2-2-VM/2026.05.10.10.00.00"); + full.setZoneId(zoneId); + full.setStatus(Backup.Status.BackedUp); + ReflectionTestUtils.setField(full, "id", 50L); + + // The last live incremental — the one being deleted. + BackupVO inc1 = new BackupVO(); + inc1.setVmId(vmId); + inc1.setBackupOfferingId(offeringId); + inc1.setExternalId("i-2-2-VM/2026.05.10.10.30.00"); + inc1.setZoneId(zoneId); + inc1.setStatus(Backup.Status.BackedUp); + ReflectionTestUtils.setField(inc1, "id", 51L); + + // A stranded tombstone deeper in the chain: its own descendants are long gone, so + // only a sweep triggered by an ancestor's deletion can ever collect it. + BackupVO orphan = new BackupVO(); + orphan.setVmId(vmId); + orphan.setBackupOfferingId(offeringId); + orphan.setExternalId("i-2-2-VM/2026.05.10.11.00.00"); + orphan.setZoneId(zoneId); + orphan.setStatus(Backup.Status.Hidden); + ReflectionTestUtils.setField(orphan, "id", 52L); + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getLastHostId()).thenReturn(hostId); + HostVO host = mock(HostVO.class); + Mockito.when(host.getStatus()).thenReturn(Status.Up); + Mockito.when(host.getId()).thenReturn(hostId); + Mockito.when(hostDao.findById(hostId)).thenReturn(host); + + BackupRepositoryVO repo = new BackupRepositoryVO(1L, "nas", "test-repo", + "nfs", "address", "sync", 1024L, null); + Mockito.when(backupRepositoryDao.findByBackupOfferingId(offeringId)).thenReturn(repo); + Mockito.when(vmInstanceDao.findByIdIncludingRemoved(vmId)).thenReturn(vm); + + Mockito.when(backupDetailsDao.findDetail(50L, NASBackupChainKeys.CHAIN_ID)) + .thenReturn(new BackupDetailVO(50L, NASBackupChainKeys.CHAIN_ID, "chain-1", true)); + Mockito.when(backupDetailsDao.findDetail(50L, NASBackupChainKeys.CHAIN_POSITION)) + .thenReturn(new BackupDetailVO(50L, NASBackupChainKeys.CHAIN_POSITION, "0", true)); + Mockito.when(backupDetailsDao.findDetail(51L, NASBackupChainKeys.CHAIN_ID)) + .thenReturn(new BackupDetailVO(51L, NASBackupChainKeys.CHAIN_ID, "chain-1", true)); + Mockito.when(backupDetailsDao.findDetail(51L, NASBackupChainKeys.CHAIN_POSITION)) + .thenReturn(new BackupDetailVO(51L, NASBackupChainKeys.CHAIN_POSITION, "1", true)); + Mockito.when(backupDetailsDao.findDetail(52L, NASBackupChainKeys.CHAIN_ID)) + .thenReturn(new BackupDetailVO(52L, NASBackupChainKeys.CHAIN_ID, "chain-1", true)); + Mockito.when(backupDetailsDao.findDetail(52L, NASBackupChainKeys.CHAIN_POSITION)) + .thenReturn(new BackupDetailVO(52L, NASBackupChainKeys.CHAIN_POSITION, "2", true)); + + Mockito.when(backupDao.listByVmId(null, vmId)).thenReturn(List.of(full, inc1, orphan)); + Mockito.when(backupDao.findById(51L)).thenReturn(inc1); + Mockito.doReturn(mock(GlobalLock.class)).when(nasBackupProvider).acquireChainDeleteLock(vmId); + + Mockito.when(agentManager.send(Mockito.anyLong(), Mockito.any(DeleteBackupCommand.class))) + .thenReturn(new BackupAnswer(new DeleteBackupCommand(null, null, null, null), true, "ok")); + + Assert.assertTrue(nasBackupProvider.deleteBackup(inc1, false)); + + // inc1 and the deeper orphan tombstone are physically removed; the live full survives. + Mockito.verify(agentManager, Mockito.times(2)) + .send(Mockito.anyLong(), Mockito.any(DeleteBackupCommand.class)); + Mockito.verify(backupDao).remove(51L); + Mockito.verify(backupDao).remove(52L); + Mockito.verify(backupDao, Mockito.never()).remove(50L); + } + + @Test + public void deletingAncestorOfTombstoneWithLiveDescendantTombstonesIt() + throws AgentUnavailableException, OperationTimedoutException { + Long zoneId = 1L; + Long vmId = 2L; + Long hostId = 3L; + Long offeringId = 4L; + + BackupVO full = new BackupVO(); + full.setVmId(vmId); + full.setBackupOfferingId(offeringId); + full.setExternalId("i-2-2-VM/2026.05.10.10.00.00"); + full.setZoneId(zoneId); + full.setStatus(Backup.Status.BackedUp); + ReflectionTestUtils.setField(full, "id", 50L); + + BackupVO inc1 = new BackupVO(); + inc1.setVmId(vmId); + inc1.setBackupOfferingId(offeringId); + inc1.setExternalId("i-2-2-VM/2026.05.10.10.30.00"); + inc1.setZoneId(zoneId); + inc1.setStatus(Backup.Status.Hidden); + ReflectionTestUtils.setField(inc1, "id", 51L); + + BackupVO inc2 = new BackupVO(); + inc2.setVmId(vmId); + inc2.setBackupOfferingId(offeringId); + inc2.setExternalId("i-2-2-VM/2026.05.10.11.00.00"); + inc2.setZoneId(zoneId); + inc2.setStatus(Backup.Status.BackedUp); + ReflectionTestUtils.setField(inc2, "id", 52L); + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getLastHostId()).thenReturn(hostId); + HostVO host = mock(HostVO.class); + Mockito.when(host.getStatus()).thenReturn(Status.Up); + Mockito.when(hostDao.findById(hostId)).thenReturn(host); + + BackupRepositoryVO repo = new BackupRepositoryVO(1L, "nas", "test-repo", + "nfs", "address", "sync", 1024L, null); + Mockito.when(backupRepositoryDao.findByBackupOfferingId(offeringId)).thenReturn(repo); + Mockito.when(vmInstanceDao.findByIdIncludingRemoved(vmId)).thenReturn(vm); + + Mockito.when(backupDetailsDao.findDetail(50L, NASBackupChainKeys.CHAIN_ID)) + .thenReturn(new BackupDetailVO(50L, NASBackupChainKeys.CHAIN_ID, "chain-1", true)); + Mockito.when(backupDetailsDao.findDetail(50L, NASBackupChainKeys.CHAIN_POSITION)) + .thenReturn(new BackupDetailVO(50L, NASBackupChainKeys.CHAIN_POSITION, "0", true)); + Mockito.when(backupDetailsDao.findDetail(51L, NASBackupChainKeys.CHAIN_ID)) + .thenReturn(new BackupDetailVO(51L, NASBackupChainKeys.CHAIN_ID, "chain-1", true)); + Mockito.when(backupDetailsDao.findDetail(52L, NASBackupChainKeys.CHAIN_ID)) + .thenReturn(new BackupDetailVO(52L, NASBackupChainKeys.CHAIN_ID, "chain-1", true)); + Mockito.when(backupDetailsDao.findDetail(52L, NASBackupChainKeys.CHAIN_POSITION)) + .thenReturn(new BackupDetailVO(52L, NASBackupChainKeys.CHAIN_POSITION, "2", true)); + + Mockito.when(backupDao.listByVmId(null, vmId)).thenReturn(List.of(full, inc1, inc2)); + Mockito.when(backupDao.findById(50L)).thenReturn(full); + Mockito.doReturn(mock(GlobalLock.class)).when(nasBackupProvider).acquireChainDeleteLock(vmId); + + Assert.assertTrue(nasBackupProvider.deleteBackup(full, false)); + + // The full anchor must NOT be physically deleted — inc2 (live) still restores + // through inc1's and full's files. It becomes a tombstone instead. + Mockito.verify(agentManager, Mockito.never()).send(Mockito.anyLong(), Mockito.any(DeleteBackupCommand.class)); + Mockito.verify(backupDao, Mockito.never()).remove(Mockito.anyLong()); + ArgumentCaptor captor = ArgumentCaptor.forClass(BackupVO.class); + Mockito.verify(backupDao).update(Mockito.eq(50L), captor.capture()); + Assert.assertEquals(Backup.Status.Hidden, captor.getValue().getStatus()); + } + + @Test + public void sweepContinuesPastFailedTombstoneDelete() + throws AgentUnavailableException, OperationTimedoutException { + Long zoneId = 1L; + Long vmId = 2L; + Long hostId = 3L; + Long offeringId = 4L; + + BackupVO full = new BackupVO(); + full.setVmId(vmId); + full.setBackupOfferingId(offeringId); + full.setExternalId("i-2-2-VM/2026.05.10.10.00.00"); + full.setZoneId(zoneId); + full.setStatus(Backup.Status.Hidden); + ReflectionTestUtils.setField(full, "id", 50L); + + BackupVO inc1 = new BackupVO(); + inc1.setVmId(vmId); + inc1.setBackupOfferingId(offeringId); + inc1.setExternalId("i-2-2-VM/2026.05.10.10.30.00"); + inc1.setZoneId(zoneId); + inc1.setStatus(Backup.Status.Hidden); + ReflectionTestUtils.setField(inc1, "id", 51L); + + // The last live member — deleting it triggers the sweep of both tombstones. + BackupVO inc2 = new BackupVO(); + inc2.setVmId(vmId); + inc2.setBackupOfferingId(offeringId); + inc2.setExternalId("i-2-2-VM/2026.05.10.11.00.00"); + inc2.setZoneId(zoneId); + inc2.setStatus(Backup.Status.BackedUp); + ReflectionTestUtils.setField(inc2, "id", 52L); + + VMInstanceVO vm = mock(VMInstanceVO.class); + Mockito.when(vm.getLastHostId()).thenReturn(hostId); + HostVO host = mock(HostVO.class); + Mockito.when(host.getStatus()).thenReturn(Status.Up); + Mockito.when(host.getId()).thenReturn(hostId); + Mockito.when(hostDao.findById(hostId)).thenReturn(host); + + BackupRepositoryVO repo = new BackupRepositoryVO(1L, "nas", "test-repo", + "nfs", "address", "sync", 1024L, null); + Mockito.when(backupRepositoryDao.findByBackupOfferingId(offeringId)).thenReturn(repo); + Mockito.when(vmInstanceDao.findByIdIncludingRemoved(vmId)).thenReturn(vm); + + Mockito.when(backupDetailsDao.findDetail(50L, NASBackupChainKeys.CHAIN_ID)) + .thenReturn(new BackupDetailVO(50L, NASBackupChainKeys.CHAIN_ID, "chain-1", true)); + Mockito.when(backupDetailsDao.findDetail(50L, NASBackupChainKeys.CHAIN_POSITION)) + .thenReturn(new BackupDetailVO(50L, NASBackupChainKeys.CHAIN_POSITION, "0", true)); + Mockito.when(backupDetailsDao.findDetail(51L, NASBackupChainKeys.CHAIN_ID)) + .thenReturn(new BackupDetailVO(51L, NASBackupChainKeys.CHAIN_ID, "chain-1", true)); + Mockito.when(backupDetailsDao.findDetail(51L, NASBackupChainKeys.CHAIN_POSITION)) + .thenReturn(new BackupDetailVO(51L, NASBackupChainKeys.CHAIN_POSITION, "1", true)); + Mockito.when(backupDetailsDao.findDetail(52L, NASBackupChainKeys.CHAIN_ID)) + .thenReturn(new BackupDetailVO(52L, NASBackupChainKeys.CHAIN_ID, "chain-1", true)); + Mockito.when(backupDetailsDao.findDetail(52L, NASBackupChainKeys.CHAIN_POSITION)) + .thenReturn(new BackupDetailVO(52L, NASBackupChainKeys.CHAIN_POSITION, "2", true)); + + Mockito.when(backupDao.listByVmId(null, vmId)).thenReturn(List.of(full, inc1, inc2)); + Mockito.when(backupDao.findById(52L)).thenReturn(inc2); + Mockito.doReturn(mock(GlobalLock.class)).when(nasBackupProvider).acquireChainDeleteLock(vmId); + + // Deletes run leaf-to-root: inc2 succeeds, inc1's rm FAILS, full succeeds. + DeleteBackupCommand dummy = new DeleteBackupCommand(null, null, null, null); + Mockito.when(agentManager.send(Mockito.anyLong(), Mockito.any(DeleteBackupCommand.class))) + .thenReturn(new BackupAnswer(dummy, true, "ok"), + new BackupAnswer(dummy, false, "rm failed"), + new BackupAnswer(dummy, true, "ok")); + + Assert.assertTrue(nasBackupProvider.deleteBackup(inc2, false)); + + // All three members were attempted; the failed inc1 keeps its row, full is collected. + Mockito.verify(agentManager, Mockito.times(3)) + .send(Mockito.anyLong(), Mockito.any(DeleteBackupCommand.class)); + Mockito.verify(backupDao).remove(52L); + Mockito.verify(backupDao, Mockito.never()).remove(51L); + Mockito.verify(backupDao).remove(50L); + } } diff --git a/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java b/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java index 25c45ed2a102..395b637d4d47 100644 --- a/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java +++ b/plugins/ca/root-ca/src/main/java/org/apache/cloudstack/ca/provider/RootCAProvider.java @@ -440,7 +440,7 @@ private boolean setupCA() { @Override public boolean start() { managementCertificateCustomSAN = CAManager.CertManagementCustomSubjectAlternativeName.value(); - return loadRootCAKeyPair() && loadRootCAKeyPair() && loadManagementKeyStore(); + return loadRootCAKeyPair() && loadManagementKeyStore(); } @Override diff --git a/plugins/hypervisors/hyperv/src/test/java/com/cloud/hypervisor/hyperv/test/HypervDirectConnectResourceTest.java b/plugins/hypervisors/hyperv/src/test/java/com/cloud/hypervisor/hyperv/test/HypervDirectConnectResourceTest.java index d2e92bd5a2b6..6610985d5841 100644 --- a/plugins/hypervisors/hyperv/src/test/java/com/cloud/hypervisor/hyperv/test/HypervDirectConnectResourceTest.java +++ b/plugins/hypervisors/hyperv/src/test/java/com/cloud/hypervisor/hyperv/test/HypervDirectConnectResourceTest.java @@ -40,6 +40,7 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import com.google.common.io.Files; @@ -248,6 +249,7 @@ public final void testGetVmStatsCommand() { Assert.assertTrue(ans.getDetails(), ans.getResult()); } + @Ignore public final void testStartupCommand() { StartupRoutingCommand defaultStartRoutCmd = new StartupRoutingCommand(0, 0, 0, 0, null, Hypervisor.HypervisorType.Hyperv, RouterPrivateIpStrategy.HostLocal); @@ -278,6 +280,7 @@ public final void testStartupCommand() { } // @Test + @Ignore public final void testJson() { StartupStorageCommand sscmd = null; com.cloud.agent.api.StoragePoolInfo pi = new com.cloud.agent.api.StoragePoolInfo("test123", "192.168.0.1", "c:\\", "c:\\", StoragePoolType.Filesystem, 100L, 50L); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java index da60f6fd7177..26fc096eda40 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/BridgeVifDriver.java @@ -190,9 +190,16 @@ protected boolean isValidProtocolAndVnetId(final String vNetId, final String pro protected String createStorageVnetBridgeIfNeeded(NicTO nic, String trafficLabel, String storageBrName) throws InternalErrorException { - if (!Networks.BroadcastDomainType.Storage.equals(nic.getBroadcastType()) || nic.getBroadcastUri() == null) { + if (nic.getBroadcastUri() == null) { return storageBrName; } + + boolean isStorageBroadcast = Networks.BroadcastDomainType.Storage.equals(nic.getBroadcastType()) || + Networks.BroadcastDomainType.Storage.equals(Networks.BroadcastDomainType.getSchemeValue(nic.getBroadcastUri())); + if (!isStorageBroadcast) { + return storageBrName; + } + String vNetId = Networks.BroadcastDomainType.getValue(nic.getBroadcastUri()); String protocol = Networks.BroadcastDomainType.Vlan.scheme(); if (!isValidProtocolAndVnetId(vNetId, protocol)) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java index 80d9a51cb855..a9ed0c62cab8 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java @@ -17,6 +17,7 @@ package com.cloud.hypervisor.kvm.resource; import static com.cloud.host.Host.HOST_INSTANCE_CONVERSION; +import static com.cloud.host.Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM; import static com.cloud.host.Host.HOST_OVFTOOL_VERSION; import static com.cloud.host.Host.HOST_VDDK_LIB_DIR; import static com.cloud.host.Host.HOST_VDDK_SUPPORT; @@ -4265,6 +4266,7 @@ public StartupCommand[] initialize() { privateIp = cmd.getPrivateIpAddress(); cmd.getHostDetails().putAll(getVersionStrings()); cmd.getHostDetails().put(KeyStoreUtils.SECURED, String.valueOf(isHostSecured()).toLowerCase()); + cmd.getHostDetails().put(HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM, Boolean.TRUE.toString()); cmd.setPool(pool); cmd.setCluster(clusterId); cmd.setGatewayIpAddress(localGateway); @@ -4348,12 +4350,12 @@ protected void calculateHostCpuMaxCapacity(int cpuCores, Long cpuSpeed) { LOGGER.info(String.format("Host uses control group [%s].", output)); if (!CGROUP_V2.equals(output)) { - LOGGER.info(String.format("Setting host CPU max capacity to 0, as it uses cgroup v1.", getHostCpuMaxCapacity())); + LOGGER.info("Setting host CPU max capacity: {} to 0, as it uses cgroup v1.", getHostCpuMaxCapacity()); setHostCpuMaxCapacity(0); return; } - LOGGER.info(String.format("Calculating the max shares of the host.")); + LOGGER.info("Calculating the max shares of the host."); setHostCpuMaxCapacity(cpuCores * cpuSpeed.intValue()); LOGGER.info(String.format("The max shares of the host is [%d].", getHostCpuMaxCapacity())); } @@ -5880,7 +5882,7 @@ public List> cleanVMSnapshotMetadata(Domain dm) } for (String snapshotName: snapshotNames) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug(String.format("Cleaning snapshot [%s] of VM [%s] metadata.", snapshotNames, dm.getName())); + LOGGER.debug("Cleaning snapshot {} of VM {} metadata.", Arrays.toString(snapshotNames), dm.getName()); } DomainSnapshot snapshot = dm.snapshotLookupByName(snapshotName); snapshot.delete(flags); // clean metadata of vm snapshot @@ -6334,6 +6336,15 @@ public String getSnapshotTemporaryPath(String diskPath, String snapshotName) { return String.join(File.separator, diskPathSplitted); } + public String getUefiNvramPath(String vmUuid) { + String nvramDirectory = uefiProperties.getProperty(LibvirtVMDef.GuestDef.GUEST_NVRAM_PATH); + if (StringUtils.isBlank(nvramDirectory) || StringUtils.isBlank(vmUuid)) { + return null; + } + + return nvramDirectory + vmUuid + ".fd"; + } + public static String generateSecretUUIDFromString(String seed) { return UuidUtils.nameUUIDFromBytes(seed.getBytes()).toString(); } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java index 6788516df741..3a42f230f845 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckVolumeCommandWrapper.java @@ -50,7 +50,8 @@ public final class LibvirtCheckVolumeCommandWrapper extends CommandWrapper STORAGE_POOL_TYPES_SUPPORTED = Arrays.asList( Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.NetworkFilesystem, - Storage.StoragePoolType.SharedMountPoint); + Storage.StoragePoolType.SharedMountPoint, + Storage.StoragePoolType.RBD); @Override public Answer execute(final CheckVolumeCommand command, final LibvirtComputingResource libvirtComputingResource) { @@ -64,14 +65,25 @@ public Answer execute(final CheckVolumeCommand command, final LibvirtComputingRe if (STORAGE_POOL_TYPES_SUPPORTED.contains(storageFilerTO.getType())) { final KVMPhysicalDisk vol = pool.getPhysicalDisk(srcFile); final String path = vol.getPath(); - try { - KVMPhysicalDisk.checkQcow2File(path); - } catch (final CloudRuntimeException e) { - return new CheckVolumeAnswer(command, false, "", 0, getVolumeDetails(pool, vol)); + final boolean isRbd = Storage.StoragePoolType.RBD.equals(storageFilerTO.getType()); + + Map volumeDetails = getVolumeDetails(pool, vol); + if (MapUtils.isEmpty(volumeDetails)) { + return new Answer(command, false, "Unable to read the volume on the storage pool"); + } + + if (!isRbd) { + try { + KVMPhysicalDisk.checkQcow2File(path); + } catch (final CloudRuntimeException e) { + return new CheckVolumeAnswer(command, false, "", 0, volumeDetails); + } } - long size = KVMPhysicalDisk.getVirtualSizeFromFile(path); - return new CheckVolumeAnswer(command, true, "", size, getVolumeDetails(pool, vol)); + // Images on RBD are raw and the path is an image name that qemu-img cannot open + // without the rbd: URI, so take the size libvirt already reported for the volume. + long size = isRbd ? vol.getVirtualSize() : KVMPhysicalDisk.getVirtualSizeFromFile(path); + return new CheckVolumeAnswer(command, true, "", size, volumeDetails); } else { return new Answer(command, false, "Unsupported Storage Pool"); } @@ -122,6 +134,9 @@ private Map getDiskFileInfo(KVMStoragePool pool, KVMPhysicalDisk try { QemuImg qemu = new QemuImg(0); QemuImgFile qemuFile = new QemuImgFile(disk.getPath(), disk.getFormat()); + if (Storage.StoragePoolType.RBD.equals(pool.getType())) { + qemuFile = new QemuImgFile(KVMPhysicalDisk.RBDStringBuilder(pool, disk.getPath()), disk.getFormat()); + } return qemu.info(qemuFile, secure); } catch (QemuImgException | LibvirtException ex) { logger.error("Failed to get info of disk file: " + ex.getMessage()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateDiskOnlyVMSnapshotCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateDiskOnlyVMSnapshotCommandWrapper.java index 84d17a1a1161..46a7da70c61f 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateDiskOnlyVMSnapshotCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateDiskOnlyVMSnapshotCommandWrapper.java @@ -19,6 +19,7 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import com.cloud.agent.api.Answer; +import com.cloud.agent.api.FreezeThawVMCommand; import com.cloud.agent.api.VMSnapshotTO; import com.cloud.agent.api.storage.CreateDiskOnlyVmSnapshotAnswer; import com.cloud.agent.api.storage.CreateDiskOnlyVmSnapshotCommand; @@ -32,24 +33,33 @@ import com.cloud.vm.VirtualMachine; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.apache.cloudstack.utils.qemu.QemuCommand; import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImgException; import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.apache.commons.lang3.StringUtils; import org.libvirt.Connect; import org.libvirt.Domain; import org.libvirt.LibvirtException; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; +import com.cloud.storage.Volume; + @ResourceWrapper(handles = CreateDiskOnlyVmSnapshotCommand.class) public class LibvirtCreateDiskOnlyVMSnapshotCommandWrapper extends CommandWrapper { + protected static final String NVRAM_SNAPSHOT_DIR = ".cloudstack-vm-snapshot-nvram"; private static final String SNAPSHOT_XML = "\n" + "%s\n" + @@ -79,6 +89,11 @@ protected Answer takeDiskOnlyVmSnapshotOfRunningVm(CreateDiskOnlyVmSnapshotComma logger.info("Taking disk-only VM snapshot of running VM [{}].", vmName); Domain dm = null; + String nvramSnapshotPath = null; + boolean suspendedByThisWrapper = false; + boolean filesystemsFrozenByThisWrapper = false; + CreateDiskOnlyVmSnapshotAnswer answer = null; + String postSnapshotCleanupIssue = null; try { LibvirtUtilitiesHelper libvirtUtilitiesHelper = resource.getLibvirtUtilitiesHelper(); Connect conn = libvirtUtilitiesHelper.getConnection(); @@ -88,26 +103,45 @@ protected Answer takeDiskOnlyVmSnapshotOfRunningVm(CreateDiskOnlyVmSnapshotComma dm = resource.getDomain(conn, vmName); if (dm == null) { - return new CreateDiskOnlyVmSnapshotAnswer(cmd, false, String.format("Creation of disk-only VM Snapshot failed as we could not find the VM [%s].", vmName), null); + answer = new CreateDiskOnlyVmSnapshotAnswer(cmd, false, + String.format("Creation of disk-only VM Snapshot failed as we could not find the VM [%s].", vmName), null, null); + return answer; } VMSnapshotTO target = cmd.getTarget(); Pair>> snapshotXmlAndVolumeToNewPathMap = createSnapshotXmlAndNewVolumePathMap(volumeObjectTOS, disks, target, resource); + if (shouldFreezeVmFilesystemsForSnapshot(cmd)) { + // The guest-agent freeze flushes guest filesystems; suspend below prevents concurrent UEFI NVRAM writes. + freezeVmFilesystems(dm, vmName); + filesystemsFrozenByThisWrapper = true; + verifyVmFilesystemsFrozen(dm, vmName); + } + if (shouldSuspendVmForSnapshot(cmd)) { + suspendedByThisWrapper = suspendVmIfNeeded(dm); + } + nvramSnapshotPath = backupNvramIfNeeded(cmd, resource); - dm.snapshotCreateXML(snapshotXmlAndVolumeToNewPathMap.first(), getFlagsToUseForRunningVmSnapshotCreation(target)); + dm.snapshotCreateXML(snapshotXmlAndVolumeToNewPathMap.first(), getFlagsToUseForRunningVmSnapshotCreation(target, filesystemsFrozenByThisWrapper)); - return new CreateDiskOnlyVmSnapshotAnswer(cmd, true, null, snapshotXmlAndVolumeToNewPathMap.second()); - } catch (LibvirtException e) { + postSnapshotCleanupIssue = recoverVmAfterSnapshot(dm, vmName, suspendedByThisWrapper, filesystemsFrozenByThisWrapper, postSnapshotCleanupIssue); + filesystemsFrozenByThisWrapper = false; + suspendedByThisWrapper = false; + + answer = new CreateDiskOnlyVmSnapshotAnswer(cmd, true, null, snapshotXmlAndVolumeToNewPathMap.second(), nvramSnapshotPath); + } catch (LibvirtException | IOException e) { String errorMsg = String.format("Creation of disk-only VM snapshot for VM [%s] failed due to %s.", vmName, e.getMessage()); logger.error(errorMsg, e); - if (e.getMessage().contains("QEMU guest agent is not connected")) { + cleanupNvramSnapshotIfNeeded(cmd, resource, nvramSnapshotPath); + if (StringUtils.contains(e.getMessage(), "QEMU guest agent is not connected")) { errorMsg = "QEMU guest agent is not connected. If the VM has been recently started, it might connect soon. Otherwise the VM does not have the" + " guest agent installed; thus the QuiesceVM parameter is not supported."; - return new CreateDiskOnlyVmSnapshotAnswer(cmd, false, errorMsg, null); + answer = new CreateDiskOnlyVmSnapshotAnswer(cmd, false, errorMsg, null, null); + } else { + answer = new CreateDiskOnlyVmSnapshotAnswer(cmd, false, e.getMessage(), null, null); } - return new CreateDiskOnlyVmSnapshotAnswer(cmd, false, e.getMessage(), null); } finally { if (dm != null) { + postSnapshotCleanupIssue = recoverVmAfterSnapshot(dm, vmName, suspendedByThisWrapper, filesystemsFrozenByThisWrapper, postSnapshotCleanupIssue); try { dm.free(); } catch (LibvirtException l) { @@ -115,6 +149,12 @@ protected Answer takeDiskOnlyVmSnapshotOfRunningVm(CreateDiskOnlyVmSnapshotComma } } } + + if (answer != null && StringUtils.isNotBlank(postSnapshotCleanupIssue)) { + answer = new CreateDiskOnlyVmSnapshotAnswer(cmd, answer.getResult(), + appendSnapshotOperationIssue(answer.getDetails(), postSnapshotCleanupIssue), answer.getMapVolumeToSnapshotSizeAndNewVolumePath(), answer.getNvramSnapshotPath()); + } + return answer; } protected Answer takeDiskOnlyVmSnapshotOfStoppedVm(CreateDiskOnlyVmSnapshotCommand cmd, LibvirtComputingResource resource) { @@ -122,10 +162,12 @@ protected Answer takeDiskOnlyVmSnapshotOfStoppedVm(CreateDiskOnlyVmSnapshotComma logger.info("Taking disk-only VM snapshot of stopped VM [{}].", vmName); Map> mapVolumeToSnapshotSizeAndNewVolumePath = new HashMap<>(); + String nvramSnapshotPath = null; List volumeObjectTos = cmd.getVolumeTOs(); KVMStoragePoolManager storagePoolMgr = resource.getStoragePoolMgr(); try { + nvramSnapshotPath = backupNvramIfNeeded(cmd, resource); for (VolumeObjectTO volumeObjectTO : volumeObjectTos) { PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO) volumeObjectTO.getDataStore(); KVMStoragePool kvmStoragePool = storagePoolMgr.getStoragePool(primaryDataStoreTO.getPoolType(), primaryDataStoreTO.getUuid()); @@ -144,7 +186,7 @@ protected Answer takeDiskOnlyVmSnapshotOfStoppedVm(CreateDiskOnlyVmSnapshotComma mapVolumeToSnapshotSizeAndNewVolumePath.put(volumeObjectTO.getUuid(), new Pair<>(getFileSize(currentDeltaFullPath), snapshotPath)); } - } catch (LibvirtException | QemuImgException e) { + } catch (LibvirtException | QemuImgException | IOException e) { logger.error("Exception while creating disk-only VM snapshot for VM [{}]. Deleting leftover deltas.", vmName, e); for (VolumeObjectTO volumeObjectTO : volumeObjectTos) { Pair volSizeAndNewPath = mapVolumeToSnapshotSizeAndNewVolumePath.get(volumeObjectTO.getUuid()); @@ -160,14 +202,15 @@ protected Answer takeDiskOnlyVmSnapshotOfStoppedVm(CreateDiskOnlyVmSnapshotComma logger.warn("Tried to delete leftover snapshot at [{}] failed.", volSizeAndNewPath.second(), ex); } } + cleanupNvramSnapshotIfNeeded(cmd, resource, nvramSnapshotPath); return new Answer(cmd, e); } - return new CreateDiskOnlyVmSnapshotAnswer(cmd, true, null, mapVolumeToSnapshotSizeAndNewVolumePath); + return new CreateDiskOnlyVmSnapshotAnswer(cmd, true, null, mapVolumeToSnapshotSizeAndNewVolumePath, nvramSnapshotPath); } - protected int getFlagsToUseForRunningVmSnapshotCreation(VMSnapshotTO target) { - int flags = target.getQuiescevm() ? Domain.SnapshotCreateFlags.QUIESCE : 0; + protected int getFlagsToUseForRunningVmSnapshotCreation(VMSnapshotTO target, boolean filesystemsFrozenByThisWrapper) { + int flags = target.getQuiescevm() && !filesystemsFrozenByThisWrapper ? Domain.SnapshotCreateFlags.QUIESCE : 0; flags += Domain.SnapshotCreateFlags.DISK_ONLY + Domain.SnapshotCreateFlags.ATOMIC + Domain.SnapshotCreateFlags.NO_METADATA; @@ -195,4 +238,193 @@ protected Pair>> createSnapshotXmlAndNewV protected long getFileSize(String path) { return new File(path).length(); } + + protected String backupNvramIfNeeded(CreateDiskOnlyVmSnapshotCommand cmd, LibvirtComputingResource resource) throws IOException, LibvirtException { + if (!cmd.isUefiEnabled()) { + return null; + } + + String activeNvramPath = resource.getUefiNvramPath(cmd.getVmUuid()); + if (StringUtils.isBlank(activeNvramPath) || !Files.exists(Path.of(activeNvramPath))) { + throw new IOException(String.format("Unable to find the active UEFI NVRAM file for VM [%s].", cmd.getVmName())); + } + + VolumeObjectTO rootVolume = getRootVolume(cmd.getVolumeTOs()); + PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO) rootVolume.getDataStore(); + KVMStoragePool storagePool = resource.getStoragePoolMgr().getStoragePool(primaryDataStoreTO.getPoolType(), primaryDataStoreTO.getUuid()); + + String nvramSnapshotPath = getNvramSnapshotRelativePath(cmd.getTarget().getId()); + Path targetPath = Path.of(storagePool.getLocalPathFor(nvramSnapshotPath)); + Files.createDirectories(targetPath.getParent()); + Files.copy(Path.of(activeNvramPath), targetPath, StandardCopyOption.REPLACE_EXISTING); + return nvramSnapshotPath; + } + + protected void cleanupNvramSnapshotIfNeeded(CreateDiskOnlyVmSnapshotCommand cmd, LibvirtComputingResource resource, String nvramSnapshotPath) { + if (StringUtils.isBlank(nvramSnapshotPath)) { + return; + } + + try { + VolumeObjectTO rootVolume = getRootVolume(cmd.getVolumeTOs()); + PrimaryDataStoreTO primaryDataStoreTO = (PrimaryDataStoreTO) rootVolume.getDataStore(); + KVMStoragePool storagePool = resource.getStoragePoolMgr().getStoragePool(primaryDataStoreTO.getPoolType(), primaryDataStoreTO.getUuid()); + Files.deleteIfExists(Path.of(storagePool.getLocalPathFor(nvramSnapshotPath))); + } catch (Exception e) { + logger.warn("Failed to clean up temporary NVRAM snapshot [{}] for VM [{}].", nvramSnapshotPath, cmd.getVmName(), e); + } + } + + protected String getNvramSnapshotRelativePath(Long vmSnapshotId) { + return String.format("%s/%s.fd", NVRAM_SNAPSHOT_DIR, vmSnapshotId); + } + + protected VolumeObjectTO getRootVolume(List volumeObjectTos) { + return volumeObjectTos.stream() + .filter(volumeObjectTO -> Volume.Type.ROOT.equals(volumeObjectTO.getVolumeType())) + .findFirst() + .orElseThrow(() -> new IllegalStateException("Unable to locate the root volume while handling the VM snapshot.")); + } + + protected boolean shouldSuspendVmForSnapshot(CreateDiskOnlyVmSnapshotCommand cmd) { + return cmd.isUefiEnabled(); + } + + protected boolean shouldFreezeVmFilesystemsForSnapshot(CreateDiskOnlyVmSnapshotCommand cmd) { + return cmd.isUefiEnabled() && cmd.getTarget().getQuiescevm(); + } + + protected boolean suspendVmIfNeeded(Domain domain) throws LibvirtException { + if (domain.getInfo().state == org.libvirt.DomainInfo.DomainState.VIR_DOMAIN_PAUSED) { + return false; + } + + domain.suspend(); + return true; + } + + protected void freezeVmFilesystems(Domain domain, String vmName) throws LibvirtException, IOException { + String result = getResultOfQemuCommand(FreezeThawVMCommand.FREEZE, domain); + if (isQemuAgentErrorResponse(result)) { + throw new IOException(String.format("Failed to freeze VM [%s] filesystems before taking the disk-only VM snapshot. Result: %s", vmName, result)); + } + } + + protected void verifyVmFilesystemsFrozen(Domain domain, String vmName) throws LibvirtException, IOException { + String status = getResultOfQemuCommand(FreezeThawVMCommand.STATUS, domain); + if (StringUtils.isBlank(status)) { + throw new IOException(String.format("Failed to verify VM [%s] filesystem freeze state before taking the disk-only VM snapshot. Result: %s", vmName, status)); + } + + JsonObject statusObject; + try { + JsonElement statusElement = new JsonParser().parse(status); + if (!statusElement.isJsonObject()) { + throw new IOException(String.format("Failed to verify VM [%s] filesystem freeze state before taking the disk-only VM snapshot. Result: %s", vmName, status)); + } + statusObject = statusElement.getAsJsonObject(); + } catch (RuntimeException e) { + throw new IOException(String.format("Failed to verify VM [%s] filesystem freeze state before taking the disk-only VM snapshot. Result: %s", vmName, status), e); + } + + if (statusObject.has("error")) { + throw new IOException(String.format("Failed to verify VM [%s] filesystem freeze state before taking the disk-only VM snapshot. Result: %s", vmName, status)); + } + + JsonElement returnElement = statusObject.get("return"); + if (returnElement == null || !returnElement.isJsonPrimitive() || !returnElement.getAsJsonPrimitive().isString()) { + throw new IOException(String.format("Failed to verify VM [%s] filesystem freeze state before taking the disk-only VM snapshot. Result: %s", vmName, status)); + } + + String statusResult = returnElement.getAsString(); + if (!FreezeThawVMCommand.FREEZE.equals(statusResult)) { + throw new IOException(String.format("Failed to freeze VM [%s] filesystems before taking the disk-only VM snapshot. Status: %s", vmName, statusResult)); + } + } + + protected boolean thawVmFilesystemsIfNeeded(Domain domain, String vmName, boolean filesystemsFrozenByThisWrapper) { + if (!filesystemsFrozenByThisWrapper) { + return true; + } + return thawVmFilesystemsIfNeeded(domain, vmName); + } + + protected boolean thawVmFilesystemsIfNeeded(Domain domain, String vmName) { + try { + String result = getResultOfQemuCommand(FreezeThawVMCommand.THAW, domain); + if (isQemuAgentErrorResponse(result)) { + logger.warn("Failed to thaw VM [{}] filesystems after taking the disk-only VM snapshot. Result: {}", vmName, result); + return false; + } + return true; + } catch (LibvirtException e) { + logger.warn("Failed to thaw VM [{}] filesystems after taking the disk-only VM snapshot.", vmName, e); + return false; + } + } + + protected boolean isQemuAgentErrorResponse(String result) { + if (StringUtils.isBlank(result) || result.startsWith("error")) { + return true; + } + + try { + JsonElement resultElement = new JsonParser().parse(result); + return resultElement.isJsonObject() && resultElement.getAsJsonObject().has("error"); + } catch (RuntimeException e) { + return false; + } + } + + protected String getResultOfQemuCommand(String cmd, Domain domain) throws LibvirtException { + if (cmd.equals(FreezeThawVMCommand.FREEZE)) { + return domain.qemuAgentCommand(QemuCommand.buildQemuCommand(QemuCommand.AGENT_FREEZE, null), 10, 0); + } else if (cmd.equals(FreezeThawVMCommand.THAW)) { + return domain.qemuAgentCommand(QemuCommand.buildQemuCommand(QemuCommand.AGENT_THAW, null), 10, 0); + } else if (cmd.equals(FreezeThawVMCommand.STATUS)) { + return domain.qemuAgentCommand(QemuCommand.buildQemuCommand(QemuCommand.AGENT_FREEZE_STATUS, null), 10, 0); + } + return null; + } + + protected boolean resumeVmIfNeeded(Domain domain, String vmName, boolean suspendedByThisWrapper) { + if (!suspendedByThisWrapper) { + return true; + } + return resumeVmIfNeeded(domain, vmName); + } + + protected boolean resumeVmIfNeeded(Domain domain, String vmName) { + try { + if (domain.getInfo().state == org.libvirt.DomainInfo.DomainState.VIR_DOMAIN_PAUSED) { + domain.resume(); + } + return true; + } catch (LibvirtException e) { + logger.warn("Failed to resume VM [{}] after taking the disk-only VM snapshot.", vmName, e); + return false; + } + } + + protected String recoverVmAfterSnapshot(Domain domain, String vmName, boolean suspendedByThisWrapper, boolean filesystemsFrozenByThisWrapper, String currentIssue) { + if (suspendedByThisWrapper && !resumeVmIfNeeded(domain, vmName)) { + currentIssue = appendSnapshotOperationIssue(currentIssue, + String.format("VM [%s] could not be resumed after taking the disk-only snapshot. Guest may still be paused.", vmName)); + } + if (filesystemsFrozenByThisWrapper && !thawVmFilesystemsIfNeeded(domain, vmName)) { + currentIssue = appendSnapshotOperationIssue(currentIssue, + String.format("VM [%s] filesystems could not be thawed after taking the disk-only snapshot. Guest may still be frozen.", vmName)); + } + return currentIssue; + } + + protected String appendSnapshotOperationIssue(String currentIssue, String newIssue) { + if (StringUtils.isBlank(newIssue)) { + return currentIssue; + } + if (StringUtils.isBlank(currentIssue)) { + return newIssue; + } + return currentIssue + " " + newIssue; + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateVMSnapshotCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateVMSnapshotCommandWrapper.java index de95f785d44b..e3b5c0363f1e 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateVMSnapshotCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCreateVMSnapshotCommandWrapper.java @@ -19,6 +19,7 @@ package com.cloud.hypervisor.kvm.resource.wrapper; +import org.apache.commons.lang.StringEscapeUtils; import org.libvirt.Connect; import org.libvirt.Domain; import org.libvirt.DomainInfo.DomainState; @@ -57,7 +58,7 @@ public Answer execute(final CreateVMSnapshotCommand cmd, final LibvirtComputingR "Create Instance Snapshot Failed due to Instance is not running: " + vmName + " with domainState = " + domainState); } - String vmSnapshotXML = "" + " " + vmSnapshotName + "" + String vmSnapshotXML = "" + " " + StringEscapeUtils.escapeXml(vmSnapshotName) + "" + " " + ""; dm.snapshotCreateXML(vmSnapshotXML); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteDiskOnlyVMSnapshotCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteDiskOnlyVMSnapshotCommandWrapper.java index 15df8627a8a7..62c7c5013374 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteDiskOnlyVMSnapshotCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDeleteDiskOnlyVMSnapshotCommandWrapper.java @@ -27,12 +27,16 @@ import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.apache.commons.lang3.StringUtils; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import com.cloud.storage.Volume; + @ResourceWrapper(handles = DeleteDiskOnlyVmSnapshotCommand.class) public class LibvirtDeleteDiskOnlyVMSnapshotCommandWrapper extends CommandWrapper { @@ -53,6 +57,40 @@ public Answer execute(DeleteDiskOnlyVmSnapshotCommand command, LibvirtComputingR return new Answer(command, e); } } + + deleteNvramSnapshotIfNeeded(command, resource, storagePoolMgr, snapshotsToDelete); return new Answer(command, true, null); } + + protected void deleteNvramSnapshotIfNeeded(DeleteDiskOnlyVmSnapshotCommand command, LibvirtComputingResource resource, KVMStoragePoolManager storagePoolMgr, + List snapshotsToDelete) { + if (StringUtils.isBlank(command.getNvramSnapshotPath())) { + return; + } + + try { + KVMStoragePool storagePool; + if (command.getPrimaryDataStore() != null) { + PrimaryDataStoreTO dataStore = command.getPrimaryDataStore(); + storagePool = storagePoolMgr.getStoragePool(dataStore.getPoolType(), dataStore.getUuid()); + } else { + SnapshotObjectTO rootVolumeSnapshot = snapshotsToDelete.stream() + .map(SnapshotObjectTO.class::cast) + .filter(snapshotObjectTO -> Volume.Type.ROOT.equals(snapshotObjectTO.getVolume().getVolumeType())) + .findFirst() + .orElse(null); + + if (rootVolumeSnapshot == null) { + logger.warn("Unable to locate the root volume snapshot while deleting NVRAM snapshot [{}].", command.getNvramSnapshotPath()); + return; + } + + storagePool = resource.getLibvirtUtilitiesHelper().getPrimaryPoolFromDataTo(rootVolumeSnapshot, storagePoolMgr); + } + + Files.deleteIfExists(Path.of(storagePool.getLocalPathFor(command.getNvramSnapshotPath()))); + } catch (Exception e) { + logger.warn("Failed to delete the UEFI NVRAM snapshot [{}]. It will be left behind on storage.", command.getNvramSnapshotPath(), e); + } + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVncPortCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVncPortCommandWrapper.java index c16c9bcb9a8b..aa628950f925 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVncPortCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtGetVncPortCommandWrapper.java @@ -41,6 +41,7 @@ public Answer execute(final GetVncPortCommand command, final LibvirtComputingRes final Integer vncPort = libvirtComputingResource.getVncPort(conn, command.getName()); return new GetVncPortAnswer(command, libvirtComputingResource.getPrivateIp(), 5900 + vncPort); } catch (final LibvirtException e) { + logger.error("Failed to get vnc port for the vm: {} due to {}", command.getName(), e.getMessage()); return new GetVncPortAnswer(command, e.toString()); } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtManageSnapshotCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtManageSnapshotCommandWrapper.java index ec900e9981e0..53813a0ed1bc 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtManageSnapshotCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtManageSnapshotCommandWrapper.java @@ -22,6 +22,7 @@ import java.io.File; import java.text.MessageFormat; +import org.apache.commons.lang.StringEscapeUtils; import org.libvirt.Connect; import org.libvirt.Domain; import org.libvirt.DomainInfo.DomainState; @@ -79,7 +80,7 @@ public Answer execute(final ManageSnapshotCommand command, final LibvirtComputin + " {1}" + " " + " "); final String vmUuid = vm.getUUIDString(); - final Object[] args = new Object[] {snapshotName, vmUuid}; + final Object[] args = new Object[] {StringEscapeUtils.escapeXml(snapshotName), vmUuid}; final String snapshot = snapshotXML.format(args); logger.debug(snapshot); if (command.getCommandSwitch().equalsIgnoreCase(ManageSnapshotCommand.CREATE_SNAPSHOT)) { diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java index b0950376a93d..acf95edeb954 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapper.java @@ -30,15 +30,29 @@ import com.cloud.resource.CommandWrapper; import com.cloud.resource.ResourceWrapper; import com.cloud.vm.VirtualMachine; +import org.apache.cloudstack.utils.security.ParserUtils; import org.libvirt.Connect; import org.libvirt.Domain; import org.libvirt.LibvirtException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.ParserConfigurationException; +import java.io.IOException; +import java.io.StringReader; +import java.util.HashSet; import java.util.List; +import java.util.Set; @ResourceWrapper(handles = PlugNicCommand.class) public final class LibvirtPlugNicCommandWrapper extends CommandWrapper { + /** Highest PCI slot number on a bus (0x00 is the host bridge). */ + private static final int MAX_PCI_SLOT = 0x1f; @Override public Answer execute(final PlugNicCommand command, final LibvirtComputingResource libvirtComputingResource) { @@ -65,6 +79,15 @@ public Answer execute(final PlugNicCommand command, final LibvirtComputingResour if (command.getDetails() != null) { libvirtComputingResource.setInterfaceDefQueueSettings(command.getDetails(), null, interfaceDef); } + + // Pin the PCI slot to the lowest free one above the existing NICs so the guest sees a + // deterministic, monotonic NIC order across hot-plugs (see findNextAvailablePciSlot). + Integer nextSlot = findNextAvailablePciSlot(vm, pluggedNics); + if (nextSlot != null) { + interfaceDef.setSlot(nextSlot); + logger.debug("Assigning PCI slot 0x" + String.format("%02x", nextSlot) + " to hot-plugged NIC"); + } + vm.attachDevice(interfaceDef.toString()); // apply default network rules on new nic @@ -96,4 +119,86 @@ public Answer execute(final PlugNicCommand command, final LibvirtComputingResour } } } + + /** + * Picks the PCI slot for the NIC being hot-plugged: the lowest free slot above the highest slot + * already used by a NIC. The choice is deterministic and monotonic with respect to the NICs that + * are already present (a new NIC never lands below an existing one), which is what keeps the + * guest's predictable interface names stable across hot-plugs. It does not guarantee contiguity: + * slots between the last NIC and the new one may already be taken by other devices (disks, + * controllers, balloon), in which case the next free slot above them is used. + * + * @return the slot to assign, or {@code null} to let libvirt auto-assign (domain XML unavailable + * or unparseable, or no free slot left). + */ + protected Integer findNextAvailablePciSlot(final Domain vm, final List pluggedNics) { + try { + final String domXml = vm.getXMLDesc(0); + // getXMLDesc can return null on certain libvirt error paths; fall back to libvirt's own choice. + if (domXml == null) { + logger.debug("Domain XML unavailable, letting libvirt auto-assign PCI slot"); + return null; + } + final Set usedSlots = getUsedPciSlots(domXml); + if (usedSlots == null) { + return null; + } + final Integer slot = getFirstFreeSlotAbove(getHighestNicSlot(pluggedNics), usedSlots); + if (slot == null) { + logger.warn("No free PCI slots available, letting libvirt auto-assign"); + } + return slot; + } catch (final LibvirtException e) { + logger.warn("Failed to get domain XML for PCI slot calculation, letting libvirt auto-assign", e); + return null; + } + } + + /** + * Collects the slot numbers of every {@code

} element in the domain XML, + * whichever device or bus they belong to. Returns {@code null} if the XML cannot be parsed. + */ + protected Set getUsedPciSlots(final String domXml) { + final Set usedSlots = new HashSet<>(); + try { + final DocumentBuilder builder = ParserUtils.getSaferDocumentBuilderFactory().newDocumentBuilder(); + final Document doc = builder.parse(new InputSource(new StringReader(domXml))); + final NodeList addresses = doc.getElementsByTagName("address"); + for (int i = 0; i < addresses.getLength(); i++) { + final Element address = (Element) addresses.item(i); + if (!"pci".equals(address.getAttribute("type")) || address.getAttribute("slot").isEmpty()) { + continue; + } + usedSlots.add(Integer.decode(address.getAttribute("slot"))); + } + } catch (final ParserConfigurationException | SAXException | IOException | NumberFormatException e) { + logger.warn("Failed to parse domain XML for PCI slot calculation, letting libvirt auto-assign", e); + return null; + } + return usedSlots; + } + + /** Highest PCI slot used by an existing NIC, or 0 when no NIC carries a slot. */ + protected static int getHighestNicSlot(final List pluggedNics) { + int highest = 0; + for (final InterfaceDef pluggedNic : pluggedNics) { + if (pluggedNic.getSlot() != null && pluggedNic.getSlot() > highest) { + highest = pluggedNic.getSlot(); + } + } + return highest; + } + + /** + * Lowest slot strictly above {@code from} (and no higher than {@link #MAX_PCI_SLOT}) that is not in + * {@code usedSlots}; {@code null} when the range is exhausted. Slot 0 is reserved for the host bridge. + */ + protected static Integer getFirstFreeSlotAbove(final int from, final Set usedSlots) { + for (int slot = from + 1; slot <= MAX_PCI_SLOT; slot++) { + if (!usedSlots.contains(slot)) { + return slot; + } + } + return null; + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java index 5a7d6d2c203a..1ff6d7851f20 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtReadyCommandWrapper.java @@ -43,6 +43,7 @@ public final class LibvirtReadyCommandWrapper extends CommandWrapper hostDetails = new HashMap(); + hostDetails.put(Host.HOST_KVM_DISK_ONLY_VM_SNAPSHOT_NVRAM, Boolean.TRUE.toString()); if (hostSupportsUefi(libvirtComputingResource.isUbuntuOrDebianHost()) && libvirtComputingResource.isUefiPropertiesFileLoaded()) { hostDetails.put(Host.HOST_UEFI_ENABLE, Boolean.TRUE.toString()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java index 9ebb180b055f..d067a29d31a1 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRestoreBackupCommandWrapper.java @@ -20,6 +20,7 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -56,6 +57,11 @@ public class LibvirtRestoreBackupCommandWrapper extends CommandWrapper { private static final String BACKUP_TEMP_FILE_PREFIX = "csbackup"; private static final String FILE_PATH_PLACEHOLDER = "%s/%s"; + // Flattens the backing-file chain into a single self-contained qcow2 written to the + // destination volume path. Used when the source backup is an incremental whose qcow2 + // has a backing reference to its parent (chain set up by nasbackup.sh's qemu-img rebase). + private static final String QEMU_IMG_HAS_BACKING_COMMAND = + "qemu-img info --output=json %s 2>/dev/null | grep -q '\"backing-filename\"'"; private String getVolumeUuidFromPath(String volumePath, PrimaryDataStoreTO volumePool) { if (Storage.StoragePoolType.Linstor.equals(volumePool.getPoolType())) { @@ -98,11 +104,11 @@ public Answer execute(RestoreBackupCommand command, LibvirtComputingResource ser newVolumeId = getVolumeUuidFromPath(volumePath, volumePool); Long size = command.getRestoreVolumeSizes().get(0); restoreVolume(storagePoolMgr, backupPath, volumePool, volumePath, diskType, backupFile, size, - new Pair<>(vmName, command.getVmState()), mountDirectory, timeout); + new Pair<>(vmName, command.getVmState()), mountDirectory, timeout, mountTimeout); } else if (Boolean.TRUE.equals(vmExists)) { - restoreVolumesOfExistingVM(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backedVolumeUUIDs, backupPath, backupFiles, mountDirectory, timeout); + restoreVolumesOfExistingVM(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backedVolumeUUIDs, backupPath, backupFiles, mountDirectory, timeout, mountTimeout); } else { - restoreVolumesOfDestroyedVMs(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backupPath, backupFiles, mountDirectory, timeout); + restoreVolumesOfDestroyedVMs(storagePoolMgr, restoreVolumePools, restoreVolumePaths, backupPath, backupFiles, mountDirectory, timeout, mountTimeout); } } catch (CloudRuntimeException e) { String errorMessage = e.getMessage() != null ? e.getMessage() : ""; @@ -123,9 +129,12 @@ private void verifyBackupFile(String backupPath, String volUuid) { private void restoreVolumesOfExistingVM(KVMStoragePoolManager storagePoolMgr, List restoreVolumePools, List restoreVolumePaths, List backedVolumesUUIDs, - String backupPath, List backupFiles, String mountDirectory, int timeout) { + String backupPath, List backupFiles, String mountDirectory, int timeout, Integer mountTimeout) { String diskType = "root"; try { + // The management server sends the instance's volumes in the order of the backed up volumes, + // each one being the volume that backup has to be written into (it matches them by uuid, see + // NASBackupProvider#getVolumesToRestoreInto), so they are paired by position here. for (int idx = 0; idx < restoreVolumePaths.size(); idx++) { PrimaryDataStoreTO restoreVolumePool = restoreVolumePools.get(idx); String restoreVolumePath = restoreVolumePaths.get(idx); @@ -140,13 +149,13 @@ private void restoreVolumesOfExistingVM(KVMStoragePoolManager storagePoolMgr, Li } } } finally { - unmountBackupDirectory(mountDirectory); + unmountBackupDirectory(mountDirectory, mountTimeout); deleteTemporaryDirectory(mountDirectory); } } private void restoreVolumesOfDestroyedVMs(KVMStoragePoolManager storagePoolMgr, List volumePools, - List volumePaths, String backupPath, List backupFiles, String mountDirectory, int timeout) { + List volumePaths, String backupPath, List backupFiles, String mountDirectory, int timeout, Integer mountTimeout) { String diskType = "root"; try { for (int i = 0; i < volumePaths.size(); i++) { @@ -162,13 +171,13 @@ private void restoreVolumesOfDestroyedVMs(KVMStoragePoolManager storagePoolMgr, } } } finally { - unmountBackupDirectory(mountDirectory); + unmountBackupDirectory(mountDirectory, mountTimeout); deleteTemporaryDirectory(mountDirectory); } } private void restoreVolume(KVMStoragePoolManager storagePoolMgr, String backupPath, PrimaryDataStoreTO volumePool, String volumePath, String diskType, String backupFile, - Long size, Pair vmNameAndState, String mountDirectory, int timeout) { + Long size, Pair vmNameAndState, String mountDirectory, int timeout, Integer mountTimeout) { String bkpPath; String volumeUuid; try { @@ -185,7 +194,7 @@ private void restoreVolume(KVMStoragePoolManager storagePoolMgr, String backupPa } } } finally { - unmountBackupDirectory(mountDirectory); + unmountBackupDirectory(mountDirectory, mountTimeout); deleteTemporaryDirectory(mountDirectory); } } @@ -201,6 +210,7 @@ private String mountBackupDirectory(String backupRepoAddress, String backupRepoT logger.error("Failed to create the tmp mount directory {} for restore", mountDirectory, e); throw new CloudRuntimeException("Failed to create the tmp mount directory for restore on the KVM host"); } + int exitValue; try { String mountPath = Script.getExecutableAbsolutePath("mount"); List mountCmd = new ArrayList<>(); @@ -221,23 +231,42 @@ private String mountBackupDirectory(String backupRepoAddress, String backupRepoT mountCmd.add("-o"); mountCmd.add(mountOptions); } - Script.executeCommand(mountCmd.toArray(new String[0])); + exitValue = Script.executeCommandForExitValue(mountTimeout, mountCmd.toArray(new String[0])); } catch (Exception e) { logger.error("Failed to mount repository {} of type {} to the directory {}", backupRepoAddress, backupRepoType, mountDirectory, e); throw new CloudRuntimeException("Failed to mount the backup repository on the KVM host"); } + if (exitValue != 0) { + logger.error("Failed to mount repository {} of type {} to the directory {}, mount exited with {}", backupRepoAddress, + backupRepoType, mountDirectory, exitValue); + removeTemporaryDirectoryQuietly(mountDirectory); + throw new CloudRuntimeException("Failed to mount the backup repository on the KVM host"); + } return mountDirectory; } - private void unmountBackupDirectory(String backupDirectory) { + private void unmountBackupDirectory(String backupDirectory, Integer mountTimeout) { + int exitValue; try { String umountPath = Script.getExecutableAbsolutePath("umount"); String[] umountCmd = new String[] { "sudo", umountPath, backupDirectory }; - Script.executeCommand(umountCmd); + exitValue = Script.executeCommandForExitValue(mountTimeout, umountCmd); } catch (Exception e) { logger.error("Failed to unmount backup directory {}", backupDirectory, e); throw new CloudRuntimeException("Failed to unmount the backup directory"); } + if (exitValue != 0) { + logger.error("Failed to unmount backup directory {}, umount exited with {}", backupDirectory, exitValue); + throw new CloudRuntimeException("Failed to unmount the backup directory"); + } + } + + private void removeTemporaryDirectoryQuietly(String backupDirectory) { + try { + Files.deleteIfExists(Paths.get(backupDirectory)); + } catch (IOException e) { + logger.warn("Failed to remove the temporary mount directory {} after the mount failed.", backupDirectory, e); + } } private void deleteTemporaryDirectory(String backupDirectory) { @@ -275,11 +304,28 @@ private boolean replaceVolumeWithBackup(KVMStoragePoolManager storagePoolMgr, Pr return replaceBlockDeviceWithBackup(storagePoolMgr, volumePool, volumePath, backupPath, timeout, createTargetVolume, size); } + // For NAS-backed incremental backups, the source qcow2 has a backing-file + // reference to its parent (set by nasbackup.sh's qemu-img rebase). A plain + // rsync would copy only the differential blocks, leaving a volume that + // depends on a backing file the primary storage doesn't have. Flatten the + // chain via qemu-img convert, which follows the backing-file links and + // produces a single self-contained qcow2. + if (hasBackingChain(backupPath)) { + String[] qemuImgCmd = new String[] { Script.getExecutableAbsolutePath("qemu-img"), "convert", "-O", "qcow2", backupPath, volumePath }; + int flattenExit = Script.executeCommandForExitValue(qemuImgCmd); + return flattenExit == 0; + } + String[] rsyncCmd = new String[] { Script.getExecutableAbsolutePath("rsync"), "-az", backupPath, volumePath }; - int exitValue = Script.executeCommandForExitValue(rsyncCmd); + int exitValue = Script.executeCommandForExitValue(timeout, rsyncCmd); return exitValue == 0; } + private boolean hasBackingChain(String qcow2Path) { + return Script.runSimpleBashScriptForExitValue( + String.format(QEMU_IMG_HAS_BACKING_COMMAND, qcow2Path)) == 0; + } + private boolean replaceBlockDeviceWithBackup(KVMStoragePoolManager storagePoolMgr, PrimaryDataStoreTO volumePool, String volumePath, String backupPath, int timeout, boolean createTargetVolume, Long size) { KVMStoragePool volumeStoragePool = storagePoolMgr.getStoragePool(volumePool.getPoolType(), volumePool.getUuid()); QemuImg qemu; @@ -340,38 +386,68 @@ private boolean replaceBlockDeviceWithBackup(KVMStoragePoolManager storagePoolMg private boolean attachVolumeToVm(KVMStoragePoolManager storagePoolMgr, String vmName, PrimaryDataStoreTO volumePool, String volumePath) { String deviceToAttachDiskTo = getDeviceToAttachDisk(vmName); + if (Storage.StoragePoolType.RBD.equals(volumePool.getPoolType())) { + return attachRbdVolumeToVm(storagePoolMgr, vmName, volumePool, volumePath, deviceToAttachDiskTo); + } List virshCmd = new ArrayList<>(); virshCmd.add(Script.getExecutableAbsolutePath("virsh")); - if (volumePool.getPoolType() == Storage.StoragePoolType.RBD) { - String xmlForRbdDisk = getXmlForRbdDisk(storagePoolMgr, volumePool, volumePath, deviceToAttachDiskTo); - logger.debug("RBD disk xml to attach: {}", xmlForRbdDisk); - virshCmd.add("attach-device"); - virshCmd.add(vmName); - virshCmd.add("/dev/stdin"); - virshCmd.add("< result = Script.executePipedCommands(Arrays.asList(domblkCmd, tailCmd, headCmd, awkCmd), 0); - String currentDevice = result.second(); + // executePipedCommands appends a line separator to every line it reads, so the device + // name has to be trimmed before the last character can be incremented. + String currentDevice = result.second() == null ? "" : result.second().trim(); + if (result.first() == null || result.first() != 0 || StringUtils.isBlank(currentDevice)) { + throw new CloudRuntimeException(String.format("Failed to determine the device to attach the restored volume to on VM [%s].", vmName)); + } char lastChar = currentDevice.charAt(currentDevice.length() - 1); char incrementedChar = (char) (lastChar + 1); return currentDevice.substring(0, currentDevice.length() - 1) + incrementedChar; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevertDiskOnlyVMSnapshotCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevertDiskOnlyVMSnapshotCommandWrapper.java index 1aa79d48eec2..e4cd527331bd 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevertDiskOnlyVMSnapshotCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtRevertDiskOnlyVMSnapshotCommandWrapper.java @@ -31,15 +31,20 @@ import org.apache.cloudstack.utils.qemu.QemuImg; import org.apache.cloudstack.utils.qemu.QemuImgException; import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.apache.commons.lang3.StringUtils; import org.libvirt.LibvirtException; import java.io.IOException; +import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import com.cloud.storage.Volume; + @ResourceWrapper(handles = RevertDiskOnlyVmSnapshotCommand.class) public class LibvirtRevertDiskOnlyVMSnapshotCommandWrapper extends CommandWrapper { @@ -55,6 +60,8 @@ public Answer execute(RevertDiskOnlyVmSnapshotCommand cmd, LibvirtComputingResou HashMap snapshotToNewDeltaPath = new HashMap<>(); try { + SnapshotObjectTO rootVolumeSnapshot = getRootVolumeSnapshot(snapshotObjectTos); + validateNvramRevertState(cmd, resource, rootVolumeSnapshot, storagePoolMgr); for (SnapshotObjectTO snapshotObjectTo : snapshotObjectTos) { KVMStoragePool kvmStoragePool = libvirtUtilitiesHelper.getPrimaryPoolFromDataTo(snapshotObjectTo, storagePoolMgr); @@ -71,7 +78,8 @@ public Answer execute(RevertDiskOnlyVmSnapshotCommand cmd, LibvirtComputingResou qemuImg.create(newDelta, currentDelta); snapshotToNewDeltaPath.put(snapshotObjectTo, deltaPath); } - } catch (LibvirtException | QemuImgException e) { + restoreNvramIfNeeded(cmd, resource, rootVolumeSnapshot, storagePoolMgr); + } catch (LibvirtException | QemuImgException | IOException e) { logger.error("Exception while reverting disk-only VM snapshot for VM [{}]. Deleting leftover deltas.", vmName, e); for (SnapshotObjectTO snapshotObjectTo : snapshotObjectTos) { String newPath = snapshotToNewDeltaPath.get(snapshotObjectTo); @@ -108,4 +116,88 @@ public Answer execute(RevertDiskOnlyVmSnapshotCommand cmd, LibvirtComputingResou return new RevertDiskOnlyVmSnapshotAnswer(cmd, volumeObjectTos); } + + protected SnapshotObjectTO getRootVolumeSnapshot(List snapshotObjectTos) { + return snapshotObjectTos.stream() + .filter(snapshotObjectTO -> Volume.Type.ROOT.equals(snapshotObjectTO.getVolume().getVolumeType())) + .findFirst() + .orElse(null); + } + + protected void validateNvramRevertState(RevertDiskOnlyVmSnapshotCommand cmd, LibvirtComputingResource resource, SnapshotObjectTO rootVolumeSnapshot, + KVMStoragePoolManager storagePoolMgr) throws IOException, LibvirtException { + String activeNvramPath = resource.getUefiNvramPath(cmd.getVmUuid()); + if (StringUtils.isBlank(cmd.getNvramSnapshotPath())) { + if (cmd.isUefiEnabled()) { + throw new IOException(String.format("Cannot safely revert disk-only VM snapshot for UEFI VM [%s] because the snapshot does not contain NVRAM state.", + cmd.getVmName())); + } + return; + } + + if (StringUtils.isBlank(activeNvramPath)) { + throw new IOException(String.format("Unable to determine the active UEFI NVRAM path for VM [%s].", cmd.getVmName())); + } + + Path snapshotNvramPath = getNvramSnapshotAbsolutePath(cmd.getNvramSnapshotPath(), rootVolumeSnapshot, resource, storagePoolMgr); + if (!Files.exists(snapshotNvramPath)) { + throw new IOException(String.format("Unable to find the UEFI NVRAM snapshot [%s] for VM [%s].", cmd.getNvramSnapshotPath(), cmd.getVmName())); + } + } + + protected void restoreNvramIfNeeded(RevertDiskOnlyVmSnapshotCommand cmd, LibvirtComputingResource resource, SnapshotObjectTO rootVolumeSnapshot, + KVMStoragePoolManager storagePoolMgr) throws IOException, LibvirtException { + if (StringUtils.isBlank(cmd.getNvramSnapshotPath())) { + return; + } + + String activeNvramPath = resource.getUefiNvramPath(cmd.getVmUuid()); + if (StringUtils.isBlank(activeNvramPath)) { + throw new IOException(String.format("Unable to determine the active UEFI NVRAM path for VM [%s].", cmd.getVmName())); + } + + Path snapshotNvramPath = getNvramSnapshotAbsolutePath(cmd.getNvramSnapshotPath(), rootVolumeSnapshot, resource, storagePoolMgr); + if (!Files.exists(snapshotNvramPath)) { + throw new IOException(String.format("Unable to find the UEFI NVRAM snapshot [%s] for VM [%s].", cmd.getNvramSnapshotPath(), cmd.getVmName())); + } + + replaceNvramAtomically(snapshotNvramPath, Path.of(activeNvramPath)); + } + + protected void replaceNvramAtomically(Path snapshotNvramPath, Path activeNvramPath) throws IOException { + Path targetDirectory = activeNvramPath.getParent(); + if (targetDirectory != null) { + Files.createDirectories(targetDirectory); + } + + Path temporaryNvramPath = Files.createTempFile(targetDirectory, activeNvramPath.getFileName().toString(), ".tmp"); + try { + copyNvramSnapshotToTemporaryPath(snapshotNvramPath, temporaryNvramPath); + moveTemporaryNvramIntoPlace(temporaryNvramPath, activeNvramPath); + } finally { + Files.deleteIfExists(temporaryNvramPath); + } + } + + protected void copyNvramSnapshotToTemporaryPath(Path snapshotNvramPath, Path temporaryNvramPath) throws IOException { + Files.copy(snapshotNvramPath, temporaryNvramPath, StandardCopyOption.REPLACE_EXISTING); + } + + protected void moveTemporaryNvramIntoPlace(Path temporaryNvramPath, Path activeNvramPath) throws IOException { + try { + Files.move(temporaryNvramPath, activeNvramPath, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException e) { + Files.move(temporaryNvramPath, activeNvramPath, StandardCopyOption.REPLACE_EXISTING); + } + } + + protected Path getNvramSnapshotAbsolutePath(String nvramSnapshotPath, SnapshotObjectTO rootVolumeSnapshot, LibvirtComputingResource resource, + KVMStoragePoolManager storagePoolMgr) throws IOException, LibvirtException { + if (rootVolumeSnapshot == null) { + throw new IOException("Unable to locate the root volume snapshot while handling the UEFI NVRAM state."); + } + + KVMStoragePool storagePool = resource.getLibvirtUtilitiesHelper().getPrimaryPoolFromDataTo(rootVolumeSnapshot, storagePoolMgr); + return Path.of(storagePool.getLocalPathFor(nvramSnapshotPath)); + } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java index 42953aa9f835..b7e609587650 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtTakeBackupCommandWrapper.java @@ -19,7 +19,6 @@ package com.cloud.hypervisor.kvm.resource.wrapper; -import com.amazonaws.util.CollectionUtils; import com.cloud.agent.api.Answer; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; import com.cloud.hypervisor.kvm.storage.KVMPhysicalDisk; @@ -33,6 +32,8 @@ import org.apache.cloudstack.backup.BackupAnswer; import org.apache.cloudstack.backup.TakeBackupCommand; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.Arrays; @@ -42,6 +43,16 @@ @ResourceWrapper(handles = TakeBackupCommand.class) public class LibvirtTakeBackupCommandWrapper extends CommandWrapper { private static final Integer EXIT_CLEANUP_FAILED = 20; + // nasbackup.sh prints this on stdout when it could not proceed as an incremental and + // completed a full backup instead; the orchestrator then records the backup as a full. + private static final String INCREMENTAL_FALLBACK_MARKER = "INCREMENTAL_FALLBACK=true"; + + private static final String MODE_FULL = "full"; + private static final String MODE_INCREMENTAL = "incremental"; + // Incremental feature disabled: plain full backup with no QEMU bitmap/checkpoint and no + // chain metadata. Matches nasbackup.sh's "legacy-full" mode (make_checkpoint=0). + private static final String MODE_LEGACY_FULL = "legacy-full"; + @Override public Answer execute(TakeBackupCommand command, LibvirtComputingResource libvirtComputingResource) { final String vmName = command.getVmName(); @@ -54,6 +65,13 @@ public Answer execute(TakeBackupCommand command, LibvirtComputingResource libvir KVMStoragePoolManager storagePoolMgr = libvirtComputingResource.getStoragePoolMgr(); int timeout = command.getWait() > 0 ? command.getWait() * 1000 : libvirtComputingResource.getCmdsTimeout(); + // Pre-validate incremental args here rather than relying on the script to error out. + // Keeps the script agnostic to caller policy (it just does what it's told). + String validationError = validateBackupArgs(command); + if (validationError != null) { + return new BackupAnswer(command, false, validationError); + } + List diskPaths = new ArrayList<>(); if (Objects.nonNull(volumePaths)) { for (int idx = 0; idx < volumePaths.size(); idx++) { @@ -69,8 +87,64 @@ public Answer execute(TakeBackupCommand command, LibvirtComputingResource libvir } } - List commands = new ArrayList<>(); - commands.add(new String[]{ + Pair result = runBackupScript(libvirtComputingResource, command, vmName, backupRepoType, backupRepoAddress, + mountOptions, backupPath, diskPaths, command.getMode(), + command.getBitmapNew(), command.getBitmapParent(), command.getParentPaths(), timeout); + + if (result.first() != 0) { + logger.debug("Failed to take VM backup: " + result.second()); + BackupAnswer answer = new BackupAnswer(command, false, StringUtils.trimToEmpty(result.second())); + if (EXIT_CLEANUP_FAILED.equals(result.first())) { + logger.debug("Backup cleanup failed"); + answer.setNeedsCleanup(true); + } + return answer; + } + + // The script self-heals to a full backup when an incremental can't proceed (e.g. the + // parent checkpoint can't be re-registered) and signals it with INCREMENTAL_FALLBACK + // on stdout. Detect it, then strip the marker line before parsing the backup size. + String rawStdout = result.second(); + boolean incrementalFallback = StringUtils.contains(rawStdout, INCREMENTAL_FALLBACK_MARKER); + String stdout = stripMarkerLines(rawStdout).trim(); + long backupSize = parseBackupSize(stdout, diskPaths); + + BackupAnswer answer = new BackupAnswer(command, true, stdout); + answer.setSize(backupSize); + // A successful run always created command.getBitmapNew() (full and incremental both do; + // it is null for legacy-full, which the orchestrator treats as "no bitmap"). + answer.setBitmapCreated(command.getBitmapNew()); + answer.setIncrementalFallback(incrementalFallback); + return answer; + } + + /** Remove nasbackup.sh's stdout signalling marker lines so they don't pollute size parsing. */ + private String stripMarkerLines(String stdout) { + if (StringUtils.isBlank(stdout)) { + return ""; + } + StringBuilder sb = new StringBuilder(); + for (String line : stdout.split("\n", -1)) { + if (line.contains(INCREMENTAL_FALLBACK_MARKER)) { + continue; + } + if (sb.length() > 0) { + sb.append('\n'); + } + sb.append(line); + } + return sb.toString(); + } + + /** + * Run nasbackup.sh once with the given args. Returns the exit code + captured stdout. + */ + private Pair runBackupScript(LibvirtComputingResource libvirtComputingResource, + TakeBackupCommand command, String vmName, String backupRepoType, String backupRepoAddress, + String mountOptions, String backupPath, List diskPaths, String mode, + String bitmapNew, String bitmapParent, List parentPaths, int timeout) { + boolean quiesce = command.getQuiesce() != null && command.getQuiesce(); + List argv = new ArrayList<>(Arrays.asList( libvirtComputingResource.getNasBackupPath(), "-o", "backup", "-v", vmName, @@ -78,37 +152,86 @@ public Answer execute(TakeBackupCommand command, LibvirtComputingResource libvir "-s", backupRepoAddress, "-m", Objects.nonNull(mountOptions) ? mountOptions : "", "-p", backupPath, - "-q", command.getQuiesce() != null && command.getQuiesce() ? "true" : "false", - "-d", diskPaths.isEmpty() ? "" : String.join(",", diskPaths) - }); + "-q", quiesce ? "true" : "false", + "-d", CollectionUtils.isEmpty(diskPaths) ? "" : String.join(",", diskPaths) + )); + Integer quiesceTimeout = command.getQuiesceTimeout(); + if (quiesce && quiesceTimeout != null && quiesceTimeout > 0) { + argv.add("--quiesce-timeout"); + argv.add(String.valueOf(quiesceTimeout)); + } + if (StringUtils.isNotBlank(mode)) { + argv.add("-M"); + argv.add(mode); + } + if (StringUtils.isNotBlank(bitmapNew)) { + argv.add("--bitmap-new"); + argv.add(bitmapNew); + } + if (StringUtils.isNotBlank(bitmapParent)) { + argv.add("--bitmap-parent"); + argv.add(bitmapParent); + } + if (CollectionUtils.isNotEmpty(parentPaths)) { + argv.add("--parent-paths"); + argv.add(String.join(",", parentPaths)); + } - Pair result = Script.executePipedCommands(commands, timeout); + List commands = new ArrayList<>(); + commands.add(argv.toArray(new String[0])); + return Script.executePipedCommands(commands, timeout); + } - if (result.first() != 0) { - logger.debug("Failed to take VM backup: " + result.second()); - BackupAnswer answer = new BackupAnswer(command, false, result.second().trim()); - if (result.first() == EXIT_CLEANUP_FAILED) { - logger.debug("Backup cleanup failed"); - answer.setNeedsCleanup(true); + /** + * Return a human-readable validation error string, or {@code null} if the command's + * incremental-backup args are internally consistent. + */ + private String validateBackupArgs(TakeBackupCommand command) { + String mode = command.getMode(); + if (StringUtils.isBlank(mode)) { + return null; // legacy full-only, no extra args expected + } + if (MODE_INCREMENTAL.equals(mode)) { + if (StringUtils.isBlank(command.getBitmapNew())) { + return "incremental mode requires bitmapNew"; } - return answer; + if (StringUtils.isBlank(command.getBitmapParent())) { + return "incremental mode requires bitmapParent"; + } + if (CollectionUtils.isEmpty(command.getParentPaths())) { + return "incremental mode requires parentPaths"; + } + return null; + } + if (MODE_FULL.equals(mode)) { + if (StringUtils.isBlank(command.getBitmapNew())) { + return "full mode requires bitmapNew (the bitmap to create for the next incremental)"; + } + return null; } + if (MODE_LEGACY_FULL.equals(mode)) { + return null; // feature-off full backup, no bitmap or chain args expected + } + return "Unknown backup mode: " + mode; + } + /** + * Sum the per-disk size lines emitted by nasbackup.sh. Single-volume mode emits one + * line containing just the byte count; multi-volume mode emits one line per disk + * whose first whitespace-separated token is the byte count. + */ + private long parseBackupSize(String stdout, List diskPaths) { long backupSize = 0L; - if (CollectionUtils.isNullOrEmpty(diskPaths)) { - List outputLines = Arrays.asList(result.second().trim().split("\n")); + if (CollectionUtils.isEmpty(diskPaths)) { + List outputLines = Arrays.asList(stdout.split("\n")); if (!outputLines.isEmpty()) { backupSize = Long.parseLong(outputLines.get(outputLines.size() - 1).trim()); } } else { - String[] outputLines = result.second().trim().split("\n"); - for(String line : outputLines) { + for (String line : stdout.split("\n")) { backupSize = backupSize + Long.parseLong(line.split(" ")[0].trim()); } } - - BackupAnswer answer = new BackupAnswer(command, true, result.second().trim()); - answer.setSize(backupSize); - return answer; + return backupSize; } } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUtilitiesHelper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUtilitiesHelper.java index 70e7f074c879..0a37c971a571 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUtilitiesHelper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtUtilitiesHelper.java @@ -26,6 +26,7 @@ import com.cloud.hypervisor.kvm.storage.KVMStoragePool; import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.commons.lang.StringEscapeUtils; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; import org.libvirt.Connect; @@ -112,9 +113,9 @@ public Script buildScript(final String scriptPath) { } public String generateVMSnapshotXML(VMSnapshotTO snapshot, VMSnapshotTO parent, String domainXmlDesc) { - String parentName = (parent == null)? "": (" " + parent.getSnapshotName() + "\n"); + String parentName = (parent == null)? "": (" " + StringEscapeUtils.escapeXml(parent.getSnapshotName()) + "\n"); String vmSnapshotXML = "\n" - + " " + snapshot.getSnapshotName() + "\n" + + " " + StringEscapeUtils.escapeXml(snapshot.getSnapshotName()) + "\n" + " running\n" + parentName + " " + (int) Math.rint(snapshot.getCreateTime()/1000) + "\n" diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java index 8a9d69c97954..b3b55f484e4d 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMPhysicalDisk.java @@ -54,9 +54,9 @@ public static String RBDStringBuilder(KVMStoragePool storagePool, String image) rbdOpts += ":mon_host=" + composeOptionForMonHosts(monHost, monPort); if (authUserName == null) { - rbdOpts += ":auth_supported=none"; + rbdOpts += ":auth_client_required=none"; } else { - rbdOpts += ":auth_supported=cephx"; + rbdOpts += ":auth_client_required=cephx"; rbdOpts += ":id=" + authUserName; rbdOpts += ":key=" + authSecret; } diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index bc82744dd857..1fba9f3e96f0 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -864,7 +864,7 @@ private Answer createTemplateFromVolumeOrSnapshot(CopyCommand cmd) { if (path == null) { path = srcData.getPath(); if (path == null) { - new CloudRuntimeException("The 'path' or 'iqn' field must be specified."); + throw new CloudRuntimeException("The 'path' or 'iqn' field must be specified."); } } } @@ -2339,33 +2339,36 @@ private CreateObjectAnswer takeClvmVolumeSnapshotOfStoppedVm(KVMPhysicalDisk dis * barriers properly (>2.6.32) this won't be any different then pulling the power * cord out of a running machine. */ - private Long takeRbdVolumeSnapshotOfStoppedVm(KVMStoragePool primaryPool, KVMPhysicalDisk disk, String snapshotName) { + protected Long takeRbdVolumeSnapshotOfStoppedVm(KVMStoragePool primaryPool, KVMPhysicalDisk disk, String snapshotName) { Long snapshotSize = null; + Rados r = null; + IoCTX io = null; + Rbd rbd = null; + RbdImage image = null; try { - Rados r = radosConnect(primaryPool); + r = radosConnect(primaryPool); - final IoCTX io = r.ioCtxCreate(primaryPool.getSourceDir()); - final Rbd rbd = new Rbd(io); - final RbdImage image = rbd.open(disk.getName()); + io = r.ioCtxCreate(primaryPool.getSourceDir()); + rbd = new Rbd(io); + image = rbd.open(disk.getName()); logger.debug("Attempting to create RBD snapshot {}@{}", disk.getName(), snapshotName); image.snapCreate(snapshotName); - image.snapCreate(snapshotName); long rbdSnapshotSize = getRbdSnapshotSize(primaryPool.getSourceDir(), disk.getName(), snapshotName, primaryPool.getSourceHost(), primaryPool.getAuthUserName(), primaryPool.getAuthSecret()); if (rbdSnapshotSize > 0) { snapshotSize = rbdSnapshotSize; } - - rbd.close(image); - r.ioCtxDestroy(io); } catch (final Exception e) { logger.error("A RBD snapshot operation on [{}] failed. The error was: {}", disk.getName(), e.getMessage(), e); + } finally { + closeRbdImage(rbd, image, disk.getName()); + destroyRadosIoCtx(r, io, disk.getName()); } return snapshotSize; } - private long getRbdSnapshotSize(String poolPath, String diskName, String snapshotName, String rbdMonitor, String authUser, String authSecret) { + protected long getRbdSnapshotSize(String poolPath, String diskName, String snapshotName, String rbdMonitor, String authUser, String authSecret) { logger.debug("Get RBD snapshot size for {}/{}@{}", poolPath, diskName, snapshotName); //cmd: rbd du /@ --format json --mon-host --id --key 2>/dev/null String snapshotDetailsInJson = Script.runSimpleBashScript(String.format("rbd du %s/%s@%s --format json --mon-host %s --id %s --key %s 2>/dev/null", poolPath, diskName, snapshotName, rbdMonitor, authUser, authSecret)); @@ -2652,7 +2655,7 @@ protected boolean isAvailablePoolSizeDividedByDiskSizeLesserThanMinRate(long ava return ((availablePoolSize * 1d) / (diskSize * 1d)) < MIN_RATE_BETWEEN_AVAILABLE_POOL_AND_DISK_SIZE_TO_TAKE_DISK_SNAPSHOT; } - private Rados radosConnect(final KVMStoragePool primaryPool) throws RadosException { + protected Rados radosConnect(final KVMStoragePool primaryPool) throws RadosException { Rados r = new Rados(primaryPool.getAuthUserName()); r.confSet(CEPH_MON_HOST, primaryPool.getSourceHost() + ":" + primaryPool.getSourcePort()); r.confSet(CEPH_AUTH_KEY, primaryPool.getAuthSecret()); @@ -2662,6 +2665,50 @@ private Rados radosConnect(final KVMStoragePool primaryPool) throws RadosExcepti return r; } + /** + * Closes an RBD image if it was opened; never throws. An image left open keeps this client's RBD + * exclusive-lock, which later makes 'rbd snap rollback' (revertSnapshot) fail with EROFS and keeps + * the image busy so it cannot be removed. + */ + protected void closeRbdImage(Rbd rbd, RbdImage image, String imageName) { + if (image == null) { + return; + } + try { + rbd.close(image); + } catch (final Exception e) { + logger.warn("Failed to close RBD image [{}]. The error was: {}", imageName, e.getMessage(), e); + } + } + + /** Destroys a RADOS IO context if it was created; never throws. */ + protected void destroyRadosIoCtx(Rados r, IoCTX io, String contextDescription) { + if (io == null) { + return; + } + try { + r.ioCtxDestroy(io); + } catch (final Exception e) { + logger.warn("Failed to destroy the RADOS IO context used for [{}]. The error was: {}", contextDescription, e.getMessage(), e); + } + } + + /** + * Unprotects an RBD snapshot if it was protected; never throws. A snapshot left protected cannot + * be deleted, and neither can its volume. + */ + protected void unprotectRbdSnapshot(RbdImage image, String snapshotName, boolean snapProtected) { + if (!snapProtected) { + return; + } + try { + image.snapUnprotect(snapshotName); + } catch (final Exception e) { + logger.error("Failed to unprotect RBD snapshot [{}]; it and its volume cannot be deleted until this is resolved manually. The error was: {}", + snapshotName, e.getMessage(), e); + } + } + @Override public Answer deleteVolume(final DeleteCommand cmd) { final VolumeObjectTO vol = (VolumeObjectTO)cmd.getData(); @@ -2785,7 +2832,7 @@ private KVMPhysicalDisk createVolumeFromSnapshotOnNFS(CopyCommand cmd, PrimaryDa if (path == null) { path = details != null ? details.get(DiskTO.IQN) : null; if (path == null) { - new CloudRuntimeException("The 'path' or 'iqn' field must be specified."); + logger.warn("The 'path' or 'iqn' field must be specified."); } } } @@ -2811,17 +2858,24 @@ private KVMPhysicalDisk createRBDvolumeFromRBDSnapshot(KVMPhysicalDisk volume, S disk.setSize(size > volume.getVirtualSize() ? size : volume.getVirtualSize()); disk.setVirtualSize(size > volume.getVirtualSize() ? size : disk.getSize()); + Rados r = null; + IoCTX io = null; + Rbd rbd = null; + RbdImage srcImage = null; + RbdImage diskImage = null; + boolean snapProtected = false; + try { - Rados r = new Rados(srcPool.getAuthUserName()); + r = new Rados(srcPool.getAuthUserName()); r.confSet("mon_host", srcPool.getSourceHost() + ":" + srcPool.getSourcePort()); r.confSet("key", srcPool.getAuthSecret()); r.confSet("client_mount_timeout", "30"); r.connect(); - IoCTX io = r.ioCtxCreate(srcPool.getSourceDir()); - Rbd rbd = new Rbd(io); - RbdImage srcImage = rbd.open(volume.getName()); + io = r.ioCtxCreate(srcPool.getSourceDir()); + rbd = new Rbd(io); + srcImage = rbd.open(volume.getName()); List snaps = srcImage.snapList(); boolean snapFound = false; @@ -2837,23 +2891,26 @@ private KVMPhysicalDisk createRBDvolumeFromRBDSnapshot(KVMPhysicalDisk volume, S return null; } srcImage.snapProtect(snapshotName); + snapProtected = true; logger.debug(String.format("Try to clone snapshot %s on RBD", snapshotName)); rbd.clone(volume.getName(), snapshotName, io, disk.getName(), LibvirtStorageAdaptor.RBD_FEATURES, 0); - RbdImage diskImage = rbd.open(disk.getName()); + diskImage = rbd.open(disk.getName()); if (disk.getVirtualSize() > volume.getVirtualSize()) { diskImage.resize(disk.getVirtualSize()); } diskImage.flatten(); - rbd.close(diskImage); - - srcImage.snapUnprotect(snapshotName); - rbd.close(srcImage); - r.ioCtxDestroy(io); } catch (RadosException | RbdException e) { logger.error(String.format("Failed due to %s", e.getMessage()), e); disk = null; + } finally { + // Every handle has to be released on all paths, including the "snapshot not found" return and + // any failure of clone/resize/flatten. + closeRbdImage(rbd, diskImage, newUuid); + unprotectRbdSnapshot(srcImage, snapshotName, snapProtected); + closeRbdImage(rbd, srcImage, volume.getName()); + destroyRadosIoCtx(r, io, snapshotName); } return disk; diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index 6c21065340cd..c949d0882479 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -659,6 +659,19 @@ public KVMStoragePool getStoragePool(String uuid, boolean refreshInfo) { public KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool) { LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool; + /* + * An RBD volume is looked up through librbd rather than through libvirt. + * A volume that was created by another host is not in the libvirt pool + * cache, so looking it up through libvirt misses and forces a refresh of + * the whole pool. Refreshing an RBD pool stats every image in it, so that + * cost grows with the number of volumes in the pool and is paid on every + * VM start. Every other RBD operation in this class already uses librbd + * directly. + */ + if (pool.getType() == StoragePoolType.RBD) { + return getRbdPhysicalDisk(volumeUuid, libvirtPool); + } + try { StorageVol vol = getVolume(libvirtPool.getPool(), volumeUuid); KVMPhysicalDisk disk; @@ -696,6 +709,74 @@ public KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool) { } } + /** + * Looks an RBD volume up directly through librbd. + * + * The size reported by librbd is used for both the size and the virtual size of + * the disk, matching what this class already does after converting an image into + * an RBD volume. + */ + private KVMPhysicalDisk getRbdPhysicalDisk(String volumeUuid, LibvirtStoragePool pool) { + Rados r = new Rados(pool.getAuthUserName()); + try { + r.confSet("mon_host", pool.getSourceHost() + ":" + pool.getSourcePort()); + /* + * The secret is null when the pool has no cephx user, and librados + * aborts the process rather than returning an error if it is handed a + * null value here. + */ + if (pool.getAuthUserName() != null) { + r.confSet("key", pool.getAuthSecret()); + } else { + r.confSet("auth_client_required", "none"); + } + r.confSet("client_mount_timeout", "30"); + r.connect(); + + IoCTX io = r.ioCtxCreate(pool.getSourceDir()); + try { + Rbd rbd = new Rbd(io); + // The image is only stat'ed, so it is opened read only and cannot take the exclusive lock. + RbdImage image = rbd.openReadOnly(volumeUuid); + try { + RbdImageInfo rbdInfo = image.stat(); + KVMPhysicalDisk disk = new KVMPhysicalDisk(pool.getSourceDir() + "/" + volumeUuid, volumeUuid, pool); + disk.setFormat(PhysicalDiskFormat.RAW); + disk.setSize(rbdInfo.size); + disk.setVirtualSize(rbdInfo.size); + return disk; + } finally { + closeRbdImage(rbd, image, volumeUuid); + } + } finally { + r.ioCtxDestroy(io); + } + } catch (RadosException e) { + logger.error("A Ceph RADOS operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage() + + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); + throw new CloudRuntimeException(e.toString(), e); + } catch (RbdException e) { + logger.error("A Ceph RBD operation failed (" + e.getReturnValue() + "). The error was: " + e.getMessage() + + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); + throw new CloudRuntimeException(e.toString(), e); + } finally { + r.shutDown(); + } + } + + /** + * Closes an RBD image without throwing, so that a failure to close cannot discard a + * successful result or hide the exception that is already on its way out. + */ + private void closeRbdImage(Rbd rbd, RbdImage image, String volumeUuid) { + try { + rbd.close(image); + } catch (RbdException e) { + logger.warn("Failed to close RBD image " + volumeUuid + " (" + e.getReturnValue() + "): " + + e.getMessage() + " - " + ErrorCode.getErrorMessage(e.getReturnValue())); + } + } + /** * adjust refcount */ @@ -1348,6 +1429,8 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, */ KVMStoragePool srcPool = template.getPool(); + Map destDetails = destPool.getDetails(); + String dataPool = (destDetails == null) ? null : destDetails.get(KVMPhysicalDisk.RBD_DEFAULT_DATA_POOL); KVMPhysicalDisk disk = null; String newUuid = name; @@ -1396,6 +1479,10 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, r.confSet("mon_host", srcPool.getSourceHost() + ":" + srcPool.getSourcePort()); r.confSet("key", srcPool.getAuthSecret()); r.confSet("client_mount_timeout", "30"); + if (dataPool != null) { + logger.debug("Setting RBD data pool to " + dataPool + " for the new image " + disk.getName()); + r.confSet(KVMPhysicalDisk.RBD_DEFAULT_DATA_POOL, dataPool); + } r.connect(); logger.debug("Successfully connected to Ceph cluster at " + r.confGet("mon_host")); @@ -1474,6 +1561,10 @@ private KVMPhysicalDisk createDiskFromTemplateOnRBD(KVMPhysicalDisk template, rDest.confSet("mon_host", destPool.getSourceHost() + ":" + destPool.getSourcePort()); rDest.confSet("key", destPool.getAuthSecret()); rDest.confSet("client_mount_timeout", "30"); + if (dataPool != null) { + logger.debug("Setting RBD data pool to " + dataPool + " on the destination cluster for the new image " + disk.getName()); + rDest.confSet(KVMPhysicalDisk.RBD_DEFAULT_DATA_POOL, dataPool); + } rDest.connect(); logger.debug("Successfully connected to source Ceph cluster at " + rDest.confGet("mon_host")); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java index 514e412559cc..13dab6942f8b 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/MultipathSCSIAdapterBase.java @@ -228,21 +228,21 @@ private boolean connectPhysicalDisk(AddressInfo address, KVMStoragePool pool, Ma @Override public boolean disconnectPhysicalDisk(String volumePath, KVMStoragePool pool) { - if (LOGGER.isDebugEnabled()) LOGGER.debug(String.format("disconnectPhysicalDisk(volumePath,pool) called with args (%s, %s) START", volumePath, pool.getUuid())); + if (LOGGER.isDebugEnabled()) LOGGER.debug("disconnectPhysicalDisk(volumePath,pool) called with args ({}, {}) START", volumePath, pool.getUuid()); AddressInfo address = this.parseAndValidatePath(volumePath); if (address.getAddress() == null) { - if (LOGGER.isDebugEnabled()) LOGGER.debug(String.format("disconnectPhysicalDisk(volumePath,pool) returning FALSE, volume path has no address field", volumePath, pool.getUuid())); + if (LOGGER.isDebugEnabled()) LOGGER.debug("disconnectPhysicalDisk(volumePath,pool) called with args ({}, {}) returning FALSE, volume path has no address field", volumePath, pool.getUuid()); return false; } ScriptResult result = runScript(disconnectScript, 60000L, address.getAddress().toLowerCase()); if (result.getExitCode() != 0) { - LOGGER.warn(String.format("Disconnect failed for path [%s] with return code [%s]", address.getAddress().toLowerCase(), result.getExitCode())); + LOGGER.warn("Disconnect failed for path {} with return code {}", address.getAddress().toLowerCase(), result.getExitCode()); } if (LOGGER.isDebugEnabled()) { - LOGGER.debug("multipath flush output: " + result.getResult()); - LOGGER.debug(String.format("disconnectPhysicalDisk(volumePath,pool) called with args (%s, %s) COMPLETE [rc=%s]", volumePath, pool.getUuid(), result.getResult())); + LOGGER.debug("multipath flush output: {}", result.getResult()); + LOGGER.debug("disconnectPhysicalDisk(volumePath,pool) called with args ({}, {}) COMPLETE [rc={}]", volumePath, pool.getUuid(), result.getResult()); } return (result.getExitCode() == 0); @@ -250,7 +250,7 @@ public boolean disconnectPhysicalDisk(String volumePath, KVMStoragePool pool) { @Override public boolean disconnectPhysicalDisk(Map volumeToDisconnect) { - LOGGER.debug(String.format("disconnectPhysicalDisk(volumeToDisconnect) called with arg bag [not implemented]:") + " " + volumeToDisconnect); + LOGGER.debug("disconnectPhysicalDisk(volumeToDisconnect) called with arg bag [not implemented]: {}", volumeToDisconnect); return false; } diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java index db665c75cc13..d8e5bed2ef09 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java @@ -140,21 +140,21 @@ private static long getCpuSpeedFromCommandLscpu() { long speed = 0L; LOGGER.info("Fetching CPU speed from command \"lscpu\"."); try { - String command = "lscpu | grep -i 'Model name' | head -n 1 | egrep -o '[[:digit:]].[[:digit:]]+GHz' | sed 's/GHz//g'"; - if(isHostS390x()) { - command = "lscpu | grep 'CPU dynamic MHz' | cut -d ':' -f 2 | tr -d ' ' | awk '{printf \"%.1f\\n\", $1 / 1000}'"; - } + String command = "lscpu | grep -i 'CPU max MHz' | head -n 1 | sed 's/^.*: //' | xargs"; String result = Script.runSimpleBashScript(command); - speed = (long) (Float.parseFloat(result) * 1000); + speed = (long) (Float.parseFloat(result)); LOGGER.info(String.format("Command [%s] resulted in the value [%s] for CPU speed.", command, speed)); return speed; } catch (NullPointerException | NumberFormatException e) { LOGGER.error(String.format("Unable to retrieve the CPU speed from lscpu."), e); } try { - String command = "lscpu | grep -i 'CPU max MHz' | head -n 1 | sed 's/^.*: //' | xargs"; + String command = "lscpu | grep -i 'Model name' | head -n 1 | egrep -o '[[:digit:]].[[:digit:]]+GHz' | sed 's/GHz//g'"; + if(isHostS390x()) { + command = "lscpu | grep 'CPU dynamic MHz' | cut -d ':' -f 2 | tr -d ' ' | awk '{printf \"%.1f\\n\", $1 / 1000}'"; + } String result = Script.runSimpleBashScript(command); - speed = (long) (Float.parseFloat(result)); + speed = (long) (Float.parseFloat(result) * 1000); LOGGER.info(String.format("Command [%s] resulted in the value [%s] for CPU speed.", command, speed)); return speed; } catch (NullPointerException | NumberFormatException e) { diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java index aaebd8c7aebc..fa204f806261 100644 --- a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResourceTest.java @@ -83,6 +83,7 @@ import org.joda.time.Duration; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.libvirt.Connect; @@ -3579,6 +3580,15 @@ public void testPlugNicCommandNoMatchMack() { when(vifDriver.plug(nic, "Other PV", "", null)).thenReturn(interfaceDef); when(interfaceDef.toString()).thenReturn("Interface"); + // Stub vm.getXMLDesc(0) so findNextAvailablePciSlot can scan the domain XML + // for in-use PCI slots. Returning a minimal with a single NIC at + // slot 0x03 exercises the production parser without forcing the production + // code into its null-fallback path. + when(vm.getXMLDesc(0)).thenReturn( + "" + + "
" + + ""); + final String interfaceDefStr = interfaceDef.toString(); doNothing().when(vm).attachDevice(interfaceDefStr); @@ -5642,6 +5652,7 @@ public void testAddExtraConfigComponentNotEmptyExtraConfig() { Mockito.verify(vmDef, times(1)).addComp(any()); } + @Ignore public void validateGetCurrentMemAccordingToMemBallooningWithoutMemBalooning(){ VirtualMachineTO vmTo = Mockito.mock(VirtualMachineTO.class); Mockito.when(vmTo.getType()).thenReturn(Type.User); @@ -5727,6 +5738,7 @@ public void validateCountDomainRunningVcpus() throws LibvirtException{ Assert.assertEquals(valueExpected, result); } + @Ignore public void setDiskIoDriverTestIoUring() { DiskDef diskDef = configureAndTestSetDiskIoDriverTest(HYPERVISOR_LIBVIRT_VERSION_SUPPORTS_IOURING, HYPERVISOR_QEMU_VERSION_SUPPORTS_IOURING); Assert.assertEquals(IoDriverPolicy.IO_URING, diskDef.getIoDriver()); diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDiskOnlyVMSnapshotCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDiskOnlyVMSnapshotCommandWrapperTest.java new file mode 100644 index 000000000000..e447e6f6cfdb --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtDiskOnlyVMSnapshotCommandWrapperTest.java @@ -0,0 +1,816 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.cloud.hypervisor.kvm.resource.wrapper; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +import org.apache.cloudstack.storage.to.PrimaryDataStoreTO; +import org.apache.cloudstack.storage.to.SnapshotObjectTO; +import org.apache.cloudstack.storage.to.VolumeObjectTO; +import org.junit.Test; +import org.libvirt.Domain; +import org.libvirt.DomainInfo; + +import com.cloud.agent.api.VMSnapshotTO; +import com.cloud.agent.api.storage.CreateDiskOnlyVmSnapshotCommand; +import com.cloud.agent.api.storage.CreateDiskOnlyVmSnapshotAnswer; +import com.cloud.agent.api.storage.DeleteDiskOnlyVmSnapshotCommand; +import com.cloud.agent.api.storage.RevertDiskOnlyVmSnapshotCommand; +import com.cloud.agent.api.to.DataTO; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.storage.Storage; +import com.cloud.storage.Volume; +import com.cloud.utils.Pair; + +public class LibvirtDiskOnlyVMSnapshotCommandWrapperTest { + + @Test + public void testBackupNvramIfNeededCopiesActiveNvram() throws Exception { + LibvirtCreateDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtCreateDiskOnlyVMSnapshotCommandWrapper(); + LibvirtComputingResource resource = mock(LibvirtComputingResource.class); + KVMStoragePoolManager storagePoolManager = mock(KVMStoragePoolManager.class); + KVMStoragePool storagePool = mock(KVMStoragePool.class); + CreateDiskOnlyVmSnapshotCommand command = mock(CreateDiskOnlyVmSnapshotCommand.class); + VMSnapshotTO vmSnapshotTO = mock(VMSnapshotTO.class); + PrimaryDataStoreTO dataStoreTO = mock(PrimaryDataStoreTO.class); + VolumeObjectTO rootVolume = mock(VolumeObjectTO.class); + + Path activeNvram = Files.createTempFile("active-", ".fd"); + Files.writeString(activeNvram, "snapshot-nvram"); + Path poolDirectory = Files.createTempDirectory("pool-"); + + when(command.isUefiEnabled()).thenReturn(true); + when(command.getVmUuid()).thenReturn("vm-uuid"); + when(command.getVmName()).thenReturn("vm-name"); + when(command.getTarget()).thenReturn(vmSnapshotTO); + when(vmSnapshotTO.getId()).thenReturn(42L); + when(command.getVolumeTOs()).thenReturn(List.of(rootVolume)); + when(rootVolume.getVolumeType()).thenReturn(Volume.Type.ROOT); + when(rootVolume.getDataStore()).thenReturn(dataStoreTO); + when(dataStoreTO.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + when(dataStoreTO.getUuid()).thenReturn("pool-uuid"); + when(resource.getUefiNvramPath("vm-uuid")).thenReturn(activeNvram.toString()); + when(resource.getStoragePoolMgr()).thenReturn(storagePoolManager); + when(storagePoolManager.getStoragePool(Storage.StoragePoolType.NetworkFilesystem, "pool-uuid")).thenReturn(storagePool); + when(storagePool.getLocalPathFor(anyString())).thenAnswer(invocation -> poolDirectory.resolve(invocation.getArgument(0, String.class)).toString()); + + String nvramSnapshotPath = wrapper.backupNvramIfNeeded(command, resource); + + assertEquals(".cloudstack-vm-snapshot-nvram/42.fd", nvramSnapshotPath); + assertEquals("snapshot-nvram", Files.readString(poolDirectory.resolve(nvramSnapshotPath))); + } + + @Test(expected = IOException.class) + public void testBackupNvramIfNeededFailsWhenUefiNvramIsMissing() throws Exception { + LibvirtCreateDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtCreateDiskOnlyVMSnapshotCommandWrapper(); + LibvirtComputingResource resource = mock(LibvirtComputingResource.class); + CreateDiskOnlyVmSnapshotCommand command = mock(CreateDiskOnlyVmSnapshotCommand.class); + + when(command.isUefiEnabled()).thenReturn(true); + when(command.getVmUuid()).thenReturn("vm-uuid"); + when(command.getVmName()).thenReturn("vm-name"); + when(resource.getUefiNvramPath("vm-uuid")).thenReturn("/tmp/" + UUID.randomUUID() + ".fd"); + + wrapper.backupNvramIfNeeded(command, resource); + } + + @Test(expected = IOException.class) + public void testValidateNvramRevertStateFailsForLegacySnapshotsOnUefiVms() throws Exception { + LibvirtRevertDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtRevertDiskOnlyVMSnapshotCommandWrapper(); + LibvirtComputingResource resource = mock(LibvirtComputingResource.class); + RevertDiskOnlyVmSnapshotCommand command = mock(RevertDiskOnlyVmSnapshotCommand.class); + + Path activeNvram = Files.createTempFile("active-", ".fd"); + when(command.getVmUuid()).thenReturn("vm-uuid"); + when(command.getVmName()).thenReturn("vm-name"); + when(command.isUefiEnabled()).thenReturn(true); + when(command.getNvramSnapshotPath()).thenReturn(null); + when(resource.getUefiNvramPath("vm-uuid")).thenReturn(activeNvram.toString()); + + wrapper.validateNvramRevertState(command, resource, null, mock(KVMStoragePoolManager.class)); + } + + @Test(expected = IOException.class) + public void testValidateNvramRevertStateFailsForLegacySnapshotsOnFallbackHostsForUefiVms() throws Exception { + LibvirtRevertDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtRevertDiskOnlyVMSnapshotCommandWrapper(); + LibvirtComputingResource resource = mock(LibvirtComputingResource.class); + RevertDiskOnlyVmSnapshotCommand command = mock(RevertDiskOnlyVmSnapshotCommand.class); + + when(command.getVmUuid()).thenReturn("vm-uuid"); + when(command.getVmName()).thenReturn("vm-name"); + when(command.isUefiEnabled()).thenReturn(true); + when(command.getNvramSnapshotPath()).thenReturn(null); + when(resource.getUefiNvramPath("vm-uuid")).thenReturn(Path.of("/tmp", UUID.randomUUID() + ".fd").toString()); + + wrapper.validateNvramRevertState(command, resource, null, mock(KVMStoragePoolManager.class)); + } + + @Test + public void testValidateNvramRevertStateAllowsFallbackHostsWithoutLocalNvram() throws Exception { + LibvirtRevertDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtRevertDiskOnlyVMSnapshotCommandWrapper(); + LibvirtComputingResource resource = mock(LibvirtComputingResource.class); + LibvirtUtilitiesHelper libvirtUtilitiesHelper = mock(LibvirtUtilitiesHelper.class); + KVMStoragePoolManager storagePoolManager = mock(KVMStoragePoolManager.class); + KVMStoragePool storagePool = mock(KVMStoragePool.class); + RevertDiskOnlyVmSnapshotCommand command = mock(RevertDiskOnlyVmSnapshotCommand.class); + SnapshotObjectTO rootSnapshot = mock(SnapshotObjectTO.class); + + Path poolDirectory = Files.createTempDirectory("pool-"); + Path snapshotNvram = poolDirectory.resolve("nvram/42.fd"); + Files.createDirectories(snapshotNvram.getParent()); + Files.writeString(snapshotNvram, "snapshot"); + + when(command.getVmUuid()).thenReturn("vm-uuid"); + when(command.getVmName()).thenReturn("vm-name"); + when(command.isUefiEnabled()).thenReturn(true); + when(command.getNvramSnapshotPath()).thenReturn("nvram/42.fd"); + when(resource.getUefiNvramPath("vm-uuid")).thenReturn(poolDirectory.resolve("missing").resolve("vm-uuid.fd").toString()); + when(resource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); + when(libvirtUtilitiesHelper.getPrimaryPoolFromDataTo(rootSnapshot, storagePoolManager)).thenReturn(storagePool); + when(storagePool.getLocalPathFor("nvram/42.fd")).thenReturn(snapshotNvram.toString()); + + wrapper.validateNvramRevertState(command, resource, rootSnapshot, storagePoolManager); + } + + @Test + public void testRestoreNvramIfNeededRestoresSnapshotBytes() throws Exception { + LibvirtRevertDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtRevertDiskOnlyVMSnapshotCommandWrapper(); + LibvirtComputingResource resource = mock(LibvirtComputingResource.class); + LibvirtUtilitiesHelper libvirtUtilitiesHelper = mock(LibvirtUtilitiesHelper.class); + KVMStoragePoolManager storagePoolManager = mock(KVMStoragePoolManager.class); + KVMStoragePool storagePool = mock(KVMStoragePool.class); + RevertDiskOnlyVmSnapshotCommand command = mock(RevertDiskOnlyVmSnapshotCommand.class); + SnapshotObjectTO rootSnapshot = mock(SnapshotObjectTO.class); + + Path activeNvram = Files.createTempFile("active-", ".fd"); + Files.writeString(activeNvram, "current"); + Path poolDirectory = Files.createTempDirectory("pool-"); + Path snapshotNvram = poolDirectory.resolve("nvram/42.fd"); + Files.createDirectories(snapshotNvram.getParent()); + Files.writeString(snapshotNvram, "snapshot"); + + when(command.getVmUuid()).thenReturn("vm-uuid"); + when(command.getVmName()).thenReturn("vm-name"); + when(command.getNvramSnapshotPath()).thenReturn("nvram/42.fd"); + when(resource.getUefiNvramPath("vm-uuid")).thenReturn(activeNvram.toString()); + when(resource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); + when(libvirtUtilitiesHelper.getPrimaryPoolFromDataTo(rootSnapshot, storagePoolManager)).thenReturn(storagePool); + when(storagePool.getLocalPathFor("nvram/42.fd")).thenReturn(snapshotNvram.toString()); + + wrapper.restoreNvramIfNeeded(command, resource, rootSnapshot, storagePoolManager); + + assertEquals("snapshot", Files.readString(activeNvram)); + } + + @Test + public void testRestoreNvramIfNeededCreatesMissingActiveNvramFile() throws Exception { + LibvirtRevertDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtRevertDiskOnlyVMSnapshotCommandWrapper(); + LibvirtComputingResource resource = mock(LibvirtComputingResource.class); + LibvirtUtilitiesHelper libvirtUtilitiesHelper = mock(LibvirtUtilitiesHelper.class); + KVMStoragePoolManager storagePoolManager = mock(KVMStoragePoolManager.class); + KVMStoragePool storagePool = mock(KVMStoragePool.class); + RevertDiskOnlyVmSnapshotCommand command = mock(RevertDiskOnlyVmSnapshotCommand.class); + SnapshotObjectTO rootSnapshot = mock(SnapshotObjectTO.class); + + Path poolDirectory = Files.createTempDirectory("pool-"); + Path snapshotNvram = poolDirectory.resolve("nvram/42.fd"); + Path activeNvram = poolDirectory.resolve("target").resolve("vm-uuid.fd"); + Files.createDirectories(snapshotNvram.getParent()); + Files.writeString(snapshotNvram, "snapshot"); + + when(command.getVmUuid()).thenReturn("vm-uuid"); + when(command.getVmName()).thenReturn("vm-name"); + when(command.getNvramSnapshotPath()).thenReturn("nvram/42.fd"); + when(resource.getUefiNvramPath("vm-uuid")).thenReturn(activeNvram.toString()); + when(resource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); + when(libvirtUtilitiesHelper.getPrimaryPoolFromDataTo(rootSnapshot, storagePoolManager)).thenReturn(storagePool); + when(storagePool.getLocalPathFor("nvram/42.fd")).thenReturn(snapshotNvram.toString()); + + wrapper.restoreNvramIfNeeded(command, resource, rootSnapshot, storagePoolManager); + + assertEquals("snapshot", Files.readString(activeNvram)); + } + + @Test + public void testRestoreNvramIfNeededPreservesActiveNvramWhenCopyFails() throws Exception { + LibvirtRevertDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtRevertDiskOnlyVMSnapshotCommandWrapper() { + @Override + protected void copyNvramSnapshotToTemporaryPath(Path snapshotNvramPath, Path temporaryNvramPath) throws IOException { + Files.writeString(temporaryNvramPath, "partial"); + throw new IOException("copy failed"); + } + }; + LibvirtComputingResource resource = mock(LibvirtComputingResource.class); + LibvirtUtilitiesHelper libvirtUtilitiesHelper = mock(LibvirtUtilitiesHelper.class); + KVMStoragePoolManager storagePoolManager = mock(KVMStoragePoolManager.class); + KVMStoragePool storagePool = mock(KVMStoragePool.class); + RevertDiskOnlyVmSnapshotCommand command = mock(RevertDiskOnlyVmSnapshotCommand.class); + SnapshotObjectTO rootSnapshot = mock(SnapshotObjectTO.class); + + Path activeNvram = Files.createTempFile("active-", ".fd"); + Files.writeString(activeNvram, "current"); + Path poolDirectory = Files.createTempDirectory("pool-"); + Path snapshotNvram = poolDirectory.resolve("nvram/42.fd"); + Files.createDirectories(snapshotNvram.getParent()); + Files.writeString(snapshotNvram, "snapshot"); + + when(command.getVmUuid()).thenReturn("vm-uuid"); + when(command.getVmName()).thenReturn("vm-name"); + when(command.getNvramSnapshotPath()).thenReturn("nvram/42.fd"); + when(resource.getUefiNvramPath("vm-uuid")).thenReturn(activeNvram.toString()); + when(resource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); + when(libvirtUtilitiesHelper.getPrimaryPoolFromDataTo(rootSnapshot, storagePoolManager)).thenReturn(storagePool); + when(storagePool.getLocalPathFor("nvram/42.fd")).thenReturn(snapshotNvram.toString()); + + try { + wrapper.restoreNvramIfNeeded(command, resource, rootSnapshot, storagePoolManager); + fail("Expected restore to fail when the snapshot copy fails."); + } catch (IOException expected) { + assertEquals("current", Files.readString(activeNvram)); + } + } + + @Test + public void testDeleteNvramSnapshotIfNeededDeletesSidecar() throws Exception { + LibvirtDeleteDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtDeleteDiskOnlyVMSnapshotCommandWrapper(); + LibvirtComputingResource resource = mock(LibvirtComputingResource.class); + LibvirtUtilitiesHelper libvirtUtilitiesHelper = mock(LibvirtUtilitiesHelper.class); + KVMStoragePoolManager storagePoolManager = mock(KVMStoragePoolManager.class); + KVMStoragePool storagePool = mock(KVMStoragePool.class); + DeleteDiskOnlyVmSnapshotCommand command = mock(DeleteDiskOnlyVmSnapshotCommand.class); + SnapshotObjectTO rootSnapshot = mock(SnapshotObjectTO.class); + VolumeObjectTO rootVolume = mock(VolumeObjectTO.class); + + Path poolDirectory = Files.createTempDirectory("pool-"); + Path snapshotNvram = poolDirectory.resolve("nvram/42.fd"); + Files.createDirectories(snapshotNvram.getParent()); + Files.writeString(snapshotNvram, "snapshot"); + + when(command.getNvramSnapshotPath()).thenReturn("nvram/42.fd"); + when(rootSnapshot.getVolume()).thenReturn(rootVolume); + when(rootVolume.getVolumeType()).thenReturn(Volume.Type.ROOT); + when(resource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); + when(libvirtUtilitiesHelper.getPrimaryPoolFromDataTo(rootSnapshot, storagePoolManager)).thenReturn(storagePool); + when(storagePool.getLocalPathFor("nvram/42.fd")).thenReturn(snapshotNvram.toString()); + + wrapper.deleteNvramSnapshotIfNeeded(command, resource, storagePoolManager, List.of(rootSnapshot)); + + assertFalse(Files.exists(snapshotNvram)); + assertTrue(Files.exists(poolDirectory)); + } + + @Test + public void testDeleteNvramSnapshotIfNeededDeletesSidecarUsingPrimaryDataStore() throws Exception { + LibvirtDeleteDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtDeleteDiskOnlyVMSnapshotCommandWrapper(); + LibvirtComputingResource resource = mock(LibvirtComputingResource.class); + KVMStoragePoolManager storagePoolManager = mock(KVMStoragePoolManager.class); + KVMStoragePool storagePool = mock(KVMStoragePool.class); + DeleteDiskOnlyVmSnapshotCommand command = mock(DeleteDiskOnlyVmSnapshotCommand.class); + PrimaryDataStoreTO primaryDataStoreTO = mock(PrimaryDataStoreTO.class); + + Path poolDirectory = Files.createTempDirectory("pool-"); + Path snapshotNvram = poolDirectory.resolve("nvram/42.fd"); + Files.createDirectories(snapshotNvram.getParent()); + Files.writeString(snapshotNvram, "snapshot"); + + when(command.getNvramSnapshotPath()).thenReturn("nvram/42.fd"); + when(command.getPrimaryDataStore()).thenReturn(primaryDataStoreTO); + when(primaryDataStoreTO.getPoolType()).thenReturn(Storage.StoragePoolType.NetworkFilesystem); + when(primaryDataStoreTO.getUuid()).thenReturn("pool-uuid"); + when(storagePoolManager.getStoragePool(Storage.StoragePoolType.NetworkFilesystem, "pool-uuid")).thenReturn(storagePool); + when(storagePool.getLocalPathFor("nvram/42.fd")).thenReturn(snapshotNvram.toString()); + + wrapper.deleteNvramSnapshotIfNeeded(command, resource, storagePoolManager, List.of()); + + assertFalse(Files.exists(snapshotNvram)); + } + + @Test + public void testResumeVmIfNeededOnlyResumesWhenWrapperSuspendedVm() throws Exception { + LibvirtCreateDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtCreateDiskOnlyVMSnapshotCommandWrapper(); + Domain domain = mock(Domain.class); + + wrapper.resumeVmIfNeeded(domain, "vm-name", false); + + verify(domain, never()).resume(); + verify(domain, never()).getInfo(); + } + + @Test + public void testSuspendVmIfNeededSkipsAlreadyPausedVm() throws Exception { + LibvirtCreateDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtCreateDiskOnlyVMSnapshotCommandWrapper(); + Domain domain = mock(Domain.class); + DomainInfo domainInfo = new DomainInfo(); + domainInfo.state = DomainInfo.DomainState.VIR_DOMAIN_PAUSED; + when(domain.getInfo()).thenReturn(domainInfo); + + assertFalse(wrapper.suspendVmIfNeeded(domain)); + verify(domain, never()).suspend(); + } + + @Test + public void testShouldSuspendVmForSnapshotWhenUefiAndNotQuiesced() { + LibvirtCreateDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtCreateDiskOnlyVMSnapshotCommandWrapper(); + CreateDiskOnlyVmSnapshotCommand command = mock(CreateDiskOnlyVmSnapshotCommand.class); + VMSnapshotTO target = mock(VMSnapshotTO.class); + + when(command.isUefiEnabled()).thenReturn(true); + when(command.getTarget()).thenReturn(target); + when(target.getQuiescevm()).thenReturn(false); + + assertTrue(wrapper.shouldSuspendVmForSnapshot(command)); + } + + @Test + public void testShouldSuspendVmForSnapshotWhenQuiesceIsRequested() { + LibvirtCreateDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtCreateDiskOnlyVMSnapshotCommandWrapper(); + CreateDiskOnlyVmSnapshotCommand command = mock(CreateDiskOnlyVmSnapshotCommand.class); + VMSnapshotTO target = mock(VMSnapshotTO.class); + + when(command.isUefiEnabled()).thenReturn(true); + when(command.getTarget()).thenReturn(target); + when(target.getQuiescevm()).thenReturn(true); + + assertTrue(wrapper.shouldSuspendVmForSnapshot(command)); + } + + @Test + public void testShouldFreezeVmFilesystemsForSnapshotWhenQuiesceIsRequested() { + LibvirtCreateDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtCreateDiskOnlyVMSnapshotCommandWrapper(); + CreateDiskOnlyVmSnapshotCommand command = mock(CreateDiskOnlyVmSnapshotCommand.class); + VMSnapshotTO target = mock(VMSnapshotTO.class); + + when(command.isUefiEnabled()).thenReturn(true); + when(command.getTarget()).thenReturn(target); + when(target.getQuiescevm()).thenReturn(true); + + assertTrue(wrapper.shouldFreezeVmFilesystemsForSnapshot(command)); + } + + @Test + public void testGetFlagsToUseForRunningVmSnapshotCreationOmitsLibvirtQuiesceWhenAlreadyFrozen() { + LibvirtCreateDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtCreateDiskOnlyVMSnapshotCommandWrapper(); + VMSnapshotTO target = mock(VMSnapshotTO.class); + + when(target.getQuiescevm()).thenReturn(true); + + int flags = wrapper.getFlagsToUseForRunningVmSnapshotCreation(target, true); + + assertEquals(0, flags & Domain.SnapshotCreateFlags.QUIESCE); + assertTrue((flags & Domain.SnapshotCreateFlags.DISK_ONLY) != 0); + assertTrue((flags & Domain.SnapshotCreateFlags.ATOMIC) != 0); + assertTrue((flags & Domain.SnapshotCreateFlags.NO_METADATA) != 0); + } + + @Test + public void testFreezeAndVerifyVmFilesystemsSucceedsWhenGuestAgentReportsFrozen() throws Exception { + LibvirtCreateDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtCreateDiskOnlyVMSnapshotCommandWrapper() { + @Override + protected String getResultOfQemuCommand(String cmd, Domain domain) { + if ("status".equals(cmd)) { + return "{\"return\":\"frozen\"}"; + } + return "{\"return\":0}"; + } + }; + + Domain domain = mock(Domain.class); + wrapper.freezeVmFilesystems(domain, "vm-name"); + wrapper.verifyVmFilesystemsFrozen(domain, "vm-name"); + } + + @Test + public void testFreezeVmFilesystemsFailsForQemuErrorResponse() throws Exception { + LibvirtCreateDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtCreateDiskOnlyVMSnapshotCommandWrapper() { + @Override + protected String getResultOfQemuCommand(String cmd, Domain domain) { + return "{\"error\":{\"class\":\"GenericError\",\"desc\":\"guest agent failure\"}}"; + } + }; + + try { + wrapper.freezeVmFilesystems(mock(Domain.class), "vm-name"); + fail("QEMU guest agent error responses must be treated as freeze failures."); + } catch (IOException e) { + assertTrue(e.getMessage().contains("Failed to freeze VM [vm-name] filesystems")); + } + } + + @Test + public void testThawVmFilesystemsFailsForQemuErrorResponse() { + LibvirtCreateDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtCreateDiskOnlyVMSnapshotCommandWrapper() { + @Override + protected String getResultOfQemuCommand(String cmd, Domain domain) { + return "{\"error\":{\"class\":\"GenericError\",\"desc\":\"guest agent failure\"}}"; + } + }; + + assertFalse(wrapper.thawVmFilesystemsIfNeeded(mock(Domain.class), "vm-name")); + } + + @Test + public void testTakeDiskOnlyVmSnapshotOfRunningVmThawsWhenFreezeVerificationFails() throws Exception { + final boolean[] thawCalled = {false}; + LibvirtCreateDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtCreateDiskOnlyVMSnapshotCommandWrapper() { + @Override + protected Pair>> createSnapshotXmlAndNewVolumePathMap(List volumeObjectTOS, + List disks, VMSnapshotTO target, LibvirtComputingResource resource) { + return new Pair<>("", Collections.emptyMap()); + } + + @Override + protected boolean shouldSuspendVmForSnapshot(CreateDiskOnlyVmSnapshotCommand cmd) { + return false; + } + + @Override + protected boolean shouldFreezeVmFilesystemsForSnapshot(CreateDiskOnlyVmSnapshotCommand cmd) { + return true; + } + + @Override + protected void freezeVmFilesystems(Domain domain, String vmName) { + } + + @Override + protected void verifyVmFilesystemsFrozen(Domain domain, String vmName) throws IOException { + throw new IOException("status verification failed"); + } + + @Override + protected boolean thawVmFilesystemsIfNeeded(Domain domain, String vmName) { + thawCalled[0] = true; + return true; + } + }; + LibvirtComputingResource resource = mock(LibvirtComputingResource.class); + LibvirtUtilitiesHelper libvirtUtilitiesHelper = mock(LibvirtUtilitiesHelper.class); + org.libvirt.Connect connect = mock(org.libvirt.Connect.class); + Domain domain = mock(Domain.class); + CreateDiskOnlyVmSnapshotCommand command = mock(CreateDiskOnlyVmSnapshotCommand.class); + VMSnapshotTO target = mock(VMSnapshotTO.class); + + when(command.getVmName()).thenReturn("vm-name"); + when(command.getVolumeTOs()).thenReturn(List.of()); + when(command.getTarget()).thenReturn(target); + when(resource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); + when(libvirtUtilitiesHelper.getConnection()).thenReturn(connect); + when(resource.getDisks(connect, "vm-name")).thenReturn(List.of()); + when(resource.getDomain(connect, "vm-name")).thenReturn(domain); + + CreateDiskOnlyVmSnapshotAnswer answer = (CreateDiskOnlyVmSnapshotAnswer) wrapper.takeDiskOnlyVmSnapshotOfRunningVm(command, resource); + + assertFalse(answer.getResult()); + assertTrue("Thaw must be attempted after a successful freeze followed by verification failure", thawCalled[0]); + } + + @Test + public void testVerifyVmFilesystemsFrozenFailsForQemuErrorResponse() throws Exception { + LibvirtCreateDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtCreateDiskOnlyVMSnapshotCommandWrapper() { + @Override + protected String getResultOfQemuCommand(String cmd, Domain domain) { + return "{\"error\":{\"class\":\"GenericError\",\"desc\":\"guest agent failure\"}}"; + } + }; + + try { + wrapper.verifyVmFilesystemsFrozen(mock(Domain.class), "vm-name"); + fail("QEMU guest agent error responses must be treated as IO failures."); + } catch (IOException e) { + assertTrue(e.getMessage().contains("Failed to verify VM [vm-name] filesystem freeze state")); + } + } + + @Test + public void testTakeDiskOnlyVmSnapshotOfRunningVmReturnsFailureAnswerWhenFreezeStatusJsonIsMalformed() throws Exception { + final boolean[] thawCalled = {false}; + LibvirtCreateDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtCreateDiskOnlyVMSnapshotCommandWrapper() { + @Override + protected Pair>> createSnapshotXmlAndNewVolumePathMap(List volumeObjectTOS, + List disks, VMSnapshotTO target, LibvirtComputingResource resource) { + return new Pair<>("", Collections.emptyMap()); + } + + @Override + protected boolean shouldSuspendVmForSnapshot(CreateDiskOnlyVmSnapshotCommand cmd) { + return false; + } + + @Override + protected boolean shouldFreezeVmFilesystemsForSnapshot(CreateDiskOnlyVmSnapshotCommand cmd) { + return true; + } + + @Override + protected void freezeVmFilesystems(Domain domain, String vmName) { + } + + @Override + protected String getResultOfQemuCommand(String cmd, Domain domain) { + return "not-json"; + } + + @Override + protected boolean thawVmFilesystemsIfNeeded(Domain domain, String vmName) { + thawCalled[0] = true; + return true; + } + }; + LibvirtComputingResource resource = mock(LibvirtComputingResource.class); + LibvirtUtilitiesHelper libvirtUtilitiesHelper = mock(LibvirtUtilitiesHelper.class); + org.libvirt.Connect connect = mock(org.libvirt.Connect.class); + Domain domain = mock(Domain.class); + CreateDiskOnlyVmSnapshotCommand command = mock(CreateDiskOnlyVmSnapshotCommand.class); + VMSnapshotTO target = mock(VMSnapshotTO.class); + + when(command.getVmName()).thenReturn("vm-name"); + when(command.getVolumeTOs()).thenReturn(List.of()); + when(command.getTarget()).thenReturn(target); + when(resource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); + when(libvirtUtilitiesHelper.getConnection()).thenReturn(connect); + when(resource.getDisks(connect, "vm-name")).thenReturn(List.of()); + when(resource.getDomain(connect, "vm-name")).thenReturn(domain); + + CreateDiskOnlyVmSnapshotAnswer answer = (CreateDiskOnlyVmSnapshotAnswer) wrapper.takeDiskOnlyVmSnapshotOfRunningVm(command, resource); + + assertFalse(answer.getResult()); + assertTrue(answer.getDetails().contains("Failed to verify VM [vm-name] filesystem freeze state")); + assertTrue("Thaw must be attempted when freeze verification fails on malformed JSON", thawCalled[0]); + } + + @Test + public void testTakeDiskOnlyVmSnapshotOfRunningVmSuspendsBeforeNvramCopyForQuiescedUefiSnapshots() throws Exception { + List operations = new ArrayList<>(); + LibvirtCreateDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtCreateDiskOnlyVMSnapshotCommandWrapper() { + @Override + protected Pair>> createSnapshotXmlAndNewVolumePathMap(List volumeObjectTOS, + List disks, VMSnapshotTO target, LibvirtComputingResource resource) { + return new Pair<>("", Collections.emptyMap()); + } + + @Override + protected void freezeVmFilesystems(Domain domain, String vmName) { + operations.add("freeze"); + } + + @Override + protected void verifyVmFilesystemsFrozen(Domain domain, String vmName) { + operations.add("verify"); + } + + @Override + protected boolean suspendVmIfNeeded(Domain domain) { + operations.add("suspend"); + return true; + } + + @Override + protected String backupNvramIfNeeded(CreateDiskOnlyVmSnapshotCommand cmd, LibvirtComputingResource resource) { + operations.add("backup"); + return "nvram/42.fd"; + } + + @Override + protected void cleanupNvramSnapshotIfNeeded(CreateDiskOnlyVmSnapshotCommand cmd, LibvirtComputingResource resource, String nvramSnapshotPath) { + } + + @Override + protected boolean resumeVmIfNeeded(Domain domain, String vmName) { + operations.add("resume"); + return true; + } + + @Override + protected boolean thawVmFilesystemsIfNeeded(Domain domain, String vmName) { + operations.add("thaw"); + return true; + } + }; + LibvirtComputingResource resource = mock(LibvirtComputingResource.class); + LibvirtUtilitiesHelper libvirtUtilitiesHelper = mock(LibvirtUtilitiesHelper.class); + org.libvirt.Connect connect = mock(org.libvirt.Connect.class); + Domain domain = mock(Domain.class); + CreateDiskOnlyVmSnapshotCommand command = mock(CreateDiskOnlyVmSnapshotCommand.class); + VMSnapshotTO target = mock(VMSnapshotTO.class); + + when(command.getVmName()).thenReturn("vm-name"); + when(command.getVolumeTOs()).thenReturn(List.of()); + when(command.getTarget()).thenReturn(target); + when(command.isUefiEnabled()).thenReturn(true); + when(target.getQuiescevm()).thenReturn(true); + when(resource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); + when(libvirtUtilitiesHelper.getConnection()).thenReturn(connect); + when(resource.getDisks(connect, "vm-name")).thenReturn(List.of()); + when(resource.getDomain(connect, "vm-name")).thenReturn(domain); + doThrow(mock(org.libvirt.LibvirtException.class)).when(domain).snapshotCreateXML(anyString(), anyInt()); + + CreateDiskOnlyVmSnapshotAnswer answer = (CreateDiskOnlyVmSnapshotAnswer) wrapper.takeDiskOnlyVmSnapshotOfRunningVm(command, resource); + + assertFalse(answer.getResult()); + assertEquals(List.of("freeze", "verify", "suspend", "backup", "resume", "thaw"), operations); + } + + @Test + public void testTakeDiskOnlyVmSnapshotOfRunningVmReturnsSuccessWithWarningWhenThawFails() throws Exception { + LibvirtCreateDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtCreateDiskOnlyVMSnapshotCommandWrapper() { + @Override + protected Pair>> createSnapshotXmlAndNewVolumePathMap(List volumeObjectTOS, + List disks, VMSnapshotTO target, LibvirtComputingResource resource) { + return new Pair<>("", Collections.emptyMap()); + } + + @Override + protected boolean shouldSuspendVmForSnapshot(CreateDiskOnlyVmSnapshotCommand cmd) { + return false; + } + + @Override + protected boolean shouldFreezeVmFilesystemsForSnapshot(CreateDiskOnlyVmSnapshotCommand cmd) { + return true; + } + + @Override + protected void freezeVmFilesystems(Domain domain, String vmName) { + } + + @Override + protected void verifyVmFilesystemsFrozen(Domain domain, String vmName) { + } + + @Override + protected String backupNvramIfNeeded(CreateDiskOnlyVmSnapshotCommand cmd, LibvirtComputingResource resource) { + return null; + } + + @Override + protected boolean thawVmFilesystemsIfNeeded(Domain domain, String vmName) { + return false; + } + + @Override + protected boolean thawVmFilesystemsIfNeeded(Domain domain, String vmName, boolean frozen) { + return false; + } + }; + LibvirtComputingResource resource = mock(LibvirtComputingResource.class); + LibvirtUtilitiesHelper libvirtUtilitiesHelper = mock(LibvirtUtilitiesHelper.class); + org.libvirt.Connect connect = mock(org.libvirt.Connect.class); + Domain domain = mock(Domain.class); + CreateDiskOnlyVmSnapshotCommand command = mock(CreateDiskOnlyVmSnapshotCommand.class); + VMSnapshotTO target = mock(VMSnapshotTO.class); + + when(command.getVmName()).thenReturn("vm-name"); + when(command.getVolumeTOs()).thenReturn(List.of()); + when(command.getTarget()).thenReturn(target); + when(resource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); + when(libvirtUtilitiesHelper.getConnection()).thenReturn(connect); + when(resource.getDisks(connect, "vm-name")).thenReturn(List.of()); + when(resource.getDomain(connect, "vm-name")).thenReturn(domain); + + CreateDiskOnlyVmSnapshotAnswer answer = (CreateDiskOnlyVmSnapshotAnswer) wrapper.takeDiskOnlyVmSnapshotOfRunningVm(command, resource); + + assertTrue("Snapshot metadata must still be returned when thaw fails after snapshot creation", answer.getResult()); + assertTrue(answer.getDetails().contains("could not be thawed")); + } + + @Test + public void testTakeDiskOnlyVmSnapshotOfRunningVmReturnsSuccessWithWarningWhenResumeFails() throws Exception { + LibvirtCreateDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtCreateDiskOnlyVMSnapshotCommandWrapper() { + @Override + protected Pair>> createSnapshotXmlAndNewVolumePathMap(List volumeObjectTOS, + List disks, VMSnapshotTO target, LibvirtComputingResource resource) { + return new Pair<>("", Collections.emptyMap()); + } + + @Override + protected boolean shouldSuspendVmForSnapshot(CreateDiskOnlyVmSnapshotCommand cmd) { + return true; + } + + @Override + protected boolean suspendVmIfNeeded(Domain domain) { + return true; + } + + @Override + protected boolean shouldFreezeVmFilesystemsForSnapshot(CreateDiskOnlyVmSnapshotCommand cmd) { + return false; + } + + @Override + protected String backupNvramIfNeeded(CreateDiskOnlyVmSnapshotCommand cmd, LibvirtComputingResource resource) { + return null; + } + + @Override + protected boolean resumeVmIfNeeded(Domain domain, String vmName) { + return false; + } + + @Override + protected boolean resumeVmIfNeeded(Domain domain, String vmName, boolean suspended) { + return false; + } + }; + LibvirtComputingResource resource = mock(LibvirtComputingResource.class); + LibvirtUtilitiesHelper libvirtUtilitiesHelper = mock(LibvirtUtilitiesHelper.class); + org.libvirt.Connect connect = mock(org.libvirt.Connect.class); + Domain domain = mock(Domain.class); + CreateDiskOnlyVmSnapshotCommand command = mock(CreateDiskOnlyVmSnapshotCommand.class); + VMSnapshotTO target = mock(VMSnapshotTO.class); + + when(command.getVmName()).thenReturn("vm-name"); + when(command.getVolumeTOs()).thenReturn(List.of()); + when(command.getTarget()).thenReturn(target); + when(resource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); + when(libvirtUtilitiesHelper.getConnection()).thenReturn(connect); + when(resource.getDisks(connect, "vm-name")).thenReturn(List.of()); + when(resource.getDomain(connect, "vm-name")).thenReturn(domain); + + CreateDiskOnlyVmSnapshotAnswer answer = (CreateDiskOnlyVmSnapshotAnswer) wrapper.takeDiskOnlyVmSnapshotOfRunningVm(command, resource); + + assertTrue("Snapshot metadata must still be returned when resume fails after snapshot creation", answer.getResult()); + assertTrue(answer.getDetails().contains("could not be resumed")); + } + + @Test + public void testTakeDiskOnlyVmSnapshotOfRunningVmHandlesNullErrorMessage() throws Exception { + LibvirtCreateDiskOnlyVMSnapshotCommandWrapper wrapper = new LibvirtCreateDiskOnlyVMSnapshotCommandWrapper() { + @Override + protected Pair>> createSnapshotXmlAndNewVolumePathMap(List volumeObjectTOS, + List disks, VMSnapshotTO target, LibvirtComputingResource resource) { + return new Pair<>("", Collections.emptyMap()); + } + + @Override + protected boolean shouldSuspendVmForSnapshot(CreateDiskOnlyVmSnapshotCommand cmd) { + return false; + } + + @Override + protected boolean shouldFreezeVmFilesystemsForSnapshot(CreateDiskOnlyVmSnapshotCommand cmd) { + return false; + } + + @Override + protected String backupNvramIfNeeded(CreateDiskOnlyVmSnapshotCommand cmd, LibvirtComputingResource resource) throws IOException { + throw new IOException(); + } + }; + LibvirtComputingResource resource = mock(LibvirtComputingResource.class); + LibvirtUtilitiesHelper libvirtUtilitiesHelper = mock(LibvirtUtilitiesHelper.class); + org.libvirt.Connect connect = mock(org.libvirt.Connect.class); + Domain domain = mock(Domain.class); + CreateDiskOnlyVmSnapshotCommand command = mock(CreateDiskOnlyVmSnapshotCommand.class); + VMSnapshotTO target = mock(VMSnapshotTO.class); + + when(command.getVmName()).thenReturn("vm-name"); + when(command.getVolumeTOs()).thenReturn(List.of()); + when(command.getTarget()).thenReturn(target); + when(command.isUefiEnabled()).thenReturn(true); + when(resource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); + when(libvirtUtilitiesHelper.getConnection()).thenReturn(connect); + when(resource.getDisks(connect, "vm-name")).thenReturn(List.of()); + when(resource.getDomain(connect, "vm-name")).thenReturn(domain); + + CreateDiskOnlyVmSnapshotAnswer answer = (CreateDiskOnlyVmSnapshotAnswer) wrapper.takeDiskOnlyVmSnapshotOfRunningVm(command, resource); + + assertFalse(answer.getResult()); + } +} diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapperTest.java new file mode 100644 index 000000000000..979ffda2f279 --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPlugNicCommandWrapperTest.java @@ -0,0 +1,147 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.libvirt.Domain; +import org.libvirt.LibvirtException; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.hypervisor.kvm.resource.LibvirtVMDef.InterfaceDef; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtPlugNicCommandWrapperTest { + + /** + * Typical q35-less layout: NICs at 0x03 and 0x06, other PCI devices at 0x01, 0x02, 0x04, 0x05 and 0x07, + * plus non-PCI addresses (drive, usb) that must be ignored. + */ + private static final String DOMAIN_XML = + "\n" + + " i-2-42-VM\n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + " \n" + + "
\n" + + " \n" + + " \n" + + "
\n" + + " \n" + + "