From 7f0d9a03045af506ea9f37dc40eec45bbeeb174f Mon Sep 17 00:00:00 2001 From: SadiJr Date: Tue, 25 Jun 2024 16:11:38 -0300 Subject: [PATCH] [Veeam] Check for failures in the restore process (#7224) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Validate failure state in Veeam restore process * Address Daan review, and properly call method * Address bryan's reviews * remove return Co-authored-by: SadiJr Co-authored-by: João Jandre <48719461+JoaoJandre@users.noreply.github.com> --- .../cloudstack/backup/veeam/VeeamClient.java | 43 +++++++++++++++++-- .../backup/veeam/VeeamClientTest.java | 40 ++++++++++++++++- 2 files changed, 78 insertions(+), 5 deletions(-) diff --git a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java index 701c45f1a9d0..befeb231015a 100644 --- a/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java +++ b/plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java @@ -345,7 +345,7 @@ private boolean checkTaskStatus(final HttpResponse response) throws IOException String type = pair.second(); String path = url.replace(apiURI.toString(), ""); if (type.equals("RestoreSession")) { - return checkIfRestoreSessionFinished(type, path); + checkIfRestoreSessionFinished(type, path); } } return true; @@ -361,17 +361,29 @@ private boolean checkTaskStatus(final HttpResponse response) throws IOException return false; } - protected boolean checkIfRestoreSessionFinished(String type, String path) throws IOException { - for (int j = 0; j < this.restoreTimeout; j++) { + + /** + * Checks the status of the restore session. Checked states are "Success" and "Failure".
+ * There is also a timeout defined in the global configuration, backup.plugin.veeam.restore.timeout,
+ * that is used to wait for the restore to complete before throwing a {@link CloudRuntimeException}. + */ + protected void checkIfRestoreSessionFinished(String type, String path) throws IOException { + for (int j = 0; j < restoreTimeout; j++) { HttpResponse relatedResponse = get(path); RestoreSession session = parseRestoreSessionResponse(relatedResponse); if (session.getResult().equals("Success")) { - return true; + return; } + if (session.getResult().equalsIgnoreCase("Failed")) { String sessionUid = session.getUid(); + LOG.error(String.format("Failed to restore backup [%s] of VM [%s] due to [%s].", + sessionUid, session.getVmDisplayName(), + getRestoreVmErrorDescription(StringUtils.substringAfterLast(sessionUid, ":")))); throw new CloudRuntimeException(String.format("Restore job [%s] failed.", sessionUid)); } + LOG.debug(String.format("Waiting %s seconds, out of a total of %s seconds, for the restore backup process to finish.", j, restoreTimeout)); + try { Thread.sleep(1000); } catch (InterruptedException ignored) { @@ -930,6 +942,29 @@ public Pair restoreVMToDifferentLocation(String restorePointId, return new Pair<>(result.first(), restoreLocation); } + /** + * Tries to retrieve the error's description of the Veeam restore task that resulted in an error. + * @param uid Session uid in Veeam of the restore process; + * @return the description found in Veeam about the cause of error in the restore process. + */ + protected String getRestoreVmErrorDescription(String uid) { + LOG.debug(String.format("Trying to find the cause of error in the restore process [%s].", uid)); + List cmds = Arrays.asList( + String.format("$restoreUid = '%s'", uid), + "$restore = Get-VBRRestoreSession -Id $restoreUid", + "if ($restore) {", + "Write-Output $restore.Description", + "} else {", + "Write-Output 'Cannot find restore session with provided uid $restoreUid'", + "}" + ); + Pair result = executePowerShellCommands(cmds); + if (result != null && result.first()) { + return result.second(); + } + return String.format("Failed to get the description of the failed restore session [%s]. Please contact an administrator.", uid); + } + private boolean isLegacyServer() { return this.veeamServerVersion != null && (this.veeamServerVersion > 0 && this.veeamServerVersion < 11); } diff --git a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java index 06804d68da27..26b2449b0fe4 100644 --- a/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java +++ b/plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java @@ -58,6 +58,8 @@ public class VeeamClientTest { private VeeamClient mockClient; private static final SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + private VeeamClient mock = Mockito.mock(VeeamClient.class); + @Rule public WireMockRule wireMockRule = new WireMockRule(9399); @@ -161,7 +163,7 @@ public void checkIfRestoreSessionFinishedTestTimeoutException() throws IOExcepti Mockito.when(mockClient.get(Mockito.anyString())).thenReturn(httpResponse); Mockito.when(mockClient.parseRestoreSessionResponse(httpResponse)).thenReturn(restoreSession); Mockito.when(restoreSession.getResult()).thenReturn("No Success"); - Mockito.when(mockClient.checkIfRestoreSessionFinished(Mockito.eq("RestoreTest"), Mockito.eq("any"))).thenCallRealMethod(); + Mockito.doCallRealMethod().when(mockClient).checkIfRestoreSessionFinished(Mockito.eq("RestoreTest"), Mockito.eq("any")); mockClient.checkIfRestoreSessionFinished("RestoreTest", "any"); fail(); } catch (Exception e) { @@ -170,6 +172,42 @@ public void checkIfRestoreSessionFinishedTestTimeoutException() throws IOExcepti Mockito.verify(mockClient, times(10)).get(Mockito.anyString()); } + @Test + public void getRestoreVmErrorDescriptionTestFindErrorDescription() { + Pair response = new Pair<>(true, "Example of error description found in Veeam."); + Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); + Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(response); + String result = mock.getRestoreVmErrorDescription("uuid"); + Assert.assertEquals("Example of error description found in Veeam.", result); + } + + @Test + public void getRestoreVmErrorDescriptionTestNotFindErrorDescription() { + Pair response = new Pair<>(true, "Cannot find restore session with provided uid uuid"); + Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); + Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(response); + String result = mock.getRestoreVmErrorDescription("uuid"); + Assert.assertEquals("Cannot find restore session with provided uid uuid", result); + } + + @Test + public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsNull() { + Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); + Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(null); + String result = mock.getRestoreVmErrorDescription("uuid"); + Assert.assertEquals("Failed to get the description of the failed restore session [uuid]. Please contact an administrator.", result); + } + + @Test + public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsFalse() { + Pair response = new Pair<>(false, null); + Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod(); + Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(response); + String result = mock.getRestoreVmErrorDescription("uuid"); + Assert.assertEquals("Failed to get the description of the failed restore session [uuid]. Please contact an administrator.", result); + } + + private void verifyBackupMetrics(Map metrics) { Assert.assertEquals(2, metrics.size());