Skip to content

Commit

Permalink
[Veeam] Check for failures in the restore process (#7224)
Browse files Browse the repository at this point in the history 
* Validate failure state in Veeam restore process

* Address Daan review, and properly call method

* Address bryan's reviews

* remove return

Co-authored-by: SadiJr <[email protected]>
Co-authored-by: João Jandre <[email protected]>
  • Loading branch information
@SadiJr @JoaoJandre
3 people authored Jun 25, 2024
1 parent 4eb4365 commit 7f0d9a0
Show file tree
Hide file tree
Showing 2 changed files with 78 additions and 5 deletions.
43 changes: 39 additions & 4 deletions plugins/backup/veeam/src/main/java/org/apache/cloudstack/backup/veeam/VeeamClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,7 @@ private boolean checkTaskStatus(final HttpResponse response) throws IOException
String type = pair.second();
String path = url.replace(apiURI.toString(), "");
if (type.equals("RestoreSession")) {
return checkIfRestoreSessionFinished(type, path);
checkIfRestoreSessionFinished(type, path);
}
}
return true;
Expand All @@ -361,17 +361,29 @@ private boolean checkTaskStatus(final HttpResponse response) throws IOException
return false;
}

protected boolean checkIfRestoreSessionFinished(String type, String path) throws IOException {
for (int j = 0; j < this.restoreTimeout; j++) {

/**
* Checks the status of the restore session. Checked states are "Success" and "Failure".<br/>
* There is also a timeout defined in the global configuration, backup.plugin.veeam.restore.timeout,<br/>
* that is used to wait for the restore to complete before throwing a {@link CloudRuntimeException}.
*/
protected void checkIfRestoreSessionFinished(String type, String path) throws IOException {
for (int j = 0; j < restoreTimeout; j++) {
HttpResponse relatedResponse = get(path);
RestoreSession session = parseRestoreSessionResponse(relatedResponse);
if (session.getResult().equals("Success")) {
return true;
return;
}

if (session.getResult().equalsIgnoreCase("Failed")) {
String sessionUid = session.getUid();
LOG.error(String.format("Failed to restore backup [%s] of VM [%s] due to [%s].",
sessionUid, session.getVmDisplayName(),
getRestoreVmErrorDescription(StringUtils.substringAfterLast(sessionUid, ":"))));
throw new CloudRuntimeException(String.format("Restore job [%s] failed.", sessionUid));
}
LOG.debug(String.format("Waiting %s seconds, out of a total of %s seconds, for the restore backup process to finish.", j, restoreTimeout));

try {
Thread.sleep(1000);
} catch (InterruptedException ignored) {
Expand Down Expand Up @@ -930,6 +942,29 @@ public Pair<Boolean, String> restoreVMToDifferentLocation(String restorePointId,
return new Pair<>(result.first(), restoreLocation);
}

/**
* Tries to retrieve the error's description of the Veeam restore task that resulted in an error.
* @param uid Session uid in Veeam of the restore process;
* @return the description found in Veeam about the cause of error in the restore process.
*/
protected String getRestoreVmErrorDescription(String uid) {
LOG.debug(String.format("Trying to find the cause of error in the restore process [%s].", uid));
List<String> cmds = Arrays.asList(
String.format("$restoreUid = '%s'", uid),
"$restore = Get-VBRRestoreSession -Id $restoreUid",
"if ($restore) {",
"Write-Output $restore.Description",
"} else {",
"Write-Output 'Cannot find restore session with provided uid $restoreUid'",
"}"
);
Pair<Boolean, String> result = executePowerShellCommands(cmds);
if (result != null && result.first()) {
return result.second();
}
return String.format("Failed to get the description of the failed restore session [%s]. Please contact an administrator.", uid);
}

private boolean isLegacyServer() {
return this.veeamServerVersion != null && (this.veeamServerVersion > 0 && this.veeamServerVersion < 11);
}
Expand Down
40 changes: 39 additions & 1 deletion plugins/backup/veeam/src/test/java/org/apache/cloudstack/backup/veeam/VeeamClientTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ public class VeeamClientTest {
private VeeamClient mockClient;
private static final SimpleDateFormat newDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

private VeeamClient mock = Mockito.mock(VeeamClient.class);

@Rule
public WireMockRule wireMockRule = new WireMockRule(9399);

Expand Down Expand Up @@ -161,7 +163,7 @@ public void checkIfRestoreSessionFinishedTestTimeoutException() throws IOExcepti
Mockito.when(mockClient.get(Mockito.anyString())).thenReturn(httpResponse);
Mockito.when(mockClient.parseRestoreSessionResponse(httpResponse)).thenReturn(restoreSession);
Mockito.when(restoreSession.getResult()).thenReturn("No Success");
Mockito.when(mockClient.checkIfRestoreSessionFinished(Mockito.eq("RestoreTest"), Mockito.eq("any"))).thenCallRealMethod();
Mockito.doCallRealMethod().when(mockClient).checkIfRestoreSessionFinished(Mockito.eq("RestoreTest"), Mockito.eq("any"));
mockClient.checkIfRestoreSessionFinished("RestoreTest", "any");
fail();
} catch (Exception e) {
Expand All @@ -170,6 +172,42 @@ public void checkIfRestoreSessionFinishedTestTimeoutException() throws IOExcepti
Mockito.verify(mockClient, times(10)).get(Mockito.anyString());
}

@Test
public void getRestoreVmErrorDescriptionTestFindErrorDescription() {
Pair<Boolean, String> response = new Pair<>(true, "Example of error description found in Veeam.");
Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod();
Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(response);
String result = mock.getRestoreVmErrorDescription("uuid");
Assert.assertEquals("Example of error description found in Veeam.", result);
}

@Test
public void getRestoreVmErrorDescriptionTestNotFindErrorDescription() {
Pair<Boolean, String> response = new Pair<>(true, "Cannot find restore session with provided uid uuid");
Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod();
Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(response);
String result = mock.getRestoreVmErrorDescription("uuid");
Assert.assertEquals("Cannot find restore session with provided uid uuid", result);
}

@Test
public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsNull() {
Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod();
Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(null);
String result = mock.getRestoreVmErrorDescription("uuid");
Assert.assertEquals("Failed to get the description of the failed restore session [uuid]. Please contact an administrator.", result);
}

@Test
public void getRestoreVmErrorDescriptionTestWhenPowerShellOutputIsFalse() {
Pair<Boolean, String> response = new Pair<>(false, null);
Mockito.when(mock.getRestoreVmErrorDescription("uuid")).thenCallRealMethod();
Mockito.when(mock.executePowerShellCommands(Mockito.any())).thenReturn(response);
String result = mock.getRestoreVmErrorDescription("uuid");
Assert.assertEquals("Failed to get the description of the failed restore session [uuid]. Please contact an administrator.", result);
}


private void verifyBackupMetrics(Map<String, Backup.Metric> metrics) {
Assert.assertEquals(2, metrics.size());

Expand Down

0 comments on commit 7f0d9a0

Please sign in to comment.