Skip to content

Commit

Permalink
[Snapshot V2] Fix for create snapshot build and test failures (opense…
Browse files Browse the repository at this point in the history
…arch-project#15602)

---------

Signed-off-by: Anshu Agarwal <[email protected]>
Signed-off-by: Sachin Kale <[email protected]>
Co-authored-by: Anshu Agarwal <[email protected]>
Co-authored-by: Sachin Kale <[email protected]>
  • Loading branch information
3 people authored Sep 3, 2024
1 parent 33be5a9 commit 70ba5a1
Show file tree
Hide file tree
Showing 8 changed files with 127 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,27 @@ public void getRepositoryData(ActionListener<RepositoryData> listener) {
in.getRepositoryData(listener);
}

@Override
public void finalizeSnapshot(
ShardGenerations shardGenerations,
long repositoryStateId,
Metadata clusterMetadata,
SnapshotInfo snapshotInfo,
Version repositoryMetaVersion,
Function<ClusterState, ClusterState> stateTransformer,
ActionListener<RepositoryData> listener
) {
in.finalizeSnapshot(
shardGenerations,
repositoryStateId,
clusterMetadata,
snapshotInfo,
repositoryMetaVersion,
stateTransformer,
listener
);
}

@Override
public void finalizeSnapshot(
ShardGenerations shardGenerations,
Expand Down
24 changes: 24 additions & 0 deletions server/src/main/java/org/opensearch/repositories/Repository.java
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,30 @@ default Repository create(RepositoryMetadata metadata, Function<String, Reposito
*/
void getRepositoryData(ActionListener<RepositoryData> listener);

/**
* Finalizes snapshotting process
* <p>
* This method is called on cluster-manager after all shards are snapshotted.
*
* @param shardGenerations updated shard generations
* @param repositoryStateId the unique id identifying the state of the repository when the snapshot began
* @param clusterMetadata cluster metadata
* @param snapshotInfo SnapshotInfo instance to write for this snapshot
* @param repositoryMetaVersion version of the updated repository metadata to write
* @param stateTransformer a function that filters the last cluster state update that the snapshot finalization will execute and
* is used to remove any state tracked for the in-progress snapshot from the cluster state
* @param listener listener to be invoked with the new {@link RepositoryData} after completing the snapshot
*/
void finalizeSnapshot(
ShardGenerations shardGenerations,
long repositoryStateId,
Metadata clusterMetadata,
SnapshotInfo snapshotInfo,
Version repositoryMetaVersion,
Function<ClusterState, ClusterState> stateTransformer,
ActionListener<RepositoryData> listener
);

/**
* Finalizes snapshotting process
* <p>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2024,6 +2024,28 @@ private DeleteResult cleanUpStaleSnapshotShardPathsFile(List<String> matchingSha
return new DeleteResult(matchingShardPaths.size(), totalBytes);
}

@Override
public void finalizeSnapshot(
final ShardGenerations shardGenerations,
final long repositoryStateId,
final Metadata clusterMetadata,
SnapshotInfo snapshotInfo,
Version repositoryMetaVersion,
Function<ClusterState, ClusterState> stateTransformer,
final ActionListener<RepositoryData> listener
) {
finalizeSnapshot(
shardGenerations,
repositoryStateId,
clusterMetadata,
snapshotInfo,
repositoryMetaVersion,
stateTransformer,
Priority.NORMAL,
listener
);
}

@Override
public void finalizeSnapshot(
final ShardGenerations shardGenerations,
Expand Down
32 changes: 32 additions & 0 deletions server/src/main/java/org/opensearch/snapshots/SnapshotInfo.java
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,38 @@ public SnapshotInfo(SnapshotsInProgress.Entry entry) {
);
}

public SnapshotInfo(
SnapshotId snapshotId,
List<String> indices,
List<String> dataStreams,
long startTime,
String reason,
long endTime,
int totalShards,
List<SnapshotShardFailure> shardFailures,
Boolean includeGlobalState,
Map<String, Object> userMetadata,
Boolean remoteStoreIndexShallowCopy
) {
this(
snapshotId,
indices,
dataStreams,
snapshotState(reason, shardFailures),
reason,
Version.CURRENT,
startTime,
endTime,
totalShards,
totalShards - shardFailures.size(),
shardFailures,
includeGlobalState,
userMetadata,
remoteStoreIndexShallowCopy,
0
);
}

public SnapshotInfo(
SnapshotId snapshotId,
List<String> indices,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
Setting.Property.Dynamic
);

private static final String SNAPSHOT_PINNED_TIMESTAMP_DELIMITER = "__";
/**
* Setting to specify the maximum number of shards that can be included in the result for the snapshot status
* API call. Note that it does not apply to V2-shallow snapshots.
Expand All @@ -216,8 +217,6 @@ public class SnapshotsService extends AbstractLifecycleComponent implements Clus
Setting.Property.NodeScope,
Setting.Property.Dynamic
);

private static final String SNAPSHOT_PINNED_TIMESTAMP_DELIMITER = ":";
private volatile int maxConcurrentOperations;

public SnapshotsService(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -681,6 +681,19 @@ public void finalizeSnapshot(
listener.onResponse(null);
}

@Override
public void finalizeSnapshot(
ShardGenerations shardGenerations,
long repositoryStateId,
Metadata clusterMetadata,
SnapshotInfo snapshotInfo,
Version repositoryMetaVersion,
Function<ClusterState, ClusterState> stateTransformer,
ActionListener<RepositoryData> listener
) {
listener.onResponse(null);
}

@Override
public void deleteSnapshots(
Collection<SnapshotId> snapshotIds,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,19 @@ public void getRepositoryData(ActionListener<RepositoryData> listener) {
);
}

@Override
public void finalizeSnapshot(
ShardGenerations shardGenerations,
long repositoryStateId,
Metadata clusterMetadata,
SnapshotInfo snapshotInfo,
Version repositoryMetaVersion,
Function<ClusterState, ClusterState> stateTransformer,
ActionListener<RepositoryData> listener
) {
listener.onResponse(null);
}

@Override
public void finalizeSnapshot(
ShardGenerations shardGenerations,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2898,7 +2898,7 @@ private static Settings buildRemoteStoreNodeAttributes(
}
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), randomFrom(PathType.values()));
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_METADATA.getKey(), randomBoolean());
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), false);
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), randomBoolean());
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_SEGMENTS_PATH_PREFIX.getKey(), translogPathFixedPrefix ? "a" : "");
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_TRANSLOG_PATH_PREFIX.getKey(), segmentsPathFixedPrefix ? "b" : "");
return settings.build();
Expand Down

0 comments on commit 70ba5a1

Please sign in to comment.