Merge branch 'master' into fix/issue-20592

changelog/unreleased/issue-13822.toml

@@ -0,0 +1,9 @@
+type = "fixed"
+message = "Fix link target for search query error documentation links."
+
+issues = ["13822"]
+pulls = ["20563"]
+
+details.user = """
+The documentation link now points to the error types section again, instead of just the search query page.
+"""

changelog/unreleased/pr-20366.toml

@@ -0,0 +1,4 @@
+type = "f"
+message = "Datanode preflight check for node.lock, preventing parallel run of opensearch and datanode"
+
+pulls = ["20366"]

data-node/src/main/java/org/graylog/datanode/rest/OpensearchLockCheckController.java

@@ -0,0 +1,89 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.datanode.rest;
+
+import jakarta.inject.Inject;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.MediaType;
+import org.graylog.datanode.configuration.DatanodeConfiguration;
+import org.graylog.plugins.views.storage.migration.state.actions.OpensearchLockCheckResult;
+import org.graylog.plugins.views.storage.migration.state.actions.OpensearchNodeLock;
+import org.graylog2.bootstrap.preflight.PreflightCheckException;
+
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.NonWritableChannelException;
+import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.Files;
+import java.nio.file.StandardOpenOption;
+import java.util.Collections;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+@Path("/lock-check")
+@Produces(MediaType.APPLICATION_JSON)
+public class OpensearchLockCheckController {
+
+    private final java.nio.file.Path dataTargetDir;
+
+    @Inject
+    public OpensearchLockCheckController(DatanodeConfiguration datanodeConfiguration) {
+        this(datanodeConfiguration.datanodeDirectories().getDataTargetDir());
+    }
+
+    public OpensearchLockCheckController(java.nio.file.Path dataTargetDir) {
+        this.dataTargetDir = dataTargetDir;
+    }
+
+    @GET
+    public OpensearchLockCheckResult checkLockFiles() {
+        final java.nio.file.Path nodesDir = dataTargetDir.resolve("nodes");
+        if (Files.isDirectory(nodesDir)) {
+            try (final Stream<java.nio.file.Path> nodes = Files.list(nodesDir)) {
+                return nodes.map(n -> new OpensearchNodeLock(n, isDirLocked(n)))
+                        .collect(Collectors.collectingAndThen(Collectors.toList(), OpensearchLockCheckResult::new));
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        } else {
+            return new OpensearchLockCheckResult(Collections.emptyList());
+        }
+    }
+
+    private static boolean isDirLocked(java.nio.file.Path nodeDir) {
+        final java.nio.file.Path lockFile = nodeDir.resolve("node.lock");
+        if (Files.exists(lockFile)) {
+            try (FileChannel channel = FileChannel.open(lockFile, StandardOpenOption.WRITE)) {
+                final FileLock fileLock = channel.tryLock();
+                if (fileLock != null) { // file was not locked, we are good to go, let's release immediately
+                    fileLock.release();
+                    return false;
+                } else {
+                    return true;
+                }
+            } catch (OverlappingFileLockException e) {
+                return true;
+            } catch (NonWritableChannelException | IOException e) {
+                throw new PreflightCheckException("Failed to verify free node.lock file", e);
+            }
+        }
+        return false;
+    }
+}

data-node/src/main/java/org/graylog/datanode/rest/RestBindings.java

@@ -28,5 +28,6 @@ protected void configure() {
         addSystemRestResource(OpensearchConnectionCheckController.class);
         addSystemRestResource(IndexStateController.class);
         addSystemRestResource(CertificatesController.class);
+        addSystemRestResource(OpensearchLockCheckController.class);
     }
 }

data-node/src/test/java/org/graylog/datanode/rest/OpensearchLockCheckControllerTest.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.datanode.rest;
+
+import jakarta.annotation.Nonnull;
+import org.assertj.core.api.Assertions;
+import org.graylog.plugins.views.storage.migration.state.actions.OpensearchLockCheckResult;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.IOException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+
+class OpensearchLockCheckControllerTest {
+    @Test
+    void testNotLockedDir(@TempDir Path tempDir) throws IOException {
+        final OpensearchLockCheckController controller = new OpensearchLockCheckController(tempDir);
+        createLockFile(tempDir);
+        final OpensearchLockCheckResult result = controller.checkLockFiles();
+        Assertions.assertThat(result.locks())
+                .hasSize(1)
+                .allSatisfy(l -> Assertions.assertThat(l.locked()).isFalse());
+    }
+
+    @Test
+    void testLockedDir(@TempDir Path tempDir) throws IOException {
+        final OpensearchLockCheckController controller = new OpensearchLockCheckController(tempDir);
+        final Path lockFile = createLockFile(tempDir);
+        lock(lockFile);
+        final OpensearchLockCheckResult result = controller.checkLockFiles();
+        Assertions.assertThat(result.locks())
+                .hasSize(1)
+                .allSatisfy(l -> Assertions.assertThat(l.locked()).isTrue());
+    }
+
+    @Test
+    void testEmptyDir(@TempDir Path tempDir) {
+        final OpensearchLockCheckController controller = new OpensearchLockCheckController(tempDir);
+        final OpensearchLockCheckResult result = controller.checkLockFiles();
+        Assertions.assertThat(result.locks())
+                .isEmpty();
+    }
+
+    @Nonnull
+    private static Path createLockFile(Path tempDir) throws IOException {
+        final Path nodeDir = tempDir.resolve("nodes").resolve("0");
+        Files.createDirectories(nodeDir);
+        final Path lockFile = nodeDir.resolve("node.lock");
+        Files.createFile(lockFile);
+        return lockFile;
+    }
+
+    private FileLock lock(Path lockFile) throws IOException {
+        FileChannel channel = FileChannel.open(lockFile, StandardOpenOption.WRITE);
+        return channel.lock();
+    }
+}

graylog-plugin-archetype/pom.xml

@@ -46,7 +46,7 @@
             <extension>
                 <groupId>org.apache.maven.archetype</groupId>
                 <artifactId>archetype-packaging</artifactId>
-                <version>3.2.1</version>
+                <version>3.3.0</version>
             </extension>
         </extensions>
 
@@ -84,7 +84,7 @@
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-archetype-plugin</artifactId>
-                    <version>3.2.1</version>
+                    <version>3.3.0</version>
                 </plugin>
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>

graylog-project-parent/pom.xml

@@ -614,7 +614,7 @@
                                 <plugin>
                                     <groupId>com.mebigfatguy.fb-contrib</groupId>
                                     <artifactId>fb-contrib</artifactId>
-                                    <version>7.6.4</version>
+                                    <version>7.6.5</version>
                                 </plugin>
                             </plugins>
                         </configuration>

graylog2-server/src/main/java/org/graylog/plugins/map/config/GeoIpProcessorConfig.java

@@ -25,6 +25,7 @@
 import java.nio.file.Path;
 import java.util.Optional;
 
+@SuppressWarnings({"unused", "FieldMayBeFinal"})
 @Singleton
 public class GeoIpProcessorConfig extends PathConfiguration {
     private static final String PREFIX = "geo_ip_processor";
@@ -40,5 +41,5 @@ public Path getS3DownloadLocation() {
     }
 
     @Parameter(value = DISABLE_IPINFO_DB_TYPE_CHECK)
-    private final boolean disableIpInfoDBTypeCheck = false;
+    private boolean disableIpInfoDBTypeCheck = false;
 }

graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/state/actions/DatanodeOpensearchClusterCheckResource.java

@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.views.storage.migration.state.actions;
+
+import retrofit2.Call;
+import retrofit2.http.GET;
+
+public interface DatanodeOpensearchClusterCheckResource {
+
+    @GET("/lock-check")
+    Call<OpensearchLockCheckResult> checkLocks();
+
+}

graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/state/actions/MigrationActions.java

@@ -51,7 +51,9 @@ public interface MigrationActions {
 
     void startDataNodes();
 
-    boolean dataNodeStartupFinished();
+    boolean allDatanodesAvailable();
+
+    void setPreflightFinished();
 
     void startRemoteReindex();
 

graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/state/actions/MigrationActionsImpl.java

@@ -116,8 +116,15 @@ public void runDirectoryCompatibilityCheck() {
 
     @Override
     public boolean isOldClusterStopped() {
-        // TODO: add real test
-        return true;
+        final Map<String, OpensearchLockCheckResult> results = datanodeProxy.remoteInterface(DatanodeResolver.ALL_NODES_KEYWORD, DatanodeOpensearchClusterCheckResource.class, DatanodeOpensearchClusterCheckResource::checkLocks);
+        final boolean anyLocked = results.values().stream().anyMatch(v -> v.locks().stream().anyMatch(OpensearchNodeLock::locked));
+
+        if (anyLocked) {
+            results.forEach((key, value) -> value.locks().stream()
+                    .filter(OpensearchNodeLock::locked)
+                    .forEach(v -> LOG.info("Data directory of datanode {} is still locked by another Opensearch process. Lock file: {}", key, v.path().toAbsolutePath())));
+        }
+        return !anyLocked;
     }
 
     @Override
@@ -232,15 +239,18 @@ private void startDataNode(DataNodeDto node) {
     }
 
     @Override
-    public boolean dataNodeStartupFinished() {
-        boolean dataNodesAvailable = nodeService.allActive().values().stream().allMatch(node -> node.getDataNodeStatus() == DataNodeStatus.AVAILABLE);
-        if (dataNodesAvailable) { // set preflight config to FINISHED to be sure that a Graylog restart will connect to the data nodes
-            var preflight = preflightConfigService.getPreflightConfigResult();
-            if (preflight == null || !preflight.equals(PreflightConfigResult.FINISHED)) {
-                preflightConfigService.setConfigResult(PreflightConfigResult.FINISHED);
-            }
+    public boolean allDatanodesAvailable() {
+        final Map<String, DataNodeDto> activeNodes = nodeService.allActive();
+        return !activeNodes.isEmpty() && activeNodes.values()
+                .stream()
+                .allMatch(node -> node.getDataNodeStatus() == DataNodeStatus.AVAILABLE);
+    }
+
+    public void setPreflightFinished() {
+        var preflight = preflightConfigService.getPreflightConfigResult();
+        if (preflight == null || !preflight.equals(PreflightConfigResult.FINISHED)) {
+            preflightConfigService.setConfigResult(PreflightConfigResult.FINISHED);
         }
-        return dataNodesAvailable;
     }
 
     @Override

graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/state/actions/OpensearchLockCheckResult.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.views.storage.migration.state.actions;
+
+import java.util.Collection;
+
+public record OpensearchLockCheckResult(Collection<OpensearchNodeLock> locks) {
+}

graylog2-server/src/main/java/org/graylog/plugins/views/storage/migration/state/actions/OpensearchNodeLock.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog.plugins.views.storage.migration.state.actions;
+
+import java.nio.file.Path;
+
+public record OpensearchNodeLock(Path path, boolean locked) {
+}