2.3.19 (#40)

* Container.groovy
 * Added prepareDevice, hasDevice, prepareCapability, hasCapability

ContainerTest.groovy
 * Added tests for devices and capabilities

* Tweaking of documentation

* Container.groovy
 * Solved bug that caused error to be thrown when stopping and removing container
 * prepareCustomEnvVar() now adds envs if other envs are present

DirectorySyncer.groovy
 * New Util container intended to sync files to Docker volume
 * WIP, needs cleanup and documentation

* DirectorySyncer.groovy
 * Cleaned up, added rsyncOptions

* DirectorySyncerTest.groovy
 * Tweaking of test

* Bumped to 2.3.19

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.eficode</groupId>
     <artifactId>devstack</artifactId>
-    <version>2.3.18</version>
+    <version>2.3.19</version>
     <packaging>jar</packaging>
 
     <name>DevStack</name>

src/main/groovy/com/eficode/devstack/container/Container.groovy

@@ -437,10 +437,14 @@ trait Container {
 
         if (self.containerId) {
 
-            dockerClient.stop(self.containerId, timeoutS)
-            if (self.status() == ContainerState.Status.Running) {
-                dockerClient.kill(self.containerId)
+
+            if (self.isRunning()) {
+                dockerClient.stop(self.containerId, timeoutS)
+                if (self.isRunning()) {
+                    dockerClient.kill(self.containerId)
+                }
             }
+
             dockerClient.rm(self.containerId)
 
 
@@ -1055,7 +1059,7 @@ trait Container {
 
         assert hasNeverBeenStarted(): "Error, cant set custom environment variables after creating container"
 
-        self.customEnvVar = keyVar
+        self.customEnvVar.addAll(keyVar.collect {it.toString()})
     }
 
 

src/main/groovy/com/eficode/devstack/util/DirectorySyncer.groovy

@@ -0,0 +1,149 @@
+package com.eficode.devstack.util
+
+import com.eficode.devstack.container.Container
+import de.gesellix.docker.client.EngineResponseContent
+import de.gesellix.docker.remote.api.ContainerSummary
+import de.gesellix.docker.remote.api.Volume
+import org.slf4j.Logger
+
+import javax.naming.NameNotFoundException
+
+
+class DirectorySyncer implements Container {
+
+    String containerName = "DirectorySyncer"
+    String containerMainPort = null
+    String containerImage = "alpine"
+    String containerImageTag = "latest"
+    String defaultShell = "/bin/sh"
+
+    DirectorySyncer(String dockerHost = "", String dockerCertPath = "") {
+        if (dockerHost && dockerCertPath) {
+            assert setupSecureRemoteConnection(dockerHost, dockerCertPath): "Error setting up secure remote docker connection"
+        }
+    }
+
+    static String getSyncScript(String rsyncOptions = "-avh") {
+
+        return """
+        
+        apk update
+        apk add inotify-tools rsync tini
+        #apt update
+        #apt install -y inotify-tools rsync tini
+
+        if [ -z "\$(which inotifywait)" ]; then
+            echo "inotifywait not installed."
+            echo "In most distros, it is available in the inotify-tools package."
+            exit 1
+        fi
+        
+        
+        function execute() {
+            eval "\$@"
+                rsync $rsyncOptions /mnt/src/*/ /mnt/dest/ 
+        }
+        
+        execute""
+        
+        inotifywait --recursive --monitor --format "%e %w%f" \\
+        --event modify,create,delete,moved_from,close_write /mnt/src \\
+        | while read changed; do
+            echo "\$changed"
+            execute "\$@"
+        done
+        """.stripIndent()
+
+    }
+
+    String getAvailableContainerName(String prefix = "DirectorySyncer") {
+
+        Integer suffixNr = null
+        String availableName = null
+
+        ArrayList<ContainerSummary> containers = dockerClient.ps().content
+
+
+        while (!availableName) {
+
+            String containerName = prefix + (suffixNr ?: "")
+            if (!containers.any { container -> container.names.any { name -> name.equalsIgnoreCase("/" + containerName) } }) {
+                availableName = containerName
+            } else if (suffixNr > 100) {
+                throw new NameNotFoundException("Could not find avaialble name for container, last test was: $containerName")
+            } else {
+                suffixNr ? suffixNr++ : (suffixNr = 1)
+            }
+        }
+
+        return availableName
+
+    }
+
+    /**
+     * <pre>
+     * Creates a Util container:
+     *  1. Listens for file changes in one or more docker engine src paths (hostAbsSourcePaths)
+     *  2. If changes are detected rsync is triggered
+     *  3. Rsync detects changes and sync them to destVolumeName
+     *
+     *  The root of all the srcPaths will be combined and synced to destVolume,
+     *  ex:
+     *      ../srcPath1/file1.txt
+     *      ../srcPath2/file2.txt
+     *      ../srcPath2/subdir/file3.txt
+     *      Will give:
+     *      destVolume/file1.txt
+     *      destVolume/file2.txt
+     *      destVolume/subdir/file3.txt
+     *  </pre>
+     *
+     *  <b>Known Issues</b>
+     *  <pre>
+     *  Delete events are not properly detected and triggered on,
+     *  thus any such actions will only be reflected after
+     *  subsequent create/update events.
+     *  </pre>
+     * @param hostAbsSourcePaths A list of one or more src dirs to sync from
+     * @param destVolumeName A docker volume to sync to, if it does not exist it will be created
+     * @param rsyncOptions Options to use when running rsync, ie: rsync $rsyncOptions /mnt/src/*\/ /mnt/dest/<p>
+     *      example: -avh --delete
+     * @param dockerHost Docker host to run on
+     * @param dockerCertPath Docker certs to use
+     * @return
+     */
+    static DirectorySyncer createSyncToVolume(ArrayList<String> hostAbsSourcePaths, String destVolumeName, String rsyncOptions = "-avh", String dockerHost = "", String dockerCertPath = "") {
+
+        DirectorySyncer container = new DirectorySyncer(dockerHost, dockerCertPath)
+        Logger log = container.log
+
+        container.containerName = container.getAvailableContainerName()
+        container.prepareCustomEnvVar(["syncScript=${getSyncScript(rsyncOptions)}"])
+
+        Volume volume = container.dockerClient.getVolumesWithName(destVolumeName).find { true }
+
+        if (volume) {
+            log.debug("\tFound existing volume:" + volume.name)
+        } else {
+            log.debug("\tCreating new volume $destVolumeName")
+            EngineResponseContent<Volume> volumeResponse = container.dockerClient.createVolume(destVolumeName)
+            volume = volumeResponse?.content
+            assert volume: "Error creating volume $destVolumeName, " + volumeResponse?.getStatus()?.text
+            log.debug("\t\tCreated volume:" + volume.name)
+        }
+
+        container.prepareVolumeMount(volume.name, "/mnt/dest/", false)
+
+        hostAbsSourcePaths.each { srcPath ->
+            String srcDirName = srcPath.substring(srcPath.lastIndexOf("/") + 1)
+            container.prepareBindMount(srcPath, "/mnt/src/$srcDirName", true)
+        }
+
+        container.createContainer(["/bin/sh", "-c", "echo \"\$syncScript\" > /syncScript.sh && /bin/sh syncScript.sh"], [])
+        container.startContainer()
+
+        return container
+    }
+
+
+}

src/test/groovy/com/eficode/devstack/container/impl/DirectorySyncerTest.groovy

@@ -0,0 +1,138 @@
+package com.eficode.devstack.container.impl
+
+import com.eficode.devstack.DevStackSpec
+import com.eficode.devstack.util.DirectorySyncer
+import de.gesellix.docker.remote.api.ContainerInspectResponse
+import de.gesellix.docker.remote.api.MountPoint
+import org.slf4j.LoggerFactory
+
+
+class DirectorySyncerTest extends DevStackSpec {
+
+
+    def setupSpec() {
+
+        DevStackSpec.log = LoggerFactory.getLogger(this.class)
+
+        cleanupContainerNames = ["DirectorySyncer", "DirectorySyncer1", "DirectorySyncer2", "DirectorySyncer-companion", "DirectorySyncer1-companion"]
+        cleanupContainerPorts = []
+
+        disableCleanup = false
+
+
+    }
+
+
+    Boolean volumeExists(String volumeName) {
+
+        Boolean result = dockerClient.volumes().content?.volumes?.any { it.name == volumeName }
+        return result
+
+    }
+
+    def "Test createSyncToVolume"() {
+
+        setup:
+        log.info("Testing createSyncToVolume")
+        File srcDir1 = File.createTempDir("srcDir1")
+        log.debug("\tCreated Engine local temp dir:" + srcDir1.canonicalPath)
+        File srcDir2 = File.createTempDir("srcDir2")
+        log.debug("\tCreated Engine local temp dir:" + srcDir2.canonicalPath)
+
+        String uniqueVolumeName = "syncVolume" + System.currentTimeMillis().toString().takeRight(3)
+        !volumeExists(uniqueVolumeName) ?: dockerClient.rmVolume(uniqueVolumeName)
+        log.debug("\tWill use sync to Docker volume:" + uniqueVolumeName)
+
+
+        when: "When creating syncer"
+
+        assert !volumeExists(uniqueVolumeName): "Destination volume already exists"
+        DirectorySyncer syncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "-avh --delete", dockerRemoteHost, dockerCertPath )
+        log.info("\tCreated sync container: ${syncer.containerName} (${syncer.shortId})")
+        ContainerInspectResponse containerInspect = syncer.inspectContainer()
+
+
+        then: "I should have the two bind mounts and one volume mount"
+        assert syncer.running: "Syncer container is not running"
+        log.debug("\tContainer is running")
+        assert containerInspect.mounts.any { it.destination == "/mnt/src/${srcDir1.name}".toString() && it.RW == false }
+        log.debug("\tContainer has mounted the first src dir")
+        assert containerInspect.mounts.any { it.destination == "/mnt/src/${srcDir2.name}".toString() && it.RW == false }
+        log.debug("\tContainer has mounted the second src dir")
+        assert containerInspect.mounts.any { it.type == MountPoint.Type.Volume && it.RW }
+        assert dockerClient.getVolumesWithName(uniqueVolumeName).size(): "Destination volume was not created"
+        log.debug("\tContainer has mounted the expected destination volume")
+
+        when: "Creating files in src directories"
+        File srcFile1 = File.createTempFile("srcFile1", "temp", srcDir1)
+        srcFile1.text = System.currentTimeMillis()
+        log.debug("\tCreated file \"${srcFile1.name}\" in first src dir")
+        File srcFile2 = File.createTempFile("srcFile2", "temp", srcDir2)
+        srcFile2.text = System.currentTimeMillis() + new Random().nextInt()
+        log.debug("\tCreated file \"${srcFile2.name}\" in second src dir")
+
+        then: "The sync container should see new source files, and sync them"
+        syncer.runBashCommandInContainer("cat /mnt/src/${srcDir1.name}/${srcFile1.name}").toString().contains(srcFile1.text)
+        syncer.runBashCommandInContainer("cat /mnt/src/${srcDir2.name}/${srcFile2.name}").toString().contains(srcFile2.text)
+        log.debug("\tContainer sees the source files")
+        sleep(2000)//Wait for sync
+        syncer.runBashCommandInContainer("cat /mnt/dest/${srcFile1.name}").toString().contains(srcFile1.text)
+        syncer.runBashCommandInContainer("cat /mnt/dest/${srcFile2.name}").toString().contains(srcFile2.text)
+        log.debug("\tContainer successfully synced the files to destination dir")
+
+        when: "Creating a recursive file"
+        File recursiveFile = new File(srcDir1.canonicalPath + "/subDir/subFile.temp").createParentDirectories()
+        recursiveFile.createNewFile()
+        recursiveFile.text = System.nanoTime()
+        log.info("\tCreate recursive file:" + recursiveFile.canonicalPath)
+
+        then: "The sync container should see the new source file, and sync it to a new recursive dir"
+        sleep(2000)
+        syncer.runBashCommandInContainer("cat /mnt/dest/subDir/subFile.temp").toString().contains(recursiveFile.text)
+        log.info("\t\tFile was successfully synced")
+
+
+        /**
+         inotify does not appear to successfully detect deletions
+         Files will however be deleted once a create/update is detected
+         */
+        when: "Deleting first source file and updating second source file"
+        assert srcFile1.delete(): "Error deleting source file:" + srcFile1.canonicalPath
+        srcFile2.text = "UPDATED FILE"
+        log.debug("\tUpdating AND deleting src files")
+
+        then: "The file should be removed from destination dir"
+        sleep(2000)
+        !syncer.runBashCommandInContainer("cat /mnt/dest/${srcFile1.name} && echo Status: \$?").toString().containsIgnoreCase("Status: 0")
+        syncer.runBashCommandInContainer("cat /mnt/dest/${srcFile2.name}").toString().containsIgnoreCase(srcFile2.text)
+        log.debug("\t\tContainer successfully synced the changes")
+
+        when:"Creating a new container and attaching it to the synced volume"
+        AlpineContainer secondContainer = new AlpineContainer(dockerRemoteHost, dockerCertPath)
+        secondContainer.containerName = syncer.containerName  + "-companion"
+        secondContainer.prepareVolumeMount(uniqueVolumeName, "/mnt/syncDir", false)
+        secondContainer.createSleepyContainer()
+
+
+        then:"The second container should see the still remaining synced files"
+        assert secondContainer.startContainer() : "Error creating/staring second container"
+        log.info("\tCreated an started second container ${secondContainer.shortId}")
+        assert secondContainer.mounts.any { it.type == MountPoint.Type.Volume && it.RW == true} : "Second container did not mount the shared volume"
+        log.info("\tSecond container was attached to volume:" + uniqueVolumeName)
+        log.info("\tChecking that second container can access synced file:" + " /mnt/syncDir/${srcFile2.name}" )
+        assert secondContainer.runBashCommandInContainer("cat /mnt/syncDir/${srcFile2.name}").toString().containsIgnoreCase(srcFile2.text) : "Error reading synced file in second container:" + " /mnt/syncDir/${srcFile2.name}"
+
+        log.info("\tChecking that second container can access recursive synced file")
+        assert secondContainer.runBashCommandInContainer("cat /mnt/syncDir/subDir/subFile.temp").toString().contains(recursiveFile.text)
+        log.info("\t\tContainer can access that file")
+        cleanup:
+        assert syncer.stopAndRemoveContainer()
+        assert secondContainer?.stopAndRemoveContainer()
+        srcDir1.deleteDir()
+        srcDir2.deleteDir()
+        dockerClient.rmVolume(containerInspect.mounts.find { it.type == MountPoint.Type.Volume }.name)
+
+    }
+
+
+}