Skip to content

Commit e3efe3d

Browse files
farthinderfarthinder
authored
2.3.19 (#40)
* Container.groovy * Added prepareDevice, hasDevice, prepareCapability, hasCapability ContainerTest.groovy * Added tests for devices and capabilities * Tweaking of documentation * Container.groovy * Solved bug that caused error to be thrown when stopping and removing container * prepareCustomEnvVar() now adds envs if other envs are present DirectorySyncer.groovy * New Util container intended to sync files to Docker volume * WIP, needs cleanup and documentation * DirectorySyncer.groovy * Cleaned up, added rsyncOptions * DirectorySyncerTest.groovy * Tweaking of test * Bumped to 2.3.19
1 parent 22cf353 commit e3efe3d

File tree

4 files changed

+296
-5
lines changed

4 files changed

+296
-5
lines changed

‎pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
<groupId>com.eficode</groupId>
88
<artifactId>devstack</artifactId>
9-
<version>2.3.18</version>
9+
<version>2.3.19</version>
1010
<packaging>jar</packaging>
1111

1212
<name>DevStack</name>

‎src/main/groovy/com/eficode/devstack/container/Container.groovy

+8-4
Original file line numberDiff line numberDiff line change
@@ -437,10 +437,14 @@ trait Container {
437437

438438
if (self.containerId) {
439439

440-
dockerClient.stop(self.containerId, timeoutS)
441-
if (self.status() == ContainerState.Status.Running) {
442-
dockerClient.kill(self.containerId)
440+
441+
if (self.isRunning()) {
442+
dockerClient.stop(self.containerId, timeoutS)
443+
if (self.isRunning()) {
444+
dockerClient.kill(self.containerId)
445+
}
443446
}
447+
444448
dockerClient.rm(self.containerId)
445449

446450

@@ -1055,7 +1059,7 @@ trait Container {
10551059

10561060
assert hasNeverBeenStarted(): "Error, cant set custom environment variables after creating container"
10571061

1058-
self.customEnvVar = keyVar
1062+
self.customEnvVar.addAll(keyVar.collect {it.toString()})
10591063
}
10601064

10611065

‎src/main/groovy/com/eficode/devstack/util/DirectorySyncer.groovy

+149
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
package com.eficode.devstack.util
2+
3+
import com.eficode.devstack.container.Container
4+
import de.gesellix.docker.client.EngineResponseContent
5+
import de.gesellix.docker.remote.api.ContainerSummary
6+
import de.gesellix.docker.remote.api.Volume
7+
import org.slf4j.Logger
8+
9+
import javax.naming.NameNotFoundException
10+
11+
12+
class DirectorySyncer implements Container {
13+
14+
String containerName = "DirectorySyncer"
15+
String containerMainPort = null
16+
String containerImage = "alpine"
17+
String containerImageTag = "latest"
18+
String defaultShell = "/bin/sh"
19+
20+
DirectorySyncer(String dockerHost = "", String dockerCertPath = "") {
21+
if (dockerHost && dockerCertPath) {
22+
assert setupSecureRemoteConnection(dockerHost, dockerCertPath): "Error setting up secure remote docker connection"
23+
}
24+
}
25+
26+
static String getSyncScript(String rsyncOptions = "-avh") {
27+
28+
return """
29+
30+
apk update
31+
apk add inotify-tools rsync tini
32+
#apt update
33+
#apt install -y inotify-tools rsync tini
34+
35+
if [ -z "\$(which inotifywait)" ]; then
36+
echo "inotifywait not installed."
37+
echo "In most distros, it is available in the inotify-tools package."
38+
exit 1
39+
fi
40+
41+
42+
function execute() {
43+
eval "\$@"
44+
rsync $rsyncOptions /mnt/src/*/ /mnt/dest/
45+
}
46+
47+
execute""
48+
49+
inotifywait --recursive --monitor --format "%e %w%f" \\
50+
--event modify,create,delete,moved_from,close_write /mnt/src \\
51+
| while read changed; do
52+
echo "\$changed"
53+
execute "\$@"
54+
done
55+
""".stripIndent()
56+
57+
}
58+
59+
String getAvailableContainerName(String prefix = "DirectorySyncer") {
60+
61+
Integer suffixNr = null
62+
String availableName = null
63+
64+
ArrayList<ContainerSummary> containers = dockerClient.ps().content
65+
66+
67+
while (!availableName) {
68+
69+
String containerName = prefix + (suffixNr ?: "")
70+
if (!containers.any { container -> container.names.any { name -> name.equalsIgnoreCase("/" + containerName) } }) {
71+
availableName = containerName
72+
} else if (suffixNr > 100) {
73+
throw new NameNotFoundException("Could not find avaialble name for container, last test was: $containerName")
74+
} else {
75+
suffixNr ? suffixNr++ : (suffixNr = 1)
76+
}
77+
}
78+
79+
return availableName
80+
81+
}
82+
83+
/**
84+
* <pre>
85+
* Creates a Util container:
86+
* 1. Listens for file changes in one or more docker engine src paths (hostAbsSourcePaths)
87+
* 2. If changes are detected rsync is triggered
88+
* 3. Rsync detects changes and sync them to destVolumeName
89+
*
90+
* The root of all the srcPaths will be combined and synced to destVolume,
91+
* ex:
92+
* ../srcPath1/file1.txt
93+
* ../srcPath2/file2.txt
94+
* ../srcPath2/subdir/file3.txt
95+
* Will give:
96+
* destVolume/file1.txt
97+
* destVolume/file2.txt
98+
* destVolume/subdir/file3.txt
99+
* </pre>
100+
*
101+
* <b>Known Issues</b>
102+
* <pre>
103+
* Delete events are not properly detected and triggered on,
104+
* thus any such actions will only be reflected after
105+
* subsequent create/update events.
106+
* </pre>
107+
* @param hostAbsSourcePaths A list of one or more src dirs to sync from
108+
* @param destVolumeName A docker volume to sync to, if it does not exist it will be created
109+
* @param rsyncOptions Options to use when running rsync, ie: rsync $rsyncOptions /mnt/src/*\/ /mnt/dest/<p>
110+
* example: -avh --delete
111+
* @param dockerHost Docker host to run on
112+
* @param dockerCertPath Docker certs to use
113+
* @return
114+
*/
115+
static DirectorySyncer createSyncToVolume(ArrayList<String> hostAbsSourcePaths, String destVolumeName, String rsyncOptions = "-avh", String dockerHost = "", String dockerCertPath = "") {
116+
117+
DirectorySyncer container = new DirectorySyncer(dockerHost, dockerCertPath)
118+
Logger log = container.log
119+
120+
container.containerName = container.getAvailableContainerName()
121+
container.prepareCustomEnvVar(["syncScript=${getSyncScript(rsyncOptions)}"])
122+
123+
Volume volume = container.dockerClient.getVolumesWithName(destVolumeName).find { true }
124+
125+
if (volume) {
126+
log.debug("\tFound existing volume:" + volume.name)
127+
} else {
128+
log.debug("\tCreating new volume $destVolumeName")
129+
EngineResponseContent<Volume> volumeResponse = container.dockerClient.createVolume(destVolumeName)
130+
volume = volumeResponse?.content
131+
assert volume: "Error creating volume $destVolumeName, " + volumeResponse?.getStatus()?.text
132+
log.debug("\t\tCreated volume:" + volume.name)
133+
}
134+
135+
container.prepareVolumeMount(volume.name, "/mnt/dest/", false)
136+
137+
hostAbsSourcePaths.each { srcPath ->
138+
String srcDirName = srcPath.substring(srcPath.lastIndexOf("/") + 1)
139+
container.prepareBindMount(srcPath, "/mnt/src/$srcDirName", true)
140+
}
141+
142+
container.createContainer(["/bin/sh", "-c", "echo \"\$syncScript\" > /syncScript.sh && /bin/sh syncScript.sh"], [])
143+
container.startContainer()
144+
145+
return container
146+
}
147+
148+
149+
}

‎src/test/groovy/com/eficode/devstack/container/impl/DirectorySyncerTest.groovy

+138
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
package com.eficode.devstack.container.impl
2+
3+
import com.eficode.devstack.DevStackSpec
4+
import com.eficode.devstack.util.DirectorySyncer
5+
import de.gesellix.docker.remote.api.ContainerInspectResponse
6+
import de.gesellix.docker.remote.api.MountPoint
7+
import org.slf4j.LoggerFactory
8+
9+
10+
class DirectorySyncerTest extends DevStackSpec {
11+
12+
13+
def setupSpec() {
14+
15+
DevStackSpec.log = LoggerFactory.getLogger(this.class)
16+
17+
cleanupContainerNames = ["DirectorySyncer", "DirectorySyncer1", "DirectorySyncer2", "DirectorySyncer-companion", "DirectorySyncer1-companion"]
18+
cleanupContainerPorts = []
19+
20+
disableCleanup = false
21+
22+
23+
}
24+
25+
26+
Boolean volumeExists(String volumeName) {
27+
28+
Boolean result = dockerClient.volumes().content?.volumes?.any { it.name == volumeName }
29+
return result
30+
31+
}
32+
33+
def "Test createSyncToVolume"() {
34+
35+
setup:
36+
log.info("Testing createSyncToVolume")
37+
File srcDir1 = File.createTempDir("srcDir1")
38+
log.debug("\tCreated Engine local temp dir:" + srcDir1.canonicalPath)
39+
File srcDir2 = File.createTempDir("srcDir2")
40+
log.debug("\tCreated Engine local temp dir:" + srcDir2.canonicalPath)
41+
42+
String uniqueVolumeName = "syncVolume" + System.currentTimeMillis().toString().takeRight(3)
43+
!volumeExists(uniqueVolumeName) ?: dockerClient.rmVolume(uniqueVolumeName)
44+
log.debug("\tWill use sync to Docker volume:" + uniqueVolumeName)
45+
46+
47+
when: "When creating syncer"
48+
49+
assert !volumeExists(uniqueVolumeName): "Destination volume already exists"
50+
DirectorySyncer syncer = DirectorySyncer.createSyncToVolume([srcDir1.canonicalPath, srcDir2.canonicalPath], uniqueVolumeName, "-avh --delete", dockerRemoteHost, dockerCertPath )
51+
log.info("\tCreated sync container: ${syncer.containerName} (${syncer.shortId})")
52+
ContainerInspectResponse containerInspect = syncer.inspectContainer()
53+
54+
55+
then: "I should have the two bind mounts and one volume mount"
56+
assert syncer.running: "Syncer container is not running"
57+
log.debug("\tContainer is running")
58+
assert containerInspect.mounts.any { it.destination == "/mnt/src/${srcDir1.name}".toString() && it.RW == false }
59+
log.debug("\tContainer has mounted the first src dir")
60+
assert containerInspect.mounts.any { it.destination == "/mnt/src/${srcDir2.name}".toString() && it.RW == false }
61+
log.debug("\tContainer has mounted the second src dir")
62+
assert containerInspect.mounts.any { it.type == MountPoint.Type.Volume && it.RW }
63+
assert dockerClient.getVolumesWithName(uniqueVolumeName).size(): "Destination volume was not created"
64+
log.debug("\tContainer has mounted the expected destination volume")
65+
66+
when: "Creating files in src directories"
67+
File srcFile1 = File.createTempFile("srcFile1", "temp", srcDir1)
68+
srcFile1.text = System.currentTimeMillis()
69+
log.debug("\tCreated file \"${srcFile1.name}\" in first src dir")
70+
File srcFile2 = File.createTempFile("srcFile2", "temp", srcDir2)
71+
srcFile2.text = System.currentTimeMillis() + new Random().nextInt()
72+
log.debug("\tCreated file \"${srcFile2.name}\" in second src dir")
73+
74+
then: "The sync container should see new source files, and sync them"
75+
syncer.runBashCommandInContainer("cat /mnt/src/${srcDir1.name}/${srcFile1.name}").toString().contains(srcFile1.text)
76+
syncer.runBashCommandInContainer("cat /mnt/src/${srcDir2.name}/${srcFile2.name}").toString().contains(srcFile2.text)
77+
log.debug("\tContainer sees the source files")
78+
sleep(2000)//Wait for sync
79+
syncer.runBashCommandInContainer("cat /mnt/dest/${srcFile1.name}").toString().contains(srcFile1.text)
80+
syncer.runBashCommandInContainer("cat /mnt/dest/${srcFile2.name}").toString().contains(srcFile2.text)
81+
log.debug("\tContainer successfully synced the files to destination dir")
82+
83+
when: "Creating a recursive file"
84+
File recursiveFile = new File(srcDir1.canonicalPath + "/subDir/subFile.temp").createParentDirectories()
85+
recursiveFile.createNewFile()
86+
recursiveFile.text = System.nanoTime()
87+
log.info("\tCreate recursive file:" + recursiveFile.canonicalPath)
88+
89+
then: "The sync container should see the new source file, and sync it to a new recursive dir"
90+
sleep(2000)
91+
syncer.runBashCommandInContainer("cat /mnt/dest/subDir/subFile.temp").toString().contains(recursiveFile.text)
92+
log.info("\t\tFile was successfully synced")
93+
94+
95+
/**
96+
inotify does not appear to successfully detect deletions
97+
Files will however be deleted once a create/update is detected
98+
*/
99+
when: "Deleting first source file and updating second source file"
100+
assert srcFile1.delete(): "Error deleting source file:" + srcFile1.canonicalPath
101+
srcFile2.text = "UPDATED FILE"
102+
log.debug("\tUpdating AND deleting src files")
103+
104+
then: "The file should be removed from destination dir"
105+
sleep(2000)
106+
!syncer.runBashCommandInContainer("cat /mnt/dest/${srcFile1.name} && echo Status: \$?").toString().containsIgnoreCase("Status: 0")
107+
syncer.runBashCommandInContainer("cat /mnt/dest/${srcFile2.name}").toString().containsIgnoreCase(srcFile2.text)
108+
log.debug("\t\tContainer successfully synced the changes")
109+
110+
when:"Creating a new container and attaching it to the synced volume"
111+
AlpineContainer secondContainer = new AlpineContainer(dockerRemoteHost, dockerCertPath)
112+
secondContainer.containerName = syncer.containerName + "-companion"
113+
secondContainer.prepareVolumeMount(uniqueVolumeName, "/mnt/syncDir", false)
114+
secondContainer.createSleepyContainer()
115+
116+
117+
then:"The second container should see the still remaining synced files"
118+
assert secondContainer.startContainer() : "Error creating/staring second container"
119+
log.info("\tCreated an started second container ${secondContainer.shortId}")
120+
assert secondContainer.mounts.any { it.type == MountPoint.Type.Volume && it.RW == true} : "Second container did not mount the shared volume"
121+
log.info("\tSecond container was attached to volume:" + uniqueVolumeName)
122+
log.info("\tChecking that second container can access synced file:" + " /mnt/syncDir/${srcFile2.name}" )
123+
assert secondContainer.runBashCommandInContainer("cat /mnt/syncDir/${srcFile2.name}").toString().containsIgnoreCase(srcFile2.text) : "Error reading synced file in second container:" + " /mnt/syncDir/${srcFile2.name}"
124+
125+
log.info("\tChecking that second container can access recursive synced file")
126+
assert secondContainer.runBashCommandInContainer("cat /mnt/syncDir/subDir/subFile.temp").toString().contains(recursiveFile.text)
127+
log.info("\t\tContainer can access that file")
128+
cleanup:
129+
assert syncer.stopAndRemoveContainer()
130+
assert secondContainer?.stopAndRemoveContainer()
131+
srcDir1.deleteDir()
132+
srcDir2.deleteDir()
133+
dockerClient.rmVolume(containerInspect.mounts.find { it.type == MountPoint.Type.Volume }.name)
134+
135+
}
136+
137+
138+
}

0 commit comments

Comments
 (0)