diff --git a/plugins/nf-nomad/build.gradle b/plugins/nf-nomad/build.gradle index c4badfd..4a34108 100644 --- a/plugins/nf-nomad/build.gradle +++ b/plugins/nf-nomad/build.gradle @@ -95,5 +95,6 @@ dependencies { // use JUnit 5 platform test { useJUnitPlatform() + jvmArgs '--add-opens=java.base/java.lang=ALL-UNNAMED' } diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/NomadConfig.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/NomadConfig.groovy index 5c3cb2e..7f181f0 100644 --- a/plugins/nf-nomad/src/main/nextflow/nomad/NomadConfig.groovy +++ b/plugins/nf-nomad/src/main/nextflow/nomad/NomadConfig.groovy @@ -28,13 +28,14 @@ import groovy.util.logging.Slf4j @Slf4j @CompileStatic class NomadConfig { + final static protected API_VERSION = "v1" final NomadClientOpts clientOpts final NomadJobOpts jobOpts NomadConfig(Map nomadConfigMap) { - clientOpts = new NomadClientOpts((nomadConfigMap.client ?: Collections.emptyMap()) as Map) - jobOpts = new NomadJobOpts((nomadConfigMap.jobs ?: Collections.emptyMap()) as Map) + clientOpts = new NomadClientOpts((nomadConfigMap?.client ?: Collections.emptyMap()) as Map) + jobOpts = new NomadJobOpts((nomadConfigMap?.jobs ?: Collections.emptyMap()) as Map) } class NomadClientOpts{ @@ -42,7 +43,10 @@ class NomadConfig { final String token NomadClientOpts(Map nomadClientOpts){ - address = (nomadClientOpts.address?.toString() ?: "http://127.0.0.1:4646")+"/v1" + def tmp = (nomadClientOpts.address?.toString() ?: "http://127.0.0.1:4646") + if( !tmp.endsWith("/")) + tmp +="/" + this.address = tmp + API_VERSION token = nomadClientOpts.token ?: null } } @@ -57,8 +61,13 @@ class NomadConfig { NomadJobOpts(Map nomadJobOpts){ deleteOnCompletion = nomadJobOpts.containsKey("deleteOnCompletion") ? nomadJobOpts.deleteOnCompletion : false - datacenters = (nomadJobOpts.containsKey("datacenters") ? - nomadJobOpts.datacenters.toString().split(",") : List.of("dc1")) as List + if( nomadJobOpts.containsKey("datacenters") ) { + datacenters = ((nomadJobOpts.datacenters instanceof List ? + nomadJobOpts.datacenters : nomadJobOpts.datacenters.toString().split(",")) + as List).findAll{it.size()}.unique() + }else{ + datacenters = [] + } region = nomadJobOpts.region ?: null namespace = nomadJobOpts.namespace ?: null dockerVolume = nomadJobOpts.dockerVolume ?: null diff --git a/plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy b/plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy index 7b5200f..e210645 100644 --- a/plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy +++ b/plugins/nf-nomad/src/main/nextflow/nomad/executor/NomadService.groovy @@ -111,7 +111,7 @@ class NomadService implements Closeable{ String state(String jobId){ JobSummary summary = jobsApi.getJobSummary(jobId, config.jobOpts.region, config.jobOpts.namespace, null, null, null, null, null, null, null) - TaskGroupSummary taskGroupSummary = summary.summary.values().first() + TaskGroupSummary taskGroupSummary = summary?.summary?.values()?.first() switch (taskGroupSummary){ case {taskGroupSummary?.starting }: return TaskGroupSummary.SERIALIZED_NAME_STARTING diff --git a/plugins/nf-nomad/src/test/nextflow/nomad/NomadConfigSpec.groovy b/plugins/nf-nomad/src/test/nextflow/nomad/NomadConfigSpec.groovy new file mode 100644 index 0000000..e85f117 --- /dev/null +++ b/plugins/nf-nomad/src/test/nextflow/nomad/NomadConfigSpec.groovy @@ -0,0 +1,122 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.nomad + +import spock.lang.Specification + +/** + * Unit test for Nomad Config + * + * @author : Jorge Aguilera + */ +class NomadConfigSpec extends Specification { + + void "should create a default config"() { + given: + def config = new NomadConfig() + + expect: + config.jobOpts + config.clientOpts + } + + void "should use localhost as default address"() { + given: + def config = new NomadConfig([:]) + + expect: + config.clientOpts.address == "http://127.0.0.1:4646/v1" + } + + void "should use address if provided"() { + given: + def config = new NomadConfig([ + client: [address: "http://nomad"] + ]) + + expect: + config.clientOpts.address == "http://nomad/v1" + } + + void "should normalize address if provided"() { + given: + def config = new NomadConfig([ + client: [address: "http://nomad/"] + ]) + + expect: + config.clientOpts.address == "http://nomad/v1" + } + + void "should use token if provided"() { + given: + def config = new NomadConfig([ + client: [token: "theToken"] + ]) + + expect: + config.clientOpts.token == "theToken" + } + + void "should use an empty list if no datacenters is provided"() { + given: + def config = new NomadConfig([ + jobs: [:] + ]) + + expect: + !config.jobOpts.datacenters.size() + } + + void "should use datacenters #dc with size #size if provided"() { + given: + def config = new NomadConfig([ + jobs: [datacenters: dc] + ]) + + expect: + config.jobOpts.datacenters.size() == size + + where: + dc | size + [] | 0 + "dc1" | 1 + ['dc1'] | 1 + "dc1,dc2" | 2 + ['dc1', 'dc2'] | 2 + ['dc1', 'dc1'] | 1 + } + + void "should use region if provided"() { + given: + def config = new NomadConfig([ + jobs: [region: "theRegion"] + ]) + + expect: + config.jobOpts.region == "theRegion" + } + + void "should use namespace if provided"() { + given: + def config = new NomadConfig([ + jobs: [namespace: "namespace"] + ]) + + expect: + config.jobOpts.namespace == "namespace" + } +} diff --git a/plugins/nf-nomad/src/test/nextflow/nomad/executor/NomadServiceSpec.groovy b/plugins/nf-nomad/src/test/nextflow/nomad/executor/NomadServiceSpec.groovy new file mode 100644 index 0000000..9fdbc73 --- /dev/null +++ b/plugins/nf-nomad/src/test/nextflow/nomad/executor/NomadServiceSpec.groovy @@ -0,0 +1,190 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nextflow.nomad.executor + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import nextflow.nomad.NomadConfig +import okhttp3.mockwebserver.MockResponse +import okhttp3.mockwebserver.MockWebServer +import spock.lang.Specification + +/** + * Unit test for Nomad Service + * + * Validate requests using a Mock WebServer + * + * @author : Jorge Aguilera + */ +class NomadServiceSpec extends Specification{ + + MockWebServer mockWebServer + + def setup() { + mockWebServer = new MockWebServer() + mockWebServer.start() + } + + def cleanup() { + mockWebServer.shutdown() + } + + void "submit a task"(){ + given: + def config = new NomadConfig( + client:[ + address : "http://${mockWebServer.hostName}:${mockWebServer.port}" + ] + ) + def service = new NomadService(config) + + String id = "theId" + String name = "theName" + String image = "theImage" + List args = ["theCommand", "theArgs"] + String workingDir = "theWorkingDir" + Mapenv = [test:"test"] + + mockWebServer.enqueue(new MockResponse() + .setBody(JsonOutput.toJson(["EvalID":"test"]).toString()) + .addHeader("Content-Type", "application/json")); + when: + + def idJob = service.submitTask(id, name, image, args, workingDir,env) + def recordedRequest = mockWebServer.takeRequest(); + def body = new JsonSlurper().parseText(recordedRequest.body.readUtf8()) + + then: + idJob + + and: + recordedRequest.method == "POST" + recordedRequest.path == "/v1/jobs" + + and: + body.Job + body.Job.ID == id + body.Job.Name == name + body.Job.Datacenters == [] + body.Job.Type == "batch" + body.Job.TaskGroups.size() == 1 + body.Job.TaskGroups[0].Name == "group" + body.Job.TaskGroups[0].Tasks.size() == 1 + body.Job.TaskGroups[0].Tasks[0].Name == "nf-task" + body.Job.TaskGroups[0].Tasks[0].Driver == "docker" + body.Job.TaskGroups[0].Tasks[0].Config.image == image + body.Job.TaskGroups[0].Tasks[0].Config.work_dir == workingDir + body.Job.TaskGroups[0].Tasks[0].Config.command == args[0] + body.Job.TaskGroups[0].Tasks[0].Config.args == args.drop(1) + + !body.Job.TaskGroups[0].Tasks[0].Config.mount + } + + void "submit a task with docker volume"(){ + given: + def config = new NomadConfig( + client:[ + address : "http://${mockWebServer.hostName}:${mockWebServer.port}" + ], + jobs:[ + dockerVolume:'test' + ] + ) + def service = new NomadService(config) + + String id = "theId" + String name = "theName" + String image = "theImage" + List args = ["theCommand", "theArgs"] + String workingDir = "a/b/c" + Mapenv = [test:"test"] + + mockWebServer.enqueue(new MockResponse() + .setBody(JsonOutput.toJson(["EvalID":"test"]).toString()) + .addHeader("Content-Type", "application/json")); + when: + + def idJob = service.submitTask(id, name, image, args, workingDir,env) + def recordedRequest = mockWebServer.takeRequest(); + def body = new JsonSlurper().parseText(recordedRequest.body.readUtf8()) + + then: + idJob + + and: + recordedRequest.method == "POST" + recordedRequest.path == "/v1/jobs" + + and: + body.Job + body.Job.ID == id + body.Job.Name == name + body.Job.Datacenters == [] + body.Job.Type == "batch" + body.Job.TaskGroups.size() == 1 + body.Job.TaskGroups[0].Name == "group" + body.Job.TaskGroups[0].Tasks.size() == 1 + body.Job.TaskGroups[0].Tasks[0].Name == "nf-task" + body.Job.TaskGroups[0].Tasks[0].Driver == "docker" + body.Job.TaskGroups[0].Tasks[0].Config.image == image + body.Job.TaskGroups[0].Tasks[0].Config.work_dir == workingDir + body.Job.TaskGroups[0].Tasks[0].Config.command == args[0] + body.Job.TaskGroups[0].Tasks[0].Config.args == args.drop(1) + + body.Job.TaskGroups[0].Tasks[0].Config.mount == [type:"volume", target:"a", source:"test", readonly:false] + } + + void "should check the state"(){ + given: + def config = new NomadConfig( + client:[ + address : "http://${mockWebServer.hostName}:${mockWebServer.port}" + ] + ) + def service = new NomadService(config) + + when: + mockWebServer.enqueue(new MockResponse() + .addHeader("Content-Type", "application/json")); + + def state = service.state("theId") + def recordedRequest = mockWebServer.takeRequest(); + + then: + recordedRequest.method == "GET" + recordedRequest.path == "/v1/job/theId/summary" + + and: + state == "Unknown" + + when: + mockWebServer.enqueue(new MockResponse() + .setBody(JsonOutput.toJson(["JobID":"test","Summary":[ + test:[Starting:1] + ]]).toString()) + .addHeader("Content-Type", "application/json")); + + state = service.state("theId") + recordedRequest = mockWebServer.takeRequest(); + + then: + recordedRequest.method == "GET" + recordedRequest.path == "/v1/job/theId/summary" + + and: + state == "Starting" + + } +} diff --git a/plugins/nf-nomad/src/test/nextflow/nomad/executor/NomadTaskHandlerSpec.groovy b/plugins/nf-nomad/src/test/nextflow/nomad/executor/NomadTaskHandlerSpec.groovy new file mode 100644 index 0000000..dd5f4af --- /dev/null +++ b/plugins/nf-nomad/src/test/nextflow/nomad/executor/NomadTaskHandlerSpec.groovy @@ -0,0 +1,85 @@ +/* + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package nextflow.nomad.executor + +import nextflow.exception.ProcessSubmitException +import nextflow.executor.Executor +import nextflow.nomad.NomadConfig +import nextflow.processor.TaskBean +import nextflow.processor.TaskProcessor +import nextflow.processor.TaskRun +import nextflow.processor.TaskStatus +import spock.lang.Specification + +import java.nio.file.Files +import java.nio.file.Path + +/** + * Unit test for Nomad Task Handler + * + * + * @author : Jorge Aguilera + */ +class NomadTaskHandlerSpec extends Specification{ + + void "a task should have a container"(){ + given: + def mockTask = Mock(TaskRun){ + getWorkDir() >> Path.of(".") + getContainer() >> null + getProcessor() >> Mock(TaskProcessor) + } + def mockConfig = Mock(NomadConfig) + def mockService = Mock(NomadService) + def taskHandler = new NomadTaskHandler(mockTask, mockConfig, mockService) + + when: + taskHandler.submitTask() + + then: + thrown(ProcessSubmitException) + } + + void "a task should be created"(){ + given: + def workDir = Files.createTempDirectory("nf") + new File(workDir.toFile(), TaskRun.CMD_INFILE).text = "infile" + + def mockTask = Mock(TaskRun){ + getWorkDir() >> workDir + getContainer() >> "ubuntu" + getProcessor() >> Mock(TaskProcessor){ + getExecutor() >> Mock(Executor){ + isFusionEnabled() >> false + } + } + toTaskBean() >> Mock(TaskBean){ + getWorkDir() >> workDir + getScript() >> "theScript" + getShell() >> ["bash"] + getInputFiles() >> [:] + } + } + def mockConfig = Mock(NomadConfig) + def mockService = Mock(NomadService) + def taskHandler = new NomadTaskHandler(mockTask, mockConfig, mockService) + + when: + def ret = taskHandler.submitTask() + + then: + ret == TaskStatus.SUBMITTED.name() + } +}