diff --git a/.gitignore b/.gitignore index ca78315e..aab45fc5 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ release.properties *.iml **/dependency-reduced-pom.xml **/target +.DS_Store diff --git a/jclouds-plugin/pom.xml b/jclouds-plugin/pom.xml index 3dec6732..88f0a430 100644 --- a/jclouds-plugin/pom.xml +++ b/jclouds-plugin/pom.xml @@ -45,6 +45,11 @@ opencsv 2.3 + + org.jenkins-ci.plugins + ssh-slaves + 1.9 + org.jenkins-ci.plugins @@ -244,4 +249,4 @@ - \ No newline at end of file + diff --git a/jclouds-plugin/src/main/java/jenkins/plugins/jclouds/compute/JCloudsCloud.java b/jclouds-plugin/src/main/java/jenkins/plugins/jclouds/compute/JCloudsCloud.java index a49d66c8..d96979ce 100644 --- a/jclouds-plugin/src/main/java/jenkins/plugins/jclouds/compute/JCloudsCloud.java +++ b/jclouds-plugin/src/main/java/jenkins/plugins/jclouds/compute/JCloudsCloud.java @@ -13,6 +13,7 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; import java.util.logging.Logger; import com.google.inject.Module; @@ -115,7 +116,7 @@ public JCloudsCloud(final String profile, final String providerName, final Strin this.retentionTime = retentionTime; this.scriptTimeout = scriptTimeout; this.startTimeout = startTimeout; - this.templates = Objects.firstNonNull(templates, Collections.emptyList()); + this.templates = Objects.firstNonNull(templates, Collections. emptyList()); this.zones = Util.fixEmptyAndTrim(zones); readResolve(); } @@ -130,11 +131,7 @@ protected Object readResolve() { * Get the retention time, defaulting to 30 minutes. */ public int getRetentionTime() { - if (retentionTime == 0) { - return 30; - } else { - return retentionTime; - } + return retentionTime == 0 ? 30 : retentionTime; } static final Iterable MODULES = ImmutableSet.of(new SshjSshClientModule(), new JDKLoggingModule() { @@ -188,44 +185,57 @@ public List getTemplates() { */ @Override public Collection provision(Label label, int excessWorkload) { - final JCloudsSlaveTemplate t = getTemplate(label); + final JCloudsSlaveTemplate template = getTemplate(label); + List plannedNodeList = new ArrayList(); + + while (excessWorkload > 0 && !Jenkins.getInstance().isQuietingDown() && !Jenkins.getInstance().isTerminating()) { - List r = new ArrayList(); - while (excessWorkload > 0 - && !Jenkins.getInstance().isQuietingDown() - && !Jenkins.getInstance().isTerminating()) { - if ((getRunningNodesCount() + r.size()) >= instanceCap) { + if ((getRunningNodesCount() + plannedNodeList.size()) >= instanceCap) { LOGGER.info("Instance cap reached while adding capacity for label " + ((label != null) ? label.toString() : "null")); break; // maxed out } - r.add(new PlannedNode(t.name, Computer.threadPoolForRemoting.submit(new Callable() { + plannedNodeList.add(new PlannedNode(template.name, Computer.threadPoolForRemoting.submit(new Callable() { public Node call() throws Exception { // TODO: record the output somewhere - JCloudsSlave s = t.provisionSlave(new StreamTaskListener(System.out)); - Hudson.getInstance().addNode(s); - // Cloud instances may have a long init script. If - // we declare - // the provisioning complete by returning without - // the connect - // operation, NodeProvisioner may decide that it - // still wants - // one more instance, because it sees that (1) all - // the slaves - // are offline (because it's still being launched) - // and - // (2) there's no capacity provisioned yet. - // - // deferring the completion of provisioning until - // the launch - // goes successful prevents this problem. - s.toComputer().connect(false).get(); - return s; + JCloudsSlave jcloudsSlave = template.provisionSlave(StreamTaskListener.fromStdout()); + Jenkins.getInstance().addNode(jcloudsSlave); + + /* Cloud instances may have a long init script. If we declare the provisioning complete by returning + without the connect operation, NodeProvisioner may decide that it still wants one more instance, + because it sees that (1) all the slaves are offline (because it's still being launched) and (2) + there's no capacity provisioned yet. Deferring the completion of provisioning until the launch goes + successful prevents this problem. */ + ensureLaunched(jcloudsSlave); + return jcloudsSlave; } - }), Util.tryParseNumber(t.numExecutors, 1).intValue())); - excessWorkload -= t.getNumExecutors(); + }), Util.tryParseNumber(template.numExecutors, 1).intValue())); + excessWorkload -= template.getNumExecutors(); + } + return plannedNodeList; + } + + private void ensureLaunched(JCloudsSlave jcloudsSlave) throws InterruptedException, ExecutionException { + Integer launchTimeoutSec = 5 * 60; + Computer computer = jcloudsSlave.toComputer(); + long startMoment = System.currentTimeMillis(); + while (computer.isOffline()) { + try { + LOGGER.info(String.format("Slave [%s] not connected yet", jcloudsSlave.getDisplayName())); + computer.connect(false).get(); + Thread.sleep(5000l); + } catch (InterruptedException e) { + LOGGER.warning(String.format("Error while launching slave: %s", e)); + } catch (ExecutionException e) { + LOGGER.warning(String.format("Error while launching slave: %s", e)); + } + + if ((System.currentTimeMillis() - startMoment) > 1000l * launchTimeoutSec) { + String message = String.format("Failed to connect to slave within timeout (%d s).", launchTimeoutSec); + LOGGER.warning(message); + throw new ExecutionException(new Throwable(message)); + } } - return r; } @Override @@ -315,7 +325,7 @@ public String getDisplayName() { } public FormValidation doTestConnection(@QueryParameter String providerName, @QueryParameter String identity, @QueryParameter String credential, - @QueryParameter String privateKey, @QueryParameter String endPointUrl, @QueryParameter String zones) throws IOException { + @QueryParameter String privateKey, @QueryParameter String endPointUrl, @QueryParameter String zones) throws IOException { if (identity == null) return FormValidation.error("Invalid (AccessId)."); if (credential == null) @@ -344,7 +354,7 @@ public FormValidation doTestConnection(@QueryParameter String providerName, @Que } catch (Exception ex) { result = FormValidation.error("Cannot connect to specified cloud, please check the identity and credentials: " + ex.getMessage()); } finally { - Closeables.close(ctx, true); + Closeables.close(ctx,true); } return result; } @@ -382,7 +392,7 @@ public ListBoxModel doFillProviderNameItems() { Thread.currentThread().setContextClassLoader(Apis.class.getClassLoader()); // TODO: apis need endpoints, providers don't; do something smarter // with this stuff :) - Builder builder = ImmutableSet.builder(); + Builder builder = ImmutableSet. builder(); builder.addAll(Iterables.transform(Apis.viewableAs(ComputeServiceContext.class), Apis.idFunction())); builder.addAll(Iterables.transform(Providers.viewableAs(ComputeServiceContext.class), Providers.idFunction())); Iterable supportedProviders = ImmutableSortedSet.copyOf(builder.build()); @@ -398,7 +408,7 @@ public AutoCompletionCandidates doAutoCompleteProviderName(@QueryParameter final Thread.currentThread().setContextClassLoader(Apis.class.getClassLoader()); // TODO: apis need endpoints, providers don't; do something smarter // with this stuff :) - Builder builder = ImmutableSet.builder(); + Builder builder = ImmutableSet. builder(); builder.addAll(Iterables.transform(Apis.viewableAs(ComputeServiceContext.class), Apis.idFunction())); builder.addAll(Iterables.transform(Providers.viewableAs(ComputeServiceContext.class), Providers.idFunction())); Iterable supportedProviders = builder.build(); diff --git a/jclouds-plugin/src/main/java/jenkins/plugins/jclouds/compute/JCloudsLauncher.java b/jclouds-plugin/src/main/java/jenkins/plugins/jclouds/compute/JCloudsLauncher.java index 05d00785..ce4ec2f0 100644 --- a/jclouds-plugin/src/main/java/jenkins/plugins/jclouds/compute/JCloudsLauncher.java +++ b/jclouds-plugin/src/main/java/jenkins/plugins/jclouds/compute/JCloudsLauncher.java @@ -2,10 +2,9 @@ import hudson.model.TaskListener; import hudson.model.Descriptor; -import hudson.model.Hudson; -import hudson.remoting.Channel; import hudson.slaves.ComputerLauncher; import hudson.slaves.SlaveComputer; +import hudson.plugins.sshslaves.SSHLauncher; import java.io.IOException; import java.io.PrintStream; @@ -13,11 +12,6 @@ import org.jclouds.compute.domain.NodeMetadata; import org.jclouds.domain.LoginCredentials; -import com.trilead.ssh2.Connection; -import com.trilead.ssh2.SCPClient; -import com.trilead.ssh2.ServerHostKeyVerifier; -import com.trilead.ssh2.Session; - /** * The launcher that launches the jenkins slave.jar on the Slave. Uses the SSHKeyPair configured in the cloud profile settings, and logs in to the server via * SSH, and starts the slave.jar. @@ -25,8 +19,6 @@ * @author Vijay Kiran */ public class JCloudsLauncher extends ComputerLauncher { - private final int FAILED = -1; - private final int SAMEUSER = 0; /** * Launch the Jenkins Slave on the SlaveComputer. @@ -41,110 +33,19 @@ public void launch(SlaveComputer computer, TaskListener listener) throws IOExcep PrintStream logger = listener.getLogger(); - final Connection bootstrapConn; - final Connection conn; - Connection cleanupConn = null; // java's code path analysis for final doesn't work that well. - boolean successful = false; final JCloudsSlave slave = (JCloudsSlave) computer.getNode(); - final LoginCredentials credentials = slave.getCredentials(); final NodeMetadata nodeMetadata = slave.getNodeMetaData(); + final String[] addresses = getConnectionAddresses(nodeMetadata, logger); + LoginCredentials credentials = slave.getCredentials(); - try { - bootstrapConn = connectToSsh(nodeMetadata, logger); - int bootstrapResult = bootstrap(bootstrapConn, nodeMetadata, credentials, logger); - if (bootstrapResult == FAILED) - return; // bootstrap closed for us. - else if (bootstrapResult == SAMEUSER) - cleanupConn = bootstrapConn; // take over the connection - else { - // connect fresh as ROOT - cleanupConn = connectToSsh(nodeMetadata, logger); - if (!authenticate(cleanupConn, credentials)) { - logger.println("Authentication failed"); - return; // failed to connect as root. - } - } - conn = cleanupConn; - - SCPClient scp = conn.createSCPClient(); - logger.println("Copying slave.jar"); - scp.put(Hudson.getInstance().getJnlpJars("slave.jar").readFully(), "slave.jar", "/tmp"); - - String launchString = "cd /tmp && java " + slave.getJvmOptions() + " -jar slave.jar"; - logger.println("Launching slave agent: " + launchString); - final Session sess = conn.openSession(); - sess.execCommand(launchString); - computer.setChannel(sess.getStdout(), sess.getStdin(), logger, new Channel.Listener() { - @Override - public void onClosed(Channel channel, IOException cause) { - sess.close(); - conn.close(); - } - }); - successful = true; - } finally { - if (cleanupConn != null && !successful) - cleanupConn.close(); + String host = addresses[0]; + if ("0.0.0.0".equals(host)) { + logger.println("Invalid host 0.0.0.0, your host is most likely waiting for an ip address."); + throw new IOException("goto sleep"); } - } - /** - * Authenticate with credentials - */ - private boolean authenticate(Connection connection, LoginCredentials credentials) throws IOException { - if (credentials.getOptionalPrivateKey().isPresent()) { - return connection.authenticateWithPublicKey(credentials.getUser(), credentials.getPrivateKey().toCharArray(), ""); - } else { - return connection.authenticateWithPassword(credentials.getUser(), credentials.getPassword()); - } - } - - /** - * Authenticates using the bootstrapConn, tries to 20 times before giving up. - * - * @param bootstrapConn - * @param nodeMetadata - JClouds compute instance {@link NodeMetadata} for IP address and credentials. - * @param logger - * @return - * @throws IOException - * @throws InterruptedException - */ - private int bootstrap(Connection bootstrapConn, NodeMetadata nodeMetadata, LoginCredentials credentials, PrintStream logger) throws IOException, - InterruptedException { - boolean closeBootstrap = true; - try { - int tries = 20; - boolean isAuthenticated = false; - while (tries-- > 0) { - logger.println("Authenticating as " + credentials.getUser()); - - isAuthenticated = authenticate(bootstrapConn, credentials); - - if (isAuthenticated) { - break; - } - logger.println("Authentication failed. Trying again..."); - Thread.sleep(10000); - } - if (!isAuthenticated) { - logger.println("Authentication failed"); - return FAILED; - } - closeBootstrap = false; - return SAMEUSER; - } catch (InterruptedException e) { - e.printStackTrace(logger); - throw e; - } catch (IOException e) { - e.printStackTrace(logger); - throw e; - } catch (Exception e) { - e.printStackTrace(logger); - throw new RuntimeException(e); - } finally { - if (closeBootstrap) - bootstrapConn.close(); - } + SSHLauncher launcher = new SSHLauncher(host, 22, credentials.getUser(), credentials.getPassword(), credentials.getPrivateKey(), slave.getJvmOptions()); + launcher.launch(computer, listener); } /** @@ -159,43 +60,6 @@ public static String[] getConnectionAddresses(NodeMetadata nodeMetadata, PrintSt } } - /** - * Connect to SSH, and return the connection. - * - * @param nodeMetadata - JClouds compute instance {@link NodeMetadata}, for credentials and the public IP. - * @param logger - the logger where the log messages need to be sent. - * @return - Connection - keeps trying forever, until the host closes the connection or we (the thread) die trying. - * @throws InterruptedException - */ - private Connection connectToSsh(NodeMetadata nodeMetadata, PrintStream logger) throws InterruptedException { - while (true) { - try { - - final String[] addresses = getConnectionAddresses(nodeMetadata, logger); - String host = addresses[0]; - if ("0.0.0.0".equals(host)) { - logger.println("Invalid host 0.0.0.0, your host is most likely waiting for an ip address."); - throw new IOException("goto sleep"); - } - - logger.println("Connecting to " + host + " on port " + 22 + ". "); - Connection conn = new Connection(host, 22); - conn.connect(new ServerHostKeyVerifier() { - @Override - public boolean verifyServerHostKey(String hostname, int port, String serverHostKeyAlgorithm, byte[] serverHostKey) throws Exception { - return true; - } - }); - logger.println("Connected via SSH."); - return conn; // successfully connected - } catch (IOException e) { - // keep retrying until SSH comes up - logger.println("Waiting for SSH to come up. Sleeping 5."); - Thread.sleep(5000); - } - } - } - @Override public Descriptor getDescriptor() { throw new UnsupportedOperationException();