Skip to content

Commit

Permalink
lets use sshslaves plugin instead of plain ssh machinery
Browse files Browse the repository at this point in the history
  • Loading branch information
@mavlyutov
mavlyutov committed Feb 5, 2015
1 parent 3566a93 commit 4a40663
Show file tree
Hide file tree
Showing 4 changed files with 65 additions and 185 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,4 @@ release.properties
*.iml
**/dependency-reduced-pom.xml
**/target
.DS_Store
7 changes: 6 additions & 1 deletion jclouds-plugin/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,11 @@
<artifactId>opencsv</artifactId>
<version>2.3</version>
</dependency>
<dependency>
<groupId>org.jenkins-ci.plugins</groupId>
<artifactId>ssh-slaves</artifactId>
<version>1.9</version>
</dependency>

<dependency>
<groupId>org.jenkins-ci.plugins</groupId>
Expand Down Expand Up @@ -244,4 +249,4 @@
</build>
</profile>
</profiles>
</project>
</project>
88 changes: 49 additions & 39 deletions jclouds-plugin/src/main/java/jenkins/plugins/jclouds/compute/JCloudsCloud.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.logging.Logger;

import com.google.inject.Module;
Expand Down Expand Up @@ -115,7 +116,7 @@ public JCloudsCloud(final String profile, final String providerName, final Strin
this.retentionTime = retentionTime;
this.scriptTimeout = scriptTimeout;
this.startTimeout = startTimeout;
this.templates = Objects.firstNonNull(templates, Collections.<JCloudsSlaveTemplate>emptyList());
this.templates = Objects.firstNonNull(templates, Collections.<JCloudsSlaveTemplate> emptyList());
this.zones = Util.fixEmptyAndTrim(zones);
readResolve();
}
Expand All @@ -130,11 +131,7 @@ protected Object readResolve() {
* Get the retention time, defaulting to 30 minutes.
*/
public int getRetentionTime() {
if (retentionTime == 0) {
return 30;
} else {
return retentionTime;
}
return retentionTime == 0 ? 30 : retentionTime;
}

static final Iterable<Module> MODULES = ImmutableSet.<Module>of(new SshjSshClientModule(), new JDKLoggingModule() {
Expand Down Expand Up @@ -188,44 +185,57 @@ public List<JCloudsSlaveTemplate> getTemplates() {
*/
@Override
public Collection<NodeProvisioner.PlannedNode> provision(Label label, int excessWorkload) {
final JCloudsSlaveTemplate t = getTemplate(label);
final JCloudsSlaveTemplate template = getTemplate(label);
List<PlannedNode> plannedNodeList = new ArrayList<PlannedNode>();

while (excessWorkload > 0 && !Jenkins.getInstance().isQuietingDown() && !Jenkins.getInstance().isTerminating()) {

List<PlannedNode> r = new ArrayList<PlannedNode>();
while (excessWorkload > 0
&& !Jenkins.getInstance().isQuietingDown()
&& !Jenkins.getInstance().isTerminating()) {
if ((getRunningNodesCount() + r.size()) >= instanceCap) {
if ((getRunningNodesCount() + plannedNodeList.size()) >= instanceCap) {
LOGGER.info("Instance cap reached while adding capacity for label " + ((label != null) ? label.toString() : "null"));
break; // maxed out
}

r.add(new PlannedNode(t.name, Computer.threadPoolForRemoting.submit(new Callable<Node>() {
plannedNodeList.add(new PlannedNode(template.name, Computer.threadPoolForRemoting.submit(new Callable<Node>() {
public Node call() throws Exception {
// TODO: record the output somewhere
JCloudsSlave s = t.provisionSlave(new StreamTaskListener(System.out));
Hudson.getInstance().addNode(s);
// Cloud instances may have a long init script. If
// we declare
// the provisioning complete by returning without
// the connect
// operation, NodeProvisioner may decide that it
// still wants
// one more instance, because it sees that (1) all
// the slaves
// are offline (because it's still being launched)
// and
// (2) there's no capacity provisioned yet.
//
// deferring the completion of provisioning until
// the launch
// goes successful prevents this problem.
s.toComputer().connect(false).get();
return s;
JCloudsSlave jcloudsSlave = template.provisionSlave(StreamTaskListener.fromStdout());
Jenkins.getInstance().addNode(jcloudsSlave);

/* Cloud instances may have a long init script. If we declare the provisioning complete by returning
without the connect operation, NodeProvisioner may decide that it still wants one more instance,
because it sees that (1) all the slaves are offline (because it's still being launched) and (2)
there's no capacity provisioned yet. Deferring the completion of provisioning until the launch goes
successful prevents this problem. */
ensureLaunched(jcloudsSlave);
return jcloudsSlave;
}
}), Util.tryParseNumber(t.numExecutors, 1).intValue()));
excessWorkload -= t.getNumExecutors();
}), Util.tryParseNumber(template.numExecutors, 1).intValue()));
excessWorkload -= template.getNumExecutors();
}
return plannedNodeList;
}

private void ensureLaunched(JCloudsSlave jcloudsSlave) throws InterruptedException, ExecutionException {
Integer launchTimeoutSec = 5 * 60;
Computer computer = jcloudsSlave.toComputer();
long startMoment = System.currentTimeMillis();
while (computer.isOffline()) {
try {
LOGGER.info(String.format("Slave [%s] not connected yet", jcloudsSlave.getDisplayName()));
computer.connect(false).get();
Thread.sleep(5000l);
} catch (InterruptedException e) {
LOGGER.warning(String.format("Error while launching slave: %s", e));
} catch (ExecutionException e) {
LOGGER.warning(String.format("Error while launching slave: %s", e));
}

if ((System.currentTimeMillis() - startMoment) > 1000l * launchTimeoutSec) {
String message = String.format("Failed to connect to slave within timeout (%d s).", launchTimeoutSec);
LOGGER.warning(message);
throw new ExecutionException(new Throwable(message));
}
}
return r;
}

@Override
Expand Down Expand Up @@ -315,7 +325,7 @@ public String getDisplayName() {
}

public FormValidation doTestConnection(@QueryParameter String providerName, @QueryParameter String identity, @QueryParameter String credential,
@QueryParameter String privateKey, @QueryParameter String endPointUrl, @QueryParameter String zones) throws IOException {
@QueryParameter String privateKey, @QueryParameter String endPointUrl, @QueryParameter String zones) throws IOException {
if (identity == null)
return FormValidation.error("Invalid (AccessId).");
if (credential == null)
Expand Down Expand Up @@ -344,7 +354,7 @@ public FormValidation doTestConnection(@QueryParameter String providerName, @Que
} catch (Exception ex) {
result = FormValidation.error("Cannot connect to specified cloud, please check the identity and credentials: " + ex.getMessage());
} finally {
Closeables.close(ctx, true);
Closeables.close(ctx,true);
}
return result;
}
Expand Down Expand Up @@ -382,7 +392,7 @@ public ListBoxModel doFillProviderNameItems() {
Thread.currentThread().setContextClassLoader(Apis.class.getClassLoader());
// TODO: apis need endpoints, providers don't; do something smarter
// with this stuff :)
Builder<String> builder = ImmutableSet.<String>builder();
Builder<String> builder = ImmutableSet.<String> builder();
builder.addAll(Iterables.transform(Apis.viewableAs(ComputeServiceContext.class), Apis.idFunction()));
builder.addAll(Iterables.transform(Providers.viewableAs(ComputeServiceContext.class), Providers.idFunction()));
Iterable<String> supportedProviders = ImmutableSortedSet.copyOf(builder.build());
Expand All @@ -398,7 +408,7 @@ public AutoCompletionCandidates doAutoCompleteProviderName(@QueryParameter final
Thread.currentThread().setContextClassLoader(Apis.class.getClassLoader());
// TODO: apis need endpoints, providers don't; do something smarter
// with this stuff :)
Builder<String> builder = ImmutableSet.<String>builder();
Builder<String> builder = ImmutableSet.<String> builder();
builder.addAll(Iterables.transform(Apis.viewableAs(ComputeServiceContext.class), Apis.idFunction()));
builder.addAll(Iterables.transform(Providers.viewableAs(ComputeServiceContext.class), Providers.idFunction()));
Iterable<String> supportedProviders = builder.build();
Expand Down
154 changes: 9 additions & 145 deletions jclouds-plugin/src/main/java/jenkins/plugins/jclouds/compute/JCloudsLauncher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,23 @@

import hudson.model.TaskListener;
import hudson.model.Descriptor;
import hudson.model.Hudson;
import hudson.remoting.Channel;
import hudson.slaves.ComputerLauncher;
import hudson.slaves.SlaveComputer;
import hudson.plugins.sshslaves.SSHLauncher;

import java.io.IOException;
import java.io.PrintStream;

import org.jclouds.compute.domain.NodeMetadata;
import org.jclouds.domain.LoginCredentials;

import com.trilead.ssh2.Connection;
import com.trilead.ssh2.SCPClient;
import com.trilead.ssh2.ServerHostKeyVerifier;
import com.trilead.ssh2.Session;

/**
* The launcher that launches the jenkins slave.jar on the Slave. Uses the SSHKeyPair configured in the cloud profile settings, and logs in to the server via
* SSH, and starts the slave.jar.
*
* @author Vijay Kiran
*/
public class JCloudsLauncher extends ComputerLauncher {
private final int FAILED = -1;
private final int SAMEUSER = 0;

/**
* Launch the Jenkins Slave on the SlaveComputer.
Expand All @@ -41,110 +33,19 @@ public void launch(SlaveComputer computer, TaskListener listener) throws IOExcep

PrintStream logger = listener.getLogger();

final Connection bootstrapConn;
final Connection conn;
Connection cleanupConn = null; // java's code path analysis for final doesn't work that well.
boolean successful = false;
final JCloudsSlave slave = (JCloudsSlave) computer.getNode();
final LoginCredentials credentials = slave.getCredentials();
final NodeMetadata nodeMetadata = slave.getNodeMetaData();
final String[] addresses = getConnectionAddresses(nodeMetadata, logger);
LoginCredentials credentials = slave.getCredentials();

try {
bootstrapConn = connectToSsh(nodeMetadata, logger);
int bootstrapResult = bootstrap(bootstrapConn, nodeMetadata, credentials, logger);
if (bootstrapResult == FAILED)
return; // bootstrap closed for us.
else if (bootstrapResult == SAMEUSER)
cleanupConn = bootstrapConn; // take over the connection
else {
// connect fresh as ROOT
cleanupConn = connectToSsh(nodeMetadata, logger);
if (!authenticate(cleanupConn, credentials)) {
logger.println("Authentication failed");
return; // failed to connect as root.
}
}
conn = cleanupConn;

SCPClient scp = conn.createSCPClient();
logger.println("Copying slave.jar");
scp.put(Hudson.getInstance().getJnlpJars("slave.jar").readFully(), "slave.jar", "/tmp");

String launchString = "cd /tmp && java " + slave.getJvmOptions() + " -jar slave.jar";
logger.println("Launching slave agent: " + launchString);
final Session sess = conn.openSession();
sess.execCommand(launchString);
computer.setChannel(sess.getStdout(), sess.getStdin(), logger, new Channel.Listener() {
@Override
public void onClosed(Channel channel, IOException cause) {
sess.close();
conn.close();
}
});
successful = true;
} finally {
if (cleanupConn != null && !successful)
cleanupConn.close();
String host = addresses[0];
if ("0.0.0.0".equals(host)) {
logger.println("Invalid host 0.0.0.0, your host is most likely waiting for an ip address.");
throw new IOException("goto sleep");
}
}

/**
* Authenticate with credentials
*/
private boolean authenticate(Connection connection, LoginCredentials credentials) throws IOException {
if (credentials.getOptionalPrivateKey().isPresent()) {
return connection.authenticateWithPublicKey(credentials.getUser(), credentials.getPrivateKey().toCharArray(), "");
} else {
return connection.authenticateWithPassword(credentials.getUser(), credentials.getPassword());
}
}

/**
* Authenticates using the bootstrapConn, tries to 20 times before giving up.
*
* @param bootstrapConn
* @param nodeMetadata - JClouds compute instance {@link NodeMetadata} for IP address and credentials.
* @param logger
* @return
* @throws IOException
* @throws InterruptedException
*/
private int bootstrap(Connection bootstrapConn, NodeMetadata nodeMetadata, LoginCredentials credentials, PrintStream logger) throws IOException,
InterruptedException {
boolean closeBootstrap = true;
try {
int tries = 20;
boolean isAuthenticated = false;
while (tries-- > 0) {
logger.println("Authenticating as " + credentials.getUser());

isAuthenticated = authenticate(bootstrapConn, credentials);

if (isAuthenticated) {
break;
}
logger.println("Authentication failed. Trying again...");
Thread.sleep(10000);
}
if (!isAuthenticated) {
logger.println("Authentication failed");
return FAILED;
}
closeBootstrap = false;
return SAMEUSER;
} catch (InterruptedException e) {
e.printStackTrace(logger);
throw e;
} catch (IOException e) {
e.printStackTrace(logger);
throw e;
} catch (Exception e) {
e.printStackTrace(logger);
throw new RuntimeException(e);
} finally {
if (closeBootstrap)
bootstrapConn.close();
}
SSHLauncher launcher = new SSHLauncher(host, 22, credentials.getUser(), credentials.getPassword(), credentials.getPrivateKey(), slave.getJvmOptions());
launcher.launch(computer, listener);
}

/**
Expand All @@ -159,43 +60,6 @@ public static String[] getConnectionAddresses(NodeMetadata nodeMetadata, PrintSt
}
}

/**
* Connect to SSH, and return the connection.
*
* @param nodeMetadata - JClouds compute instance {@link NodeMetadata}, for credentials and the public IP.
* @param logger - the logger where the log messages need to be sent.
* @return - Connection - keeps trying forever, until the host closes the connection or we (the thread) die trying.
* @throws InterruptedException
*/
private Connection connectToSsh(NodeMetadata nodeMetadata, PrintStream logger) throws InterruptedException {
while (true) {
try {

final String[] addresses = getConnectionAddresses(nodeMetadata, logger);
String host = addresses[0];
if ("0.0.0.0".equals(host)) {
logger.println("Invalid host 0.0.0.0, your host is most likely waiting for an ip address.");
throw new IOException("goto sleep");
}

logger.println("Connecting to " + host + " on port " + 22 + ". ");
Connection conn = new Connection(host, 22);
conn.connect(new ServerHostKeyVerifier() {
@Override
public boolean verifyServerHostKey(String hostname, int port, String serverHostKeyAlgorithm, byte[] serverHostKey) throws Exception {
return true;
}
});
logger.println("Connected via SSH.");
return conn; // successfully connected
} catch (IOException e) {
// keep retrying until SSH comes up
logger.println("Waiting for SSH to come up. Sleeping 5.");
Thread.sleep(5000);
}
}
}

@Override
public Descriptor<ComputerLauncher> getDescriptor() {
throw new UnsupportedOperationException();
Expand Down

0 comments on commit 4a40663

Please sign in to comment.