From 8a3e909d266ea59d7179fef21350e9aec1f4b245 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Thu, 21 Mar 2013 17:59:55 +0100 Subject: [PATCH] [FIXED JENKINS-7813] Fixed the throughput problem between master/slave communication. This fix contains two independent problems. One was in the remoting. During a large sustained data transfer (such as artifact archiving and large test reports), the way we were doing flow control and ACK-ing were penalizing us badly. I improved the flow control algorithm in remoting 1.23, and also increased advertised window size so that the transfer can saturate available bandwidth even when a latency is large. (And unless the reader side is excessivesly slow, this shouldn't increase any memory consumption.) The other fix was in trilead-ssh2, which is our SSH client implementation used by ssh-slaves plugin. The buffer size for flow control was too small. I improved the way buffering is done to reduce the memory footprint when the reader closely follows the writer, then I increased the advertised window size. Again, this shouldn't increase memory consumption (in fact it'll likely actually reduce them) unless the reader end gets abandoned. On my simulated latency-injected network, the sustained transfer rate is now on par with scp. We win for smaller files because of the TCP slow start penality that scp would incur, and we lose a bit as files get larger due to additional framing overhead. If you have manually extracted slave.jar and placed them on slaves, you need to update them to 2.23 to see the performance benefits. --- changelog.html | 3 +++ core/pom.xml | 2 +- .../main/java/hudson/tasks/ArtifactArchiver.java | 4 ++++ pom.xml | 2 +- war/pom.xml | 14 +++++++++++++- 5 files changed, 22 insertions(+), 3 deletions(-) diff --git a/changelog.html b/changelog.html index a8b3e4e68c..c25e886fa0 100644 --- a/changelog.html +++ b/changelog.html @@ -64,6 +64,9 @@ Upcoming changes
  • Master node mode not correctly displayed in /computer/(master)/configure. (issue 17263) +
  • + Performance improvement in master/slave communication throughput + (issue 7813) diff --git a/core/pom.xml b/core/pom.xml index 8117b8d303..c7e723e419 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -125,7 +125,7 @@ THE SOFTWARE. org.jenkins-ci trilead-ssh2 - build214-jenkins-1 + build214-jenkins-2 org.kohsuke.stapler diff --git a/core/src/main/java/hudson/tasks/ArtifactArchiver.java b/core/src/main/java/hudson/tasks/ArtifactArchiver.java index 899eecb91e..cbe81e93f0 100644 --- a/core/src/main/java/hudson/tasks/ArtifactArchiver.java +++ b/core/src/main/java/hudson/tasks/ArtifactArchiver.java @@ -32,6 +32,7 @@ import hudson.model.AbstractProject; import hudson.model.BuildListener; import hudson.model.Result; import hudson.util.FormValidation; +import hudson.util.TimeUnit2; import org.kohsuke.stapler.StaplerRequest; import org.kohsuke.stapler.DataBoundConstructor; import org.kohsuke.stapler.AncestorInPath; @@ -122,6 +123,7 @@ public class ArtifactArchiver extends Recorder { File dir = build.getArtifactsDir(); dir.mkdirs(); + final long start = System.nanoTime(); listener.getLogger().println(Messages.ArtifactArchiver_ARCHIVING_ARTIFACTS()); try { FilePath ws = build.getWorkspace(); @@ -157,6 +159,8 @@ public class ArtifactArchiver extends Recorder { return true; } + // System.out.println("Took "+ TimeUnit2.NANOSECONDS.toMillis(System.nanoTime()-start)); + return true; } diff --git a/pom.xml b/pom.xml index 5cf673226d..c4af7b637a 100644 --- a/pom.xml +++ b/pom.xml @@ -175,7 +175,7 @@ THE SOFTWARE. org.jenkins-ci.main remoting - 2.22 + 2.23 diff --git a/war/pom.xml b/war/pom.xml index 965493f8e2..f7179d904d 100644 --- a/war/pom.xml +++ b/war/pom.xml @@ -271,7 +271,19 @@ THE SOFTWARE. org.jenkins-ci.plugins ssh-slaves - 0.22 + 0.23 + hpi + + + org.jenkins-ci.plugins + credentials + 1.3.1 + hpi + + + org.jenkins-ci.plugins + ssh-credentials + 0.2 hpi