From 91b0ed3e5ffece9979e473cc6c3da3b6fa01d682 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 17 Sep 2019 16:49:12 +0300 Subject: [PATCH] [JENKINS-26008] Add an option to disable gzip compression for StandardArtifactManager 1. There are cases where saving CPU is more important than network bandwidth between Jenkins master and slaves 2. According to benchmarks[1], gzip compression is unable to saturate 1Gbps network even on high-end desktop CPUs, so can easily become a bottleneck. [1]: https://github.com/lz4/lz4#benchmarks --- core/src/main/java/hudson/FilePath.java | 40 ++++++++++++++----- .../model/StandardArtifactManager.java | 6 ++- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/hudson/FilePath.java b/core/src/main/java/hudson/FilePath.java index d3962ca8e1..fbdaa2c670 100644 --- a/core/src/main/java/hudson/FilePath.java +++ b/core/src/main/java/hudson/FilePath.java @@ -2349,6 +2349,19 @@ public final class FilePath implements SerializableOnlyOverRemoting { * @since 1.532 */ public int copyRecursiveTo(final DirScanner scanner, final FilePath target, final String description) throws IOException, InterruptedException { + return copyRecursiveTo(scanner, target, description, GZIP); + } + + /** + * Copies files according to a specified scanner to a target node. + * @param scanner a way of enumerating some files (must be serializable for possible delivery to remote side) + * @param target the destination basedir + * @param description a description of the fileset, for logging purposes + * @param compression compression to use + * @return the number of files copied + * @since TODO + */ + public int copyRecursiveTo(final DirScanner scanner, final FilePath target, final String description, @Nonnull TarCompression compression) throws IOException, InterruptedException { if(this.channel==target.channel) { // local to local copy. return act(new CopyRecursiveLocal(target, scanner)); @@ -2357,8 +2370,8 @@ public final class FilePath implements SerializableOnlyOverRemoting { // local -> remote copy final Pipe pipe = Pipe.createLocalToRemote(); - Future future = target.actAsync(new ReadToTar(pipe, description)); - Future future2 = actAsync(new WriteToTar(scanner, pipe)); + Future future = target.actAsync(new ReadToTar(pipe, description, compression)); + Future future2 = actAsync(new WriteToTar(scanner, pipe, compression)); try { // JENKINS-9540 in case the reading side failed, report that error first future.get(); @@ -2370,9 +2383,9 @@ public final class FilePath implements SerializableOnlyOverRemoting { // remote -> local copy final Pipe pipe = Pipe.createRemoteToLocal(); - Future future = actAsync(new CopyRecursiveRemoteToLocal(pipe, scanner)); + Future future = actAsync(new CopyRecursiveRemoteToLocal(pipe, scanner, compression)); try { - readFromTar(remote + '/' + description,new File(target.remote),TarCompression.GZIP.extract(pipe.getIn())); + readFromTar(remote + '/' + description,new File(target.remote),compression.extract(pipe.getIn())); } catch (IOException e) {// BuildException or IOException try { future.get(3,TimeUnit.SECONDS); @@ -2473,15 +2486,18 @@ public final class FilePath implements SerializableOnlyOverRemoting { private class ReadToTar extends SecureFileCallable { private final Pipe pipe; private final String description; - ReadToTar(Pipe pipe, String description) { + private final TarCompression compression; + + ReadToTar(Pipe pipe, String description, @Nonnull TarCompression compression) { this.pipe = pipe; this.description = description; + this.compression = compression; } private static final long serialVersionUID = 1L; @Override public Void invoke(File f, VirtualChannel channel) throws IOException { try (InputStream in = pipe.getIn()) { - readFromTar(remote + '/' + description, f, TarCompression.GZIP.extract(in)); + readFromTar(remote + '/' + description, f, compression.extract(in)); return null; } } @@ -2489,28 +2505,32 @@ public final class FilePath implements SerializableOnlyOverRemoting { private class WriteToTar extends SecureFileCallable { private final DirScanner scanner; private final Pipe pipe; - WriteToTar(DirScanner scanner, Pipe pipe) { + private final TarCompression compression; + WriteToTar(DirScanner scanner, Pipe pipe, @Nonnull TarCompression compression) { this.scanner = scanner; this.pipe = pipe; + this.compression = compression; } private static final long serialVersionUID = 1L; @Override public Integer invoke(File f, VirtualChannel channel) throws IOException, InterruptedException { - return writeToTar(new File(remote), scanner, TarCompression.GZIP.compress(pipe.getOut())); + return writeToTar(new File(remote), scanner, compression.compress(pipe.getOut())); } } private class CopyRecursiveRemoteToLocal extends SecureFileCallable { private static final long serialVersionUID = 1L; private final Pipe pipe; private final DirScanner scanner; - CopyRecursiveRemoteToLocal(Pipe pipe, DirScanner scanner) { + private final TarCompression compression; + CopyRecursiveRemoteToLocal(Pipe pipe, DirScanner scanner, @Nonnull TarCompression compression) { this.pipe = pipe; this.scanner = scanner; + this.compression = compression; } @Override public Integer invoke(File f, VirtualChannel channel) throws IOException { try (OutputStream out = pipe.getOut()) { - return writeToTar(f, scanner, TarCompression.GZIP.compress(out)); + return writeToTar(f, scanner, compression.compress(out)); } } } diff --git a/core/src/main/java/jenkins/model/StandardArtifactManager.java b/core/src/main/java/jenkins/model/StandardArtifactManager.java index a4f8b9df8d..faf53624d9 100644 --- a/core/src/main/java/jenkins/model/StandardArtifactManager.java +++ b/core/src/main/java/jenkins/model/StandardArtifactManager.java @@ -45,6 +45,10 @@ public class StandardArtifactManager extends ArtifactManager { private static final Logger LOG = Logger.getLogger(StandardArtifactManager.class.getName()); + private static FilePath.TarCompression TAR_COMPRESSION = Boolean.getBoolean(StandardArtifactManager.class.getName() + ".disableTrafficCompression") + ? FilePath.TarCompression.NONE + : FilePath.TarCompression.GZIP; + protected transient Run build; public StandardArtifactManager(Run build) { @@ -58,7 +62,7 @@ public class StandardArtifactManager extends ArtifactManager { @Override public void archive(FilePath workspace, Launcher launcher, BuildListener listener, final Map artifacts) throws IOException, InterruptedException { File dir = getArtifactsDir(); String description = "transfer of " + artifacts.size() + " files"; // TODO improve when just one file - workspace.copyRecursiveTo(new FilePath.ExplicitlySpecifiedDirScanner(artifacts), new FilePath(dir), description); + workspace.copyRecursiveTo(new FilePath.ExplicitlySpecifiedDirScanner(artifacts), new FilePath(dir), description, TAR_COMPRESSION); } @Override public final boolean delete() throws IOException, InterruptedException {