diff --git a/core/src/main/java/hudson/FilePath.java b/core/src/main/java/hudson/FilePath.java index 7466d9d6af..21b6b6088d 100644 --- a/core/src/main/java/hudson/FilePath.java +++ b/core/src/main/java/hudson/FilePath.java @@ -53,10 +53,8 @@ import static hudson.FilePath.TarCompression.GZIP; import hudson.org.apache.tools.tar.TarInputStream; import hudson.util.io.Archiver; import hudson.util.io.ArchiverFactory; -import org.apache.tools.ant.BuildException; import org.apache.tools.ant.DirectoryScanner; import org.apache.tools.ant.Project; -import org.apache.tools.ant.taskdefs.Copy; import org.apache.tools.ant.types.FileSet; import org.apache.tools.tar.TarEntry; import org.apache.commons.io.input.CountingInputStream; @@ -98,8 +96,9 @@ import com.jcraft.jzlib.GZIPOutputStream; import com.sun.jna.Native; import hudson.os.PosixException; -import java.io.BufferedInputStream; +import hudson.util.FileVisitor; import java.util.Enumeration; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Logger; import org.apache.tools.ant.taskdefs.Chmod; @@ -1875,6 +1874,18 @@ public final class FilePath implements Serializable { * the number of files copied. */ public int copyRecursiveTo(final String fileMask, final String excludes, final FilePath target) throws IOException, InterruptedException { + return copyRecursiveTo(new DirScanner.Glob(fileMask, excludes), target, fileMask); + } + + /** + * Copies files according to a specified scanner to a target node. + * @param scanner a way of enumerating some files + * @param target the destination basedir + * @param description a description of the fileset, for logging purposes + * @return the number of files copied + * @since TODO + */ + public int copyRecursiveTo(final DirScanner scanner, final FilePath target, final String description) throws IOException, InterruptedException { if(this.channel==target.channel) { // local to local copy. return act(new FileCallable() { @@ -1882,37 +1893,30 @@ public final class FilePath implements Serializable { public Integer invoke(File base, VirtualChannel channel) throws IOException { if(!base.exists()) return 0; assert target.channel==null; - - try { - class CopyImpl extends Copy { - private int copySize; - - public CopyImpl() { - setProject(new org.apache.tools.ant.Project()); - } - - @Override - protected void doFileOperations() { - copySize = super.fileCopyMap.size(); - super.doFileOperations(); - } - - public int getNumCopied() { - return copySize; + final File dest = new File(target.remote); + final AtomicInteger count = new AtomicInteger(); + scanner.scan(base, new FileVisitor() { + @Override public void visit(File f, String relativePath) throws IOException { + if (f.isFile()) { + File target = new File(dest, relativePath); + target.getParentFile().mkdirs(); + Util.copyFile(f, target); + count.incrementAndGet(); } } - - CopyImpl copyTask = new CopyImpl(); - copyTask.setTodir(new File(target.remote)); - copyTask.addFileset(Util.createFileSet(base,fileMask,excludes)); - copyTask.setOverwrite(true); - copyTask.setIncludeEmptyDirs(false); - - copyTask.execute(); - return copyTask.getNumCopied(); - } catch (BuildException e) { - throw new IOException2("Failed to copy "+base+"/"+fileMask+" to "+target,e); - } + @Override public boolean understandsSymlink() { + return true; + } + @Override public void visitSymlink(File link, String target, String relativePath) throws IOException { + try { + Util.createSymlink(dest, target, relativePath, TaskListener.NULL); + } catch (InterruptedException x) { + throw new IOException(x); + } + count.incrementAndGet(); + } + }); + return count.get(); } }); } else @@ -1924,14 +1928,14 @@ public final class FilePath implements Serializable { private static final long serialVersionUID = 1L; public Void invoke(File f, VirtualChannel channel) throws IOException { try { - readFromTar(remote+'/'+fileMask, f,TarCompression.GZIP.extract(pipe.getIn())); + readFromTar(remote + '/' + description, f,TarCompression.GZIP.extract(pipe.getIn())); return null; } finally { pipe.getIn().close(); } } }); - int r = writeToTar(new File(remote),fileMask,excludes,TarCompression.GZIP.compress(pipe.getOut())); + int r = writeToTar(new File(remote), scanner, TarCompression.GZIP.compress(pipe.getOut())); try { future.get(); } catch (ExecutionException e) { @@ -1946,14 +1950,14 @@ public final class FilePath implements Serializable { private static final long serialVersionUID = 1L; public Integer invoke(File f, VirtualChannel channel) throws IOException { try { - return writeToTar(f,fileMask,excludes,TarCompression.GZIP.compress(pipe.getOut())); + return writeToTar(f, scanner, TarCompression.GZIP.compress(pipe.getOut())); } finally { pipe.getOut().close(); } } }); try { - readFromTar(remote+'/'+fileMask,new File(target.remote),TarCompression.GZIP.extract(pipe.getIn())); + readFromTar(remote + '/' + description,new File(target.remote),TarCompression.GZIP.extract(pipe.getIn())); } catch (IOException e) {// BuildException or IOException try { future.get(3,TimeUnit.SECONDS); @@ -2002,10 +2006,10 @@ public final class FilePath implements Serializable { * @return * number of files/directories that are written. */ - private static Integer writeToTar(File baseDir, String fileMask, String excludes, OutputStream out) throws IOException { + private static Integer writeToTar(File baseDir, DirScanner scanner, OutputStream out) throws IOException { Archiver tw = ArchiverFactory.TAR.create(out); try { - new DirScanner.Glob(fileMask,excludes).scan(baseDir,tw); + scanner.scan(baseDir,tw); } finally { tw.close(); } diff --git a/core/src/main/java/hudson/tasks/ArtifactArchiver.java b/core/src/main/java/hudson/tasks/ArtifactArchiver.java index 87568c6d9b..df11840d30 100644 --- a/core/src/main/java/hudson/tasks/ArtifactArchiver.java +++ b/core/src/main/java/hudson/tasks/ArtifactArchiver.java @@ -31,13 +31,17 @@ import hudson.model.AbstractBuild; import hudson.model.AbstractProject; import hudson.model.BuildListener; import hudson.model.Result; +import hudson.remoting.VirtualChannel; import hudson.util.FormValidation; +import java.io.File; import org.kohsuke.stapler.StaplerRequest; import org.kohsuke.stapler.DataBoundConstructor; import org.kohsuke.stapler.AncestorInPath; import org.kohsuke.stapler.QueryParameter; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import net.sf.json.JSONObject; import javax.annotation.Nonnull; @@ -131,22 +135,25 @@ public class ArtifactArchiver extends Recorder { String artifacts = build.getEnvironment(listener).expand(this.artifacts); - if (build.pickArtifactManager().archive(build, ws, launcher, listener, artifacts, excludes) == 0) { + Map files = ws.act(new ListFiles(artifacts, excludes)); + if (!files.isEmpty()) { + build.pickArtifactManager().archive(build, ws, launcher, listener, files); + } else { if(build.getResult().isBetterOrEqualTo(Result.UNSTABLE)) { // If the build failed, don't complain that there was no matching artifact. // The build probably didn't even get to the point where it produces artifacts. listenerWarnOrError(listener, Messages.ArtifactArchiver_NoMatchFound(artifacts)); String msg = null; try { - msg = ws.validateAntFileMask(artifacts); + msg = ws.validateAntFileMask(artifacts); } catch (Exception e) { - listenerWarnOrError(listener, e.getMessage()); + listenerWarnOrError(listener, e.getMessage()); } if(msg!=null) listenerWarnOrError(listener, msg); } if (!allowEmptyArchive) { - build.setResult(Result.FAILURE); + build.setResult(Result.FAILURE); } return true; } @@ -161,6 +168,23 @@ public class ArtifactArchiver extends Recorder { return true; } + private static final class ListFiles implements FilePath.FileCallable> { + private static final long serialVersionUID = 1; + private final String includes, excludes; + ListFiles(String includes, String excludes) { + this.includes = includes; + this.excludes = excludes; + } + @Override public Map invoke(File basedir, VirtualChannel channel) throws IOException, InterruptedException { + Map r = new HashMap(); + for (String f : Util.createFileSet(basedir, includes, excludes).getDirectoryScanner().getIncludedFiles()) { + f = f.replace(File.separatorChar, '/'); + r.put(f, f); + } + return r; + } + } + @Override public boolean prebuild(AbstractBuild build, BuildListener listener) { if(latestOnly) { diff --git a/core/src/main/java/hudson/util/DirScanner.java b/core/src/main/java/hudson/util/DirScanner.java index c11ae9c622..b4e06abc71 100644 --- a/core/src/main/java/hudson/util/DirScanner.java +++ b/core/src/main/java/hudson/util/DirScanner.java @@ -26,29 +26,36 @@ public abstract class DirScanner implements Serializable { */ public abstract void scan(File dir, FileVisitor visitor) throws IOException; + /** + * @since TODO + */ + protected final void scanSingle(File f, String relative, FileVisitor visitor) throws IOException { + if (visitor.understandsSymlink()) { + try { + String target; + try { + target = Util.resolveSymlink(f); + } catch (IOException x) { // JENKINS-13202 + target = null; + } + if (target != null) { + visitor.visitSymlink(f, target, relative); + return; + } + } catch (InterruptedException e) { + throw (IOException) new InterruptedIOException().initCause(e); + } + } + visitor.visit(f, relative); + } + /** * Scans everything recursively. */ public static class Full extends DirScanner { private void scan(File f, String path, FileVisitor visitor) throws IOException { if (f.canRead()) { - if (visitor.understandsSymlink()) { - try { - String target; - try { - target = Util.resolveSymlink(f); - } catch (IOException x) { // JENKINS-13202 - target = null; - } - if (target!=null) { - visitor.visitSymlink(f,target,path+f.getName()); - return; - } - } catch (InterruptedException e) { - throw (IOException)new InterruptedIOException().initCause(e); - } - } - visitor.visit(f,path+f.getName()); + scanSingle(f, path + f.getName(), visitor); if(f.isDirectory()) { for( File child : f.listFiles() ) scan(child,path+f.getName()+'/',visitor); @@ -113,24 +120,7 @@ public abstract class DirScanner implements Serializable { DirectoryScanner ds = fs.getDirectoryScanner(new org.apache.tools.ant.Project()); for( String f : ds.getIncludedFiles()) { File file = new File(dir, f); - - if (visitor.understandsSymlink()) { - try { - String target; - try { - target = Util.resolveSymlink(file); - } catch (IOException x) { // JENKINS-13202 - target = null; - } - if (target!=null) { - visitor.visitSymlink(file,target,f); - continue; - } - } catch (InterruptedException e) { - throw (IOException)new InterruptedIOException().initCause(e); - } - } - visitor.visit(file,f); + scanSingle(file, f, visitor); } } } diff --git a/core/src/main/java/jenkins/model/ArtifactManager.java b/core/src/main/java/jenkins/model/ArtifactManager.java index 07da8c9ee3..24fa5c2dbb 100644 --- a/core/src/main/java/jenkins/model/ArtifactManager.java +++ b/core/src/main/java/jenkins/model/ArtifactManager.java @@ -30,16 +30,15 @@ import hudson.Launcher; import hudson.model.AbstractBuild; import hudson.model.BuildListener; import hudson.model.DirectoryBrowserSupport; -import hudson.model.Executor; import hudson.model.Job; import hudson.model.Run; import hudson.tasks.ArtifactArchiver; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import javax.annotation.CheckForNull; import hudson.util.HttpResponses; +import java.util.Map; import org.kohsuke.stapler.HttpResponse; import org.kohsuke.stapler.StaplerRequest; @@ -69,7 +68,8 @@ public abstract class ArtifactManager implements ExtensionPoint { /** * Archive all configured artifacts from a build. - * (If called multiple times for the same build, do not delete the old artifacts but keep them all, unless overwritten.) + *

If called multiple times for the same build, do not delete the old artifacts but keep them all, unless overwritten. + * For example, the XVNC plugin could use this to save {@code screenshot.jpg} if so configured. * *

* This method should only be invoked on a running build. @@ -78,30 +78,11 @@ public abstract class ArtifactManager implements ExtensionPoint { * @param workspace the workspace from which to copy files * @param launcher a launcher to use if external processes need to be forked * @param listener a way to print messages about progress or problems - * @param artifacts comma- or space-separated list of patterns of files/directories to be archived relative to the workspace (Ant format, any variables already substituted) - * @param excludes patterns of files to be excluded from the artifact list (Ant format, may be null for no excludes) - * @return the number of files actually archived (may be zero) + * @param artifacts map from paths in the archive area to paths in the workspace (all paths {@code /}-separated) * @throws IOException if transfer or copying failed in any way * @see ArtifactArchiver#perform(AbstractBuild, Launcher, BuildListener) */ - public abstract int archive(Run build, FilePath workspace, Launcher launcher, BuildListener listener, String artifacts, @CheckForNull String excludes) throws IOException, InterruptedException; - - /** - * Add a single file to the list of archives for a build. - * For example, the XVNC plugin could use this to save {@code screenshot.jpg} if so configured. - * If called on a target path which was already recorded, the old artifact is overwritten. - * - *

- * This method should only be invoked on a running build. - * - * @param build a build which may or may not already have archives - * @param launcher a launcher to use if external processes need to be forked - * @param listener a way to print messages about progress or problems - * @param source a file to copy - * @param target the full relative path to which this file should be archived, e.g. {@code subdir/something.jar} - * @throws IOException if transfer or copying failed in any way - */ - public abstract void archiveSingle(Run build, Launcher launcher, BuildListener listener, FilePath source, String target) throws IOException, InterruptedException; + public abstract void archive(Run build, FilePath workspace, Launcher launcher, BuildListener listener, Map artifacts) throws IOException, InterruptedException; /** * Delete all artifacts associated with an earlier build (if any). diff --git a/core/src/main/java/jenkins/model/StandardArtifactManager.java b/core/src/main/java/jenkins/model/StandardArtifactManager.java index 38887fb7b4..220f1dc2e9 100644 --- a/core/src/main/java/jenkins/model/StandardArtifactManager.java +++ b/core/src/main/java/jenkins/model/StandardArtifactManager.java @@ -31,6 +31,8 @@ import hudson.model.BuildListener; import hudson.model.DirectoryBrowserSupport; import hudson.model.Job; import hudson.model.Run; +import hudson.util.DirScanner; +import hudson.util.FileVisitor; import java.io.File; import java.io.FileInputStream; import java.io.IOException; @@ -39,12 +41,7 @@ import java.util.Arrays; import java.util.concurrent.atomic.AtomicInteger; import hudson.util.HttpResponses; -import org.kohsuke.stapler.HttpResponse; -import org.kohsuke.stapler.HttpResponses.HttpResponseException; -import org.kohsuke.stapler.StaplerRequest; -import org.kohsuke.stapler.StaplerResponse; - -import javax.servlet.ServletException; +import java.util.Map; /** * Default artifact manager which transfers files over the remoting channel and stores them inside the build directory. @@ -53,16 +50,20 @@ import javax.servlet.ServletException; */ public class StandardArtifactManager extends ArtifactManager { - @Override public int archive(Run build, FilePath workspace, Launcher launcher, BuildListener listener, String artifacts, String excludes) throws IOException, InterruptedException { + @Override public void archive(Run build, FilePath workspace, Launcher launcher, BuildListener listener, final Map artifacts) throws IOException, InterruptedException { File dir = getArtifactsDir(build); - dir.mkdirs(); - return workspace.copyRecursiveTo(artifacts, excludes, new FilePath(dir)); - } - - @Override public void archiveSingle(Run build, Launcher launcher, BuildListener listener, FilePath source, String target) throws IOException, InterruptedException { - FilePath dest = new FilePath(getArtifactsDir(build)).child(target); - dest.getParent().mkdirs(); - source.copyTo(dest); + String description = "transfer of " + artifacts.size() + " files"; // TODO improve when just one file + workspace.copyRecursiveTo(new DirScanner() { + @Override public void scan(File dir, FileVisitor visitor) throws IOException { + for (Map.Entry entry : artifacts.entrySet()) { + String archivedPath = entry.getKey(); + assert archivedPath.indexOf('\\') == -1; + String workspacePath = entry.getValue(); + assert workspacePath.indexOf('\\') == -1; + scanSingle(new File(dir, workspacePath), archivedPath, visitor); + } + } + }, new FilePath(dir), description); } @Override public boolean deleteArtifacts(Run build) throws IOException, InterruptedException { diff --git a/maven-plugin/src/main/java/hudson/maven/MavenBuild.java b/maven-plugin/src/main/java/hudson/maven/MavenBuild.java index 9d30fcc660..590d369003 100644 --- a/maven-plugin/src/main/java/hudson/maven/MavenBuild.java +++ b/maven-plugin/src/main/java/hudson/maven/MavenBuild.java @@ -26,7 +26,6 @@ package hudson.maven; import hudson.EnvVars; import hudson.FilePath; import hudson.Launcher; -import hudson.Util; import hudson.maven.reporters.MavenArtifactRecord; import hudson.maven.reporters.SurefireArchiver; import hudson.maven.reporters.TestFailureDetector; @@ -444,20 +443,10 @@ public class MavenBuild extends AbstractMavenBuild { return new FilePath(MavenBuild.this.getArtifactsDir()); } - @Override public void archiveSingle(FilePath source, String target) throws IOException, InterruptedException { - ArtifactManager am = pickArtifactManager(); - try { - if (Util.getDigestOf(am.loadArtifact(MavenBuild.this, target)).equals(source.digest())) { - LOGGER.fine("Not actually archiving " + source + " due to digest match"); - listener.getLogger().println("Not actually archiving " + target + " due to digest match"); - return; - } else { - listener.getLogger().println("[JENKINS] Re-archiving " + source); - } - } catch (FileNotFoundException x) { - listener.getLogger().println("[JENKINS] Archiving " + source + " to " + target); - } - am.archiveSingle(MavenBuild.this, launcher, listener, source, target); + @Override public void archive(FilePath basedir, Map artifacts) throws IOException, InterruptedException { + listener.getLogger().println("[JENKINS] Archiving " + artifacts); + // TODO queue this up till the end of the build + pickArtifactManager().archive(MavenBuild.this, basedir, launcher, listener, artifacts); } public void setResult(Result result) { diff --git a/maven-plugin/src/main/java/hudson/maven/MavenBuildProxy.java b/maven-plugin/src/main/java/hudson/maven/MavenBuildProxy.java index f4ef764f03..17cb93c2bc 100644 --- a/maven-plugin/src/main/java/hudson/maven/MavenBuildProxy.java +++ b/maven-plugin/src/main/java/hudson/maven/MavenBuildProxy.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.io.Serializable; import java.util.Calendar; import java.util.List; +import java.util.Map; import jenkins.model.ArtifactManager; /** @@ -100,10 +101,10 @@ public interface MavenBuildProxy { FilePath getArtifactsDir(); /** - * @see ArtifactManager#archiveSingle + * @see ArtifactManager#archive * @since TODO */ - void archiveSingle(FilePath source, String target) throws IOException, InterruptedException; + void archive(FilePath basedir, Map artifacts) throws IOException, InterruptedException; /** * @see MavenBuild#setResult(Result) @@ -219,8 +220,8 @@ public interface MavenBuildProxy { return core.getArtifactsDir(); } - @Override public void archiveSingle(FilePath source, String target) throws IOException, InterruptedException { - core.archiveSingle(source, target); + @Override public void archive(FilePath basedir, Map artifacts) throws IOException, InterruptedException { + core.archive(basedir, artifacts); } public void setResult(Result result) { diff --git a/maven-plugin/src/main/java/hudson/maven/reporters/MavenArtifact.java b/maven-plugin/src/main/java/hudson/maven/reporters/MavenArtifact.java index 6b8e33360f..8ff23723d7 100755 --- a/maven-plugin/src/main/java/hudson/maven/reporters/MavenArtifact.java +++ b/maven-plugin/src/main/java/hudson/maven/reporters/MavenArtifact.java @@ -52,6 +52,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.io.Serializable; +import java.util.Collections; import java.util.Map; import java.util.logging.Logger; import javax.servlet.ServletException; @@ -243,7 +244,7 @@ public final class MavenArtifact implements Serializable { else { String target = groupId + '/' + artifactId + '/' + version + '/' + canonicalName; FilePath origin = new FilePath(file); - build.archiveSingle(origin, target); + build.archive(origin.getParent(), Collections.singletonMap(target, origin.getName())); } } diff --git a/maven-plugin/src/main/java/hudson/maven/reporters/MavenArtifactArchiver.java b/maven-plugin/src/main/java/hudson/maven/reporters/MavenArtifactArchiver.java index 9462a8b3b3..03de3679db 100644 --- a/maven-plugin/src/main/java/hudson/maven/reporters/MavenArtifactArchiver.java +++ b/maven-plugin/src/main/java/hudson/maven/reporters/MavenArtifactArchiver.java @@ -37,6 +37,7 @@ import java.io.IOException; import java.lang.reflect.InvocationHandler; import java.lang.reflect.Method; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -160,7 +161,8 @@ public class MavenArtifactArchiver extends MavenReporter { continue; // looks like this is already archived String target = assembly.getName(); listener.getLogger().println("[JENKINS] Archiving "+ assembly+" to "+target); - build.archiveSingle(new FilePath(assembly), target); + FilePath origin = new FilePath(assembly); + build.archive(origin.getParent(), Collections.singletonMap(target, origin.getName())); // TODO: fingerprint } } diff --git a/maven-plugin/src/test/java/hudson/maven/reporters/SurefireArchiverUnitTest.java b/maven-plugin/src/test/java/hudson/maven/reporters/SurefireArchiverUnitTest.java index bb825dfd06..a0e6a226e2 100644 --- a/maven-plugin/src/test/java/hudson/maven/reporters/SurefireArchiverUnitTest.java +++ b/maven-plugin/src/test/java/hudson/maven/reporters/SurefireArchiverUnitTest.java @@ -28,6 +28,7 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.Calendar; import java.util.List; +import java.util.Map; import org.apache.commons.io.output.NullOutputStream; import org.codehaus.plexus.component.configurator.ComponentConfigurationException; @@ -246,9 +247,7 @@ public class SurefireArchiverUnitTest { return null; } - @Override - public void archiveSingle(FilePath source, String target) throws IOException, InterruptedException { - } + @Override public void archive(FilePath basedir, Map artifacts) throws IOException, InterruptedException {} @Override public void setResult(Result result) {