merging remoting-integration branch

git-svn-id: https://hudson.dev.java.net/svn/hudson/trunk/hudson/main@1523 71c3de6d-444a-0410-be80-ed276b4c234a

core/src/main/java/hudson/FilePath.java

@@ -1,44 +1,129 @@
 package hudson;
 
+import hudson.remoting.Callable;
+import hudson.remoting.Channel;
+import hudson.remoting.Pipe;
+import hudson.remoting.RemoteOutputStream;
+import hudson.remoting.VirtualChannel;
+import hudson.remoting.DelegatingCallable;
 import hudson.util.IOException2;
+import hudson.model.Hudson;
+import org.apache.tools.ant.BuildException;
+import org.apache.tools.ant.DirectoryScanner;
+import org.apache.tools.ant.taskdefs.Copy;
+import org.apache.tools.ant.types.FileSet;
 
 import java.io.File;
+import java.io.FileFilter;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.List;
+import java.net.URI;
 
 /**
- * {@link File} like path-manipulation object.
+ * {@link File} like object with remoting support.
  *
  * <p>
- * In general, because programs could be executed remotely,
- * we need two path strings to identify the same directory.
- * One from a point of view of the master (local), the other
- * from a point of view of the slave (remote).
+ * Unlike {@link File}, which always implies a file path on the current computer,
+ * {@link FilePath} represents a file path on a specific slave or the master.
+ *
+ * Despite that, {@link FilePath} can be used much like {@link File}. It exposes
+ * a bunch of operations (and we should add more operations as long as they are
+ * generally useful), and when invoked against a file on a remote node, {@link FilePath}
+ * executes the necessary code remotely, thereby providing semi-transparent file
+ * operations.
+ *
+ * <h2>Using {@link FilePath} smartly</h2>
+ * <p>
+ * The transparency makes it easy to write plugins without worrying too much about
+ * remoting, by making it works like NFS, where remoting happens at the file-system
+ * later.
  *
  * <p>
- * This class allows path manipulation to be done
- * and allow the local/remote versions to be obtained
- * after the computation.
+ * But one should note that such use of remoting may not be optional. Sometimes,
+ * it makes more sense to move some computation closer to the data, as opposed to
+ * move the data to the computation. For example, if you are just computing a MD5
+ * digest of a file, then it would make sense to do the digest on the host where
+ * the file is located, as opposed to send the whole data to the master and do MD5
+ * digesting there.
+ *
+ * <p>
+ * {@link FilePath} supports this "code migration" by in the
+ * {@link #act(FileCallable)} method. One can pass in a custom implementation
+ * of {@link FileCallable}, to be executed on the node where the data is located.
+ * The following code shows the example:
+ *
+ * <pre>
+ * FilePath file = ...;
+ *
+ * // make 'file' a fresh empty directory.
+ * file.act(new FileCallable&lt;Void>() {
+ *   // if 'file' is on a different node, this FileCallable will
+ *   // be transfered to that node and executed there.
+ *   public Void invoke(File f,VirtualChannel channel) {
+ *     // f and file represents the same thing
+ *     f.deleteContents();
+ *     f.mkdirs();
+ *   }
+ * });
+ * </pre>
+ *
+ * <p>
+ * When {@link FileCallable} is transfered to a remote node, it will be done so
+ * by using the same Java serializaiton scheme that the remoting module uses.
+ * See {@link Channel} for more about this. 
+ *
+ * <p>
+ * {@link FilePath} itself can be sent over to a remote node as a part of {@link Callable}
+ * serialization. For example, sending a {@link FilePath} of a remote node to that
+ * node causes {@link FilePath} to become "local". Similarly, sending a
+ * {@link FilePath} that represents the local computer causes it to become "remote."
  *
  * @author Kohsuke Kawaguchi
  */
-public final class FilePath {
-    private final File local;
+public final class FilePath implements Serializable {
+    /**
+     * When this {@link FilePath} represents the remote path,
+     * this field is always non-null on master (the field represents
+     * the channel to the remote slave.) When transferred to a slave via remoting,
+     * this field reverts back to null, since it's transient.
+     *
+     * When this {@link FilePath} represents a path on the master,
+     * this field is null on master. When transferred to a slave via remoting,
+     * this field becomes non-null, representing the {@link Channel}
+     * back to the master.
+     *
+     * This is used to determine whether we are running on the master or the slave.
+     */
+    private transient VirtualChannel channel;
+
+    // since the platform of the slave might be different, can't use java.io.File
     private final String remote;
 
-    public FilePath(File local, String remote) {
-        this.local = local;
+    public FilePath(VirtualChannel channel, String remote) {
+        this.channel = channel;
         this.remote = remote;
     }
 
     /**
-     * Useful when there's no remote path.
+     * To create {@link FilePath} on the master computer.
      */
-    public FilePath(File local) {
-        this(local,local.getPath());
+    public FilePath(File localPath) {
+        this.channel = null;
+        this.remote = localPath.getPath();
     }
 
     public FilePath(FilePath base, String rel) {
-        this.local = new File(base.local,rel);
+        this.channel = base.channel;
         if(base.isUnix()) {
             this.remote = base.remote+'/'+rel;
         } else {
@@ -55,28 +140,116 @@ public final class FilePath {
         return remote.indexOf("\\")==-1;
     }
 
-    public File getLocal() {
-        return local;
-    }
-
     public String getRemote() {
         return remote;
     }
 
+    /**
+     * Code that gets executed on the machine where the {@link FilePath} is local.
+     * Used to act on {@link FilePath}.
+     *
+     * @see FilePath#act(FileCallable)
+     */
+    public static interface FileCallable<T> extends Serializable {
+        /**
+         * Performs the computational task on the node where the data is located.
+         *
+         * @param f
+         *      {@link File} that represents the local file that {@link FilePath} has represented.
+         * @param channel
+         *      The "back pointer" of the {@link Channel} that represents the communication
+         *      with the node from where the code was sent.
+         */
+        T invoke(File f, VirtualChannel channel) throws IOException;
+    }
+
+    /**
+     * Executes some program on the machine that this {@link FilePath} exists,
+     * so that one can perform local file operations.
+     */
+    public <T> T act(final FileCallable<T> callable) throws IOException, InterruptedException {
+        if(channel!=null) {
+            // run this on a remote system
+            try {
+                return channel.call(new DelegatingCallable<T,IOException>() {
+                    public T call() throws IOException {
+                        return callable.invoke(new File(remote), Channel.current());
+                    }
+
+                    public ClassLoader getClassLoader() {
+                        return callable.getClass().getClassLoader();
+                    }
+                });
+            } catch (IOException e) {
+                // wrap it into a new IOException so that we get the caller's stack trace as well.
+                throw new IOException2("remote file operation failed",e);
+            }
+        } else {
+            // the file is on the local machine.
+            return callable.invoke(new File(remote), Hudson.MasterComputer.localChannel);
+        }
+    }
+
+    /**
+     * Executes some program on the machine that this {@link FilePath} exists,
+     * so that one can perform local file operations.
+     */
+    public <V,E extends Throwable> V act(Callable<V,E> callable) throws IOException, InterruptedException, E {
+        if(channel!=null) {
+            // run this on a remote system
+            return channel.call(callable);
+        } else {
+            // the file is on the local machine
+            return callable.call();
+        }
+    }
+
+    /**
+     * Converts this file to the URI, relative to the machine
+     * on which this file is available.
+     */
+    public URI toURI() throws IOException, InterruptedException {
+        return act(new FileCallable<URI>() {
+            public URI invoke(File f, VirtualChannel channel) {
+                return f.toURI();
+            }
+        });
+    }
+
     /**
      * Creates this directory.
      */
-    public void mkdirs() throws IOException {
-        if(!local.mkdirs() && !local.exists())
-            throw new IOException("Failed to mkdirs: "+local);
+    public void mkdirs() throws IOException, InterruptedException {
+        if(act(new FileCallable<Boolean>() {
+            public Boolean invoke(File f, VirtualChannel channel) throws IOException {
+                return !f.mkdirs() && !f.exists();
+            }
+        }))
+            throw new IOException("Failed to mkdirs: "+remote);
+    }
+
+    /**
+     * Deletes this directory, including all its contents recursively.
+     */
+    public void deleteRecursive() throws IOException, InterruptedException {
+        act(new FileCallable<Void>() {
+            public Void invoke(File f, VirtualChannel channel) throws IOException {
+                Util.deleteRecursive(f);
+                return null;
+            }
+        });
     }
 
     /**
      * Deletes all the contents of this directory, but not the directory itself
      */
-    public void deleteContents() throws IOException {
-        // TODO: consider doing this remotely if possible
-        Util.deleteContentsRecursive(getLocal());
+    public void deleteContents() throws IOException, InterruptedException {
+        act(new FileCallable<Void>() {
+            public Void invoke(File f, VirtualChannel channel) throws IOException {
+                Util.deleteContentsRecursive(f);
+                return null;
+            }
+        });
     }
 
     /**
@@ -85,7 +258,15 @@ public final class FilePath {
      * This method assumes that the file name is the same between local and remote.
      */
     public String getName() {
-        return local.getName();
+        int len = remote.length()-1;
+        while(len>=0) {
+            char ch = remote.charAt(len);
+            if(ch=='\\' || ch=='/')
+                break;
+            len--;
+        }
+
+        return remote.substring(len+1);
     }
 
     /**
@@ -107,47 +288,352 @@ public final class FilePath {
             len--;
         }
 
-        return new FilePath( local.getParentFile(), remote.substring(0,len) );
+        return new FilePath( channel, remote.substring(0,len) );
     }
 
     /**
      * Creates a temporary file.
      */
-    public FilePath createTempFile(String prefix, String suffix) throws IOException {
+    public FilePath createTempFile(final String prefix, final String suffix) throws IOException, InterruptedException {
         try {
-            File f = File.createTempFile(prefix, suffix, getLocal());
-            return new FilePath(this,f.getName());
+            return new FilePath(this,act(new FileCallable<String>() {
+                public String invoke(File dir, VirtualChannel channel) throws IOException {
+                    File f = File.createTempFile(prefix, suffix, dir);
+                    return f.getName();
+                }
+            }));
         } catch (IOException e) {
-            throw new IOException2("Failed to create a temp file on "+getLocal(),e);
+            throw new IOException2("Failed to create a temp file on "+remote,e);
+        }
+    }
+
+    /**
+     * Creates a temporary file in this directory and set the contents by the
+     * given text (encoded in the platform default encoding)
+     */
+    public FilePath createTextTempFile(final String prefix, final String suffix, final String contents) throws IOException, InterruptedException {
+        try {
+            return new FilePath(this,act(new FileCallable<String>() {
+                public String invoke(File dir, VirtualChannel channel) throws IOException {
+                    File f = File.createTempFile(prefix, suffix, dir);
+
+                    Writer w = new FileWriter(f);
+                    w.write(contents);
+                    w.close();
+
+                    return f.getName();
+                }
+            }));
+        } catch (IOException e) {
+            throw new IOException2("Failed to create a temp file on "+remote,e);
         }
     }
 
     /**
      * Deletes this file.
      */
-    public boolean delete() {
-       return local.delete();
-    }
-
-    public boolean exists() {
-        return local.exists();
-    }
-
-    public boolean isDirectory() {
-        return local.isDirectory();
+    public boolean delete() throws IOException, InterruptedException {
+        return act(new FileCallable<Boolean>() {
+            public Boolean invoke(File f, VirtualChannel channel) throws IOException {
+                return f.delete();
+            }
+        });
     }
 
     /**
-     * Always use {@link #getLocal()} or {@link #getRemote()}
+     * Checks if the file exists.
      */
+    public boolean exists() throws IOException, InterruptedException {
+        return act(new FileCallable<Boolean>() {
+            public Boolean invoke(File f, VirtualChannel channel) throws IOException {
+                return f.exists();
+            }
+        });
+    }
+
+    /**
+     * Gets the last modified time stamp of this file, by using the clock
+     * of the machine where this file actually resides.
+     *
+     * @see File#lastModified()
+     */
+    public long lastModified() throws IOException, InterruptedException {
+        return act(new FileCallable<Long>() {
+            public Long invoke(File f, VirtualChannel channel) throws IOException {
+                return f.lastModified();
+            }
+        });
+    }
+
+    /**
+     * Checks if the file is a directory.
+     */
+    public boolean isDirectory() throws IOException, InterruptedException {
+        return act(new FileCallable<Boolean>() {
+            public Boolean invoke(File f, VirtualChannel channel) throws IOException {
+                return f.isDirectory();
+            }
+        });
+    }
+
+    /**
+     * List up files in this directory.
+     *
+     * @param filter
+     *      The optional filter used to narrow down the result.
+     *      If non-null, must be {@link Serializable}.
+     *      If this {@link FilePath} represents a remote path,
+     *      the filter object will be executed on the remote machine.
+     */
+    public List<FilePath> list(final FileFilter filter) throws IOException, InterruptedException {
+        return act(new FileCallable<List<FilePath>>() {
+            public List<FilePath> invoke(File f, VirtualChannel channel) throws IOException {
+                File[] children = f.listFiles(filter);
+                if(children ==null)     return null;
+
+                ArrayList<FilePath> r = new ArrayList<FilePath>(children.length);
+                for (File child : children)
+                    r.add(new FilePath(child));
+
+                return r;
+            }
+        });
+    }
+
+    /**
+     * Reads this file.
+     */
+    public InputStream read() throws IOException {
+        if(channel==null)
+            return new FileInputStream(new File(remote));
+
+        final Pipe p = Pipe.createRemoteToLocal();
+        channel.callAsync(new Callable<Void,IOException>() {
+            public Void call() throws IOException {
+                FileInputStream fis = new FileInputStream(new File(remote));
+                Util.copyStream(fis,p.getOut());
+                fis.close();
+                p.getOut().close();
+                return null;
+            }
+        });
+
+        return p.getIn();
+    }
+
+    /**
+     * Writes to this file.
+     * If this file already exists, it will be overwritten.
+     */
+    public OutputStream write() throws IOException {
+        if(channel==null)
+            return new FileOutputStream(new File(remote));
+
+        final Pipe p = Pipe.createLocalToRemote();
+        channel.callAsync(new Callable<Void,IOException>() {
+            public Void call() throws IOException {
+                FileOutputStream fos = new FileOutputStream(new File(remote));
+                Util.copyStream(p.getIn(),fos);
+                fos.close();
+                p.getIn().close();
+                return null;
+            }
+        });
+
+        return p.getOut();
+    }
+
+    /**
+     * Computes the MD5 digest of the file in hex string.
+     */
+    public String digest() throws IOException, InterruptedException {
+        return act(new FileCallable<String>() {
+            public String invoke(File f, VirtualChannel channel) throws IOException {
+                return Util.getDigestOf(new FileInputStream(f));
+            }
+        });
+    }
+
+    /**
+     * Copies this file to the specified target.
+     */
+    public void copyTo(FilePath target) throws IOException, InterruptedException {
+        OutputStream out = target.write();
+        try {
+            copyTo(out);
+        } finally {
+            out.close();
+        }
+    }
+
+    /**
+     * Sends the contents of this file into the given {@link OutputStream}.
+     */
+    public void copyTo(OutputStream os) throws IOException, InterruptedException {
+        final OutputStream out = new RemoteOutputStream(os);
+
+        act(new FileCallable<Void>() {
+            public Void invoke(File f, VirtualChannel channel) throws IOException {
+                FileInputStream fis = new FileInputStream(f);
+                Util.copyStream(fis,out);
+                fis.close();
+                out.close();
+                return null;
+            }
+        });
+    }
+
+    /**
+     * Remoting interface used for {@link FilePath#copyRecursiveTo(String, FilePath)}.
+     *
+     * TODO: this might not be the most efficient way to do the copy.
+     */
+    interface RemoteCopier {
+        void open(String fileName) throws IOException;
+        void write(byte[] buf, int len) throws IOException;
+        void close() throws IOException;
+    }
+
+    /**
+     * Copies the files that match the given file mask to the specified target node.
+     *
+     * @return
+     *      the number of files copied.
+     */
+    public int copyRecursiveTo(final String fileMask, final FilePath target) throws IOException, InterruptedException {
+        if(this.channel==target.channel) {
+            // local to local copy.
+            return act(new FileCallable<Integer>() {
+                public Integer invoke(File base, VirtualChannel channel) throws IOException {
+                    assert target.channel==null;
+
+                    try {
+                        class CopyImpl extends Copy {
+                            private int copySize;
+
+                            public CopyImpl() {
+                                setProject(new org.apache.tools.ant.Project());
+                            }
+
+                            protected void doFileOperations() {
+                                copySize = super.fileCopyMap.size();
+                                super.doFileOperations();
+                            }
+
+                            public int getNumCopied() {
+                                return copySize;
+                            }
+                        }
+
+                        CopyImpl copyTask = new CopyImpl();
+                        copyTask.setTodir(new File(target.remote));
+                        FileSet src = new FileSet();
+                        src.setDir(base);
+                        src.setIncludes(fileMask);
+                        copyTask.addFileset(src);
+
+                        copyTask.execute();
+                        return copyTask.getNumCopied();
+                    } catch (BuildException e) {
+                        throw new IOException2("Failed to copy "+base+"/"+fileMask+" to "+target,e);
+                    }
+                }
+            });
+        } else {
+            // remote copy
+            final FilePath src = this;
+
+            return target.act(new FileCallable<Integer>() {
+                // this code is executed on the node that receives files.
+                public Integer invoke(final File dest, VirtualChannel channel) throws IOException {
+                    final RemoteCopier copier = src.getChannel().export(
+                        RemoteCopier.class,
+                        new RemoteCopier() {
+                            private OutputStream os;
+                            public void open(String fileName) throws IOException {
+                                File file = new File(dest, fileName);
+                                file.getParentFile().mkdirs();
+                                os = new FileOutputStream(file);
+                            }
+
+                            public void write(byte[] buf, int len) throws IOException {
+                                os.write(buf,0,len);
+                            }
+
+                            public void close() throws IOException {
+                                os.close();
+                                os = null;
+                            }
+                        });
+
+                    try {
+                        return src.act(new FileCallable<Integer>() {
+                            public Integer invoke(File base, VirtualChannel channel) throws IOException {
+                                // copy to a remote node
+                                FileSet fs = new FileSet();
+                                fs.setDir(base);
+                                fs.setIncludes(fileMask);
+
+                                byte[] buf = new byte[8192];
+
+                                DirectoryScanner ds = fs.getDirectoryScanner(new org.apache.tools.ant.Project());
+                                String[] files = ds.getIncludedFiles();
+                                for( String f : files) {
+                                    File file = new File(base, f);
+
+                                    copier.open(f);
+
+                                    FileInputStream in = new FileInputStream(file);
+                                    int len;
+                                    while((len=in.read(buf))>=0)
+                                        copier.write(buf,len);
+                                    in.close();
+
+                                    copier.close();
+                                }
+                                return files.length;
+                            }
+                        });
+                    } catch (InterruptedException e) {
+                        throw new IOException2("Copy operation interrupted",e);
+                    }
+                }
+            });
+        }
+    }
+
     @Deprecated
     public String toString() {
         // to make writing JSPs easily, return local
-        return local.toString();
+        return remote;
     }
 
-    /**
-     * {@link FilePath} constant that can be used if the directory is not important.
-     */
-    public static final FilePath RANDOM = new FilePath(new File("."));
+    public VirtualChannel getChannel() {
+        if(channel!=null)   return channel;
+        else                return Hudson.MasterComputer.localChannel;
+    }
+
+    private void writeObject(ObjectOutputStream oos) throws IOException {
+        Channel target = Channel.current();
+
+        if(channel!=null && channel!=target)
+            throw new IllegalStateException("Can't send a remote FilePath to a different remote channel");
+
+        oos.defaultWriteObject();
+        oos.writeBoolean(channel==null);
+    }
+
+    private void readObject(ObjectInputStream ois) throws IOException, ClassNotFoundException {
+        Channel channel = Channel.current();
+        assert channel!=null;
+
+        ois.defaultReadObject();
+        if(ois.readBoolean()) {
+            this.channel = channel;
+        } else {
+            this.channel = null;
+        }
+    }
+
+    private static final long serialVersionUID = 1L;
 }