[SECURITY-1452]

This commit is contained in:
Jeff Thompson authored and Jenkins CERT CI committed 2021-01-11 16:53:00 +00:00
1 parent f576b2eb43
commit 71d2ecf1a4
10 files changed
+1899 -114

No files matched your search

+257 -9
View File
@@ -83,9 +83,10 @@ import java.net.URLConnection;
import java.nio.file.FileSystemException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.LinkOption;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.FileAttribute;
import java.nio.file.attribute.PosixFilePermission;
@@ -474,6 +475,25 @@ public final class FilePath implements SerializableOnlyOverRemoting {
return archive(ArchiverFactory.ZIP, out, scanner);
}
/**
* Uses the given scanner on 'this' directory to list up files and then archive it to a zip stream.
*
* @param out The OutputStream to write the zip into.
* @param scanner A DirScanner for scanning the directory and filtering its contents.
* @param verificationRoot A root or base directory for checking for any symlinks in this files parentage.
* Any symlinks between a file and root should be ignored.
* Symlinks in the parentage outside root will not be checked.
* @param noFollowLinks true if it should not follow links.
*
* @return The number of files/directories archived.
* This is only really useful to check for a situation where nothing
*/
@Restricted(NoExternalUse.class)
public int zip(OutputStream out, DirScanner scanner, String verificationRoot, boolean noFollowLinks) throws IOException, InterruptedException {
ArchiverFactory archiverFactory = noFollowLinks ? ArchiverFactory.ZIP_WTHOUT_FOLLOWING_SYMLINKS : ArchiverFactory.ZIP;
return archive(archiverFactory, out, scanner, verificationRoot, noFollowLinks);
}
/**
* Archives this directory into the specified archive format, to the given {@link OutputStream}, by using
* {@link DirScanner} to choose what files to include.
@@ -483,23 +503,48 @@ public final class FilePath implements SerializableOnlyOverRemoting {
* is archived.
*/
public int archive(final ArchiverFactory factory, OutputStream os, final DirScanner scanner) throws IOException, InterruptedException {
return archive(factory, os, scanner, null, false);
}
/**
* Archives this directory into the specified archive format, to the given {@link OutputStream}, by using
* {@link DirScanner} to choose what files to include.
*
* @param factory The ArchiverFactory for creating the archive.
* @param os The OutputStream to write the zip into.
* @param verificationRoot A root or base directory for checking for any symlinks in this files parentage.
* Any symlinks between a file and root should be ignored.
* Symlinks in the parentage outside root will not be checked.
* @param noFollowLinks true if it should not follow links.
*
* @return The number of files/directories archived.
* This is only really useful to check for a situation where nothing
*/
@Restricted(NoExternalUse.class)
public int archive(final ArchiverFactory factory, OutputStream os, final DirScanner scanner,
String verificationRoot, boolean noFollowLinks) throws IOException, InterruptedException {
final OutputStream out = (channel!=null)?new RemoteOutputStream(os):os;
return act(new Archive(factory, out, scanner));
return act(new Archive(factory, out, scanner, verificationRoot, noFollowLinks));
}
private class Archive extends SecureFileCallable<Integer> {
private final ArchiverFactory factory;
private final OutputStream out;
private final DirScanner scanner;
Archive(ArchiverFactory factory, OutputStream out, DirScanner scanner) {
private final String verificationRoot;
private final boolean noFollowLinks;
Archive(ArchiverFactory factory, OutputStream out, DirScanner scanner, String verificationRoot, boolean noFollowLinks) {
this.factory = factory;
this.out = out;
this.scanner = scanner;
this.verificationRoot = verificationRoot;
this.noFollowLinks = noFollowLinks;
}
@Override
public Integer invoke(File f, VirtualChannel channel) throws IOException {
Archiver a = factory.create(out);
try {
scanner.scan(f,reading(a));
scanner.scan(f, ignoringSymlinks(reading(a), verificationRoot, noFollowLinks));
} finally {
a.close();
}
@@ -691,6 +736,46 @@ public final class FilePath implements SerializableOnlyOverRemoting {
}
}
@Restricted(NoExternalUse.class)
public boolean hasSymlink(FilePath verificationRoot, boolean noFollowLinks) throws IOException, InterruptedException {
return act(new HasSymlink(verificationRoot == null ? null : verificationRoot.remote, noFollowLinks));
}
private static class HasSymlink extends SecureFileCallable<Boolean> {
private static final long serialVersionUID = 1L;
private final String verificationRoot;
private final boolean noFollowLinks;
public HasSymlink(String verificationRoot, boolean noFollowLinks) {
this.verificationRoot = verificationRoot;
this.noFollowLinks = noFollowLinks;
}
public Boolean invoke(File f, VirtualChannel channel) throws IOException {
return isSymlink(f, verificationRoot, noFollowLinks);
}
}
@Restricted(NoExternalUse.class)
public boolean containsSymlink(FilePath verificationRoot, boolean noFollowLinks) throws IOException, InterruptedException {
return !list(new SymlinkRetainingFileFilter(verificationRoot, noFollowLinks)).isEmpty();
}
private static class SymlinkRetainingFileFilter implements FileFilter, Serializable {
private final String verificationRoot;
private final boolean noFollowLinks;
public SymlinkRetainingFileFilter(FilePath verificationRoot, boolean noFollowLinks) {
this.verificationRoot = verificationRoot == null ? null : verificationRoot.remote;
this.noFollowLinks = noFollowLinks;
}
public boolean accept(File file) {
return isSymlink(file, verificationRoot, noFollowLinks);
}
private static final long serialVersionUID = 1L;
}
/**
* Creates a symlink to the specified target.
*
@@ -1838,6 +1923,23 @@ public final class FilePath implements SerializableOnlyOverRemoting {
return list((FileFilter)null);
}
/**
* List up files and directories in this directory.
*
* This is intended to allow the caller to provide {@link java.nio.file.LinkOption#NOFOLLOW_LINKS} to ignore
* symlinks.
* @param verificationRoot A root or base directory for checking for any symlinks in this files parentage.
* Any symlinks between a file and root should be ignored.
* Symlinks in the parentage outside root will not be checked.
* @param noFollowLinks true if it should not follow links.
* @return Direct children of this directory.
*/
@Restricted(NoExternalUse.class)
@NonNull
public List<FilePath> list(FilePath verificationRoot, boolean noFollowLinks) throws IOException, InterruptedException {
return list(new SymlinkDiscardingFileFilter(verificationRoot, noFollowLinks));
}
/**
* List up subdirectories.
*
@@ -1982,25 +2084,112 @@ public final class FilePath implements SerializableOnlyOverRemoting {
* Reads this file.
*/
public InputStream read() throws IOException, InterruptedException {
return read(null, false);
}
@Restricted(NoExternalUse.class)
public InputStream read(FilePath rootPath, boolean noFollowLinks) throws IOException, InterruptedException {
String rootPathString = rootPath == null ? null : rootPath.remote;
if(channel==null) {
return Files.newInputStream(fileToPath(reading(new File(remote))));
File file = reading(new File(remote));
InputStream inputStream = newInputStreamDenyingSymlinkAsNeeded(file, rootPathString, noFollowLinks);
return inputStream;
}
final Pipe p = Pipe.createRemoteToLocal();
actAsync(new Read(p));
actAsync(new Read(p, rootPathString, noFollowLinks));
return p.getIn();
}
@Restricted(NoExternalUse.class)
public static InputStream newInputStreamDenyingSymlinkAsNeeded(File file, String verificationRoot, boolean noFollowLinks) throws IOException {
InputStream inputStream = null;
try {
denySymlink(file, verificationRoot, noFollowLinks);
inputStream = noFollowLinks ? Files.newInputStream(fileToPath(file), LinkOption.NOFOLLOW_LINKS) : Files.newInputStream(fileToPath(file));
denySymlink(file, verificationRoot, noFollowLinks);
} catch (IOException ioe) {
if (inputStream != null) {
inputStream.close();
}
throw ioe;
}
return inputStream;
}
private static void denySymlink(File file, String root, boolean noFollowLinks) throws IOException {
/* This should be checked right before the file is opened or otherwise traversed.
If at all possible, it should also be checked immediately afterwards.
This narrows any possible race conditions that may exist in weird situations,
platforms, or implementations.
newInputStreamDenyingSymlinkAsNeeded(...) demonstrates how this would be done.
This is useful for preventing symlink following on systems that don't support
LinkOption.NOFOLLOW_LINK. Notable among those is AIX. It is also important for
prohibiting Windows Junctions, which are not considered symlinks by the
Files.newInputStream(path, LinkOption.NOFOLLOW_LINKS) implementation.
*/
if (isSymlink(file, root, noFollowLinks)) {
throw new IOException("Symlinks are prohibited.");
}
}
@Restricted(NoExternalUse.class)
public static boolean isSymlink(File file, String root, boolean noFollowLinks) {
if (noFollowLinks) {
if (Util.isSymlink(file.toPath())) {
return true;
}
return isFileAncestorSymlink(file, root);
}
return false;
}
/**
* Determines whether an ancestor of this file is a symlink, between the specified
* file and the root path. Ancestors further up the tree are not considered.
* @param file The base file for the beginning of the search.
* @param root The root path for ending the search.
* @return True if there is a symlink within the domain. False otherwise.
*/
private static boolean isFileAncestorSymlink(File file, String root) {
if (root != null) {
Path rootPath = Paths.get(root);
Path currPath = file.toPath();
try {
while (!getRealPath(currPath).equals(getRealPath(rootPath))) {
if (Util.isSymlink(currPath)) {
return true;
}
currPath = currPath.getParent();
if (currPath == null) {
return false;
}
}
} catch (IOException ioe) {
return false;
}
}
return false;
}
private class Read extends SecureFileCallable<Void> {
private static final long serialVersionUID = 1L;
private final Pipe p;
Read(Pipe p) {
private String verificationRoot;
private boolean noFollowLinks;
Read(Pipe p, String verificationRoot, boolean noFollowLinks) {
this.p = p;
this.verificationRoot = verificationRoot;
this.noFollowLinks = noFollowLinks;
}
@Override
public Void invoke(File f, VirtualChannel channel) throws IOException, InterruptedException {
try (InputStream fis = Files.newInputStream(fileToPath(reading(f)));
OutputStream out = p.getOut()) {
try (InputStream fis = newInputStreamDenyingSymlinkAsNeeded(reading(f), verificationRoot, noFollowLinks); OutputStream out = p.getOut()) {
org.apache.commons.io.IOUtils.copy(fis, out);
} catch (Exception x) {
p.error(x);
@@ -3249,6 +3438,29 @@ public final class FilePath implements SerializableOnlyOverRemoting {
};
}
/**
* Wraps {@link FileVisitor} to ignore symlinks.
*/
@Restricted(NoExternalUse.class)
public static FileVisitor ignoringSymlinks(final FileVisitor v, String verificationRoot, boolean noFollowLinks) {
if (noFollowLinks) {
return new FileVisitor() {
@Override
public void visit(File f, String relativePath) throws IOException {
if (verificationRoot == null || !FilePath.isSymlink(f, verificationRoot, noFollowLinks)) {
v.visit(f, relativePath);
}
}
@Override
public boolean understandsSymlink() {
return false;
}
};
}
return v;
}
/**
* Pass through 'f' after ensuring that we can read that file.
*/
@@ -3429,5 +3641,41 @@ public final class FilePath implements SerializableOnlyOverRemoting {
}
}
private static Path getRealPath(Path path) throws IOException {
return Functions.isWindows() ? windowsToRealPath(path) : path.toRealPath();
}
private static @NonNull Path windowsToRealPath(@NonNull Path path) throws IOException {
try {
return path.toRealPath();
}
catch (IOException e) {
if (LOGGER.isLoggable(Level.FINE)) {
LOGGER.log(Level.FINE, String.format("relaxedToRealPath cannot use the regular toRealPath on %s, trying with toRealPath(LinkOption.NOFOLLOW_LINKS)", path), e);
}
}
// that's required for specific environment like Windows Server 2016, running MSFT docker
// where the root is a <SYMLINKD>
return path.toRealPath(LinkOption.NOFOLLOW_LINKS);
}
private static final SoloFilePathFilter UNRESTRICTED = SoloFilePathFilter.wrap(FilePathFilter.UNRESTRICTED);
private static class SymlinkDiscardingFileFilter implements FileFilter, Serializable {
private final String verificationRoot;
private final boolean noFollowLinks;
public SymlinkDiscardingFileFilter(FilePath verificationRoot, boolean noFollowLinks) {
this.verificationRoot = verificationRoot == null ? null : verificationRoot.remote;
this.noFollowLinks = noFollowLinks;
}
public boolean accept(File file) {
return !isSymlink(file, verificationRoot, noFollowLinks);
}
private static final long serialVersionUID = 1L;
}
}
@@ -61,6 +61,7 @@ import jenkins.util.VirtualFile;
import org.apache.commons.io.IOUtils;
import org.apache.tools.zip.ZipEntry;
import org.apache.tools.zip.ZipOutputStream;
import org.apache.commons.lang.StringUtils;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.NoExternalUse;
import org.kohsuke.stapler.HttpResponse;
@@ -240,25 +241,23 @@ public final class DirectoryBrowserSupport implements HttpResponse {
String base = _base.toString();
String rest = _rest.toString();
if(!ALLOW_SYMLINK_ESCAPE && (root.supportIsDescendant() && !root.isDescendant(base))){
LOGGER.log(Level.WARNING, "Trying to access a file outside of the directory, target: "+ base);
rsp.sendError(HttpServletResponse.SC_FORBIDDEN, "Trying to access a file outside of the directory, target: " + base);
return;
}
// this is the base file/directory
VirtualFile baseFile = base.isEmpty() ? root : root.child(base);
if (baseFile.hasSymlink(getNoFollowLinks())) {
rsp.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
}
if(baseFile.isDirectory()) {
if(zip) {
rsp.setContentType("application/zip");
zip(rsp, root, baseFile, rest);
baseFile.zip(rsp.getOutputStream(), StringUtils.isBlank(rest) ? "**" : rest, null, true, getNoFollowLinks());
return;
}
if (plain) {
rsp.setContentType("text/plain;charset=UTF-8");
try (OutputStream os = rsp.getOutputStream()) {
for (VirtualFile kid : baseFile.list()) {
for (VirtualFile kid : baseFile.list(getNoFollowLinks())) {
os.write(kid.getName().getBytes(StandardCharsets.UTF_8));
if (kid.isDirectory()) {
os.write('/');
@@ -281,29 +280,30 @@ public final class DirectoryBrowserSupport implements HttpResponse {
List<List<Path>> glob = null;
boolean patternUsed = rest.length() > 0;
boolean containsSymlink = false;
if(patternUsed) {
// the rest is Ant glob pattern
glob = patternScan(baseFile, rest, createBackRef(restSize));
} else
if(serveDirIndex) {
// serve directory index
glob = baseFile.run(new BuildChildPaths(baseFile, req.getLocale()));
glob = baseFile.run(new BuildChildPaths(root, baseFile, req.getLocale()));
containsSymlink = baseFile.containsSymLinkChild(getNoFollowLinks());
}
if(glob!=null) {
List<List<Path>> filteredGlob = keepReadabilityOnlyOnDescendants(baseFile, patternUsed, glob);
// serve glob
req.setAttribute("it", this);
List<Path> parentPaths = buildParentPath(base,restSize);
req.setAttribute("parentPath",parentPaths);
req.setAttribute("backPath", createBackRef(restSize));
req.setAttribute("topPath", createBackRef(parentPaths.size()+restSize));
req.setAttribute("files", filteredGlob);
req.setAttribute("files", glob);
req.setAttribute("icon", icon);
req.setAttribute("path", path);
req.setAttribute("pattern",rest);
req.setAttribute("dir", baseFile);
req.setAttribute("showSymlinkWarning", containsSymlink);
if (ResourceDomainConfiguration.isResourceRequest(req)) {
req.getView(this, "plaindir.jelly").forward(req, rsp);
} else {
@@ -346,12 +346,19 @@ public final class DirectoryBrowserSupport implements HttpResponse {
if(LOGGER.isLoggable(Level.FINE))
LOGGER.fine("Serving "+baseFile+" with lastModified=" + lastModified + ", length=" + length);
InputStream in;
try {
in = baseFile.open(getNoFollowLinks());
} catch (IOException ioe) {
rsp.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
}
if (view) {
// for binary files, provide the file name for download
rsp.setHeader("Content-Disposition", "inline; filename=" + baseFile.getName());
// pseudo file name to let the Stapler set text/plain
rsp.serveFile(req, baseFile.open(), lastModified, -1, length, "plain.txt");
rsp.serveFile(req, in, lastModified, -1, length, "plain.txt");
} else {
if (resourceToken != null) {
// redirect to second domain
@@ -675,9 +682,11 @@ public final class DirectoryBrowserSupport implements HttpResponse {
}
private static final class BuildChildPaths extends MasterToSlaveCallable<List<List<Path>>,IOException> {
private VirtualFile root;
private final VirtualFile cur;
private final Locale locale;
BuildChildPaths(VirtualFile cur, Locale locale) {
BuildChildPaths(VirtualFile root, VirtualFile cur, Locale locale) {
this.root = root;
this.cur = cur;
this.locale = locale;
}
@@ -693,7 +702,7 @@ public final class DirectoryBrowserSupport implements HttpResponse {
private static List<List<Path>> buildChildPaths(VirtualFile cur, Locale locale) throws IOException {
List<List<Path>> r = new ArrayList<>();
VirtualFile[] files = cur.list();
VirtualFile[] files = cur.list(getNoFollowLinks());
Arrays.sort(files,new FileComparator(locale));
for( VirtualFile f : files ) {
@@ -708,7 +717,7 @@ public final class DirectoryBrowserSupport implements HttpResponse {
while(true) {
// files that don't start with '.' qualify for 'meaningful files', nor SCM related files
List<VirtualFile> sub = new ArrayList<>();
for (VirtualFile vf : f.list()) {
for (VirtualFile vf : f.list(getNoFollowLinks())) {
String name = vf.getName();
if (!name.startsWith(".") && !name.equals("CVS") && !name.equals(".svn")) {
sub.add(vf);
@@ -733,7 +742,7 @@ public final class DirectoryBrowserSupport implements HttpResponse {
* @param baseRef String like "../../../" that cancels the 'rest' portion. Can be "./"
*/
private static List<List<Path>> patternScan(VirtualFile baseDir, String pattern, String baseRef) throws IOException {
Collection<String> files = baseDir.list(pattern, null, /* TODO what is the user expectation? */true);
Collection<String> files = baseDir.list(pattern, null, /* TODO what is the user expectation? */true, getNoFollowLinks());
if (!files.isEmpty()) {
List<List<Path>> r = new ArrayList<>(files.size());
@@ -776,6 +785,9 @@ public final class DirectoryBrowserSupport implements HttpResponse {
pathList.add(path);
}
private static boolean getNoFollowLinks() {
return !ALLOW_SYMLINK_ESCAPE;
}
private static final Logger LOGGER = Logger.getLogger(DirectoryBrowserSupport.class.getName());
+16 -7
View File
@@ -6,6 +6,8 @@ import org.apache.tools.ant.BuildException;
import org.apache.tools.ant.DirectoryScanner;
import org.apache.tools.ant.types.FileSet;
import org.apache.tools.ant.types.selectors.FileSelector;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.NoExternalUse;
import java.io.File;
import java.io.FileFilter;
@@ -103,15 +105,25 @@ public abstract class DirScanner implements Serializable {
private final String includes, excludes;
private boolean useDefaultExcludes = true;
private final boolean followSymlinks;
public Glob(String includes, String excludes) {
this.includes = includes;
this.excludes = excludes;
this(includes, excludes, true, true);
}
public Glob(String includes, String excludes, boolean useDefaultExcludes) {
this(includes, excludes);
this(includes, excludes, useDefaultExcludes, true);
}
/**
* @since TODO
*/
@Restricted(NoExternalUse.class)
public Glob(String includes, String excludes, boolean useDefaultExcludes, boolean followSymlinks) {
this.includes = includes;
this.excludes = excludes;
this.useDefaultExcludes = useDefaultExcludes;
this.followSymlinks = followSymlinks;
}
public void scan(File dir, FileVisitor visitor) throws IOException {
@@ -122,14 +134,11 @@ public abstract class DirScanner implements Serializable {
}
FileSet fs = Util.createFileSet(dir,includes,excludes);
fs.setFollowSymlinks(followSymlinks);
fs.setDefaultexcludes(useDefaultExcludes);
fs.appendSelector(new DescendantFileSelector(fs.getDir()));
if(dir.exists()) {
DirectoryScanner ds = fs.getDirectoryScanner(new org.apache.tools.ant.Project());
// due to the DescendantFileSelector usage,
// the includedFiles are only the ones that are descendant
for( String f : ds.getIncludedFiles()) {
File file = new File(dir, f);
scanSingle(file, f, visitor);
@@ -25,6 +25,8 @@
package hudson.util.io;
import hudson.FilePath.TarCompression;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.NoExternalUse;
import java.io.IOException;
import java.io.OutputStream;
@@ -57,6 +59,11 @@ public abstract class ArchiverFactory implements Serializable {
*/
public static ArchiverFactory ZIP = new ZipArchiverFactory();
/**
* Zip format, without following symlinks.
*/
@Restricted(NoExternalUse.class)
public static ArchiverFactory ZIP_WTHOUT_FOLLOWING_SYMLINKS = new ZipWithoutSymLinksArchiverFactory();
private static final class TarArchiverFactory extends ArchiverFactory {
@@ -81,5 +88,13 @@ public abstract class ArchiverFactory implements Serializable {
private static final long serialVersionUID = 1L;
}
private static final class ZipWithoutSymLinksArchiverFactory extends ArchiverFactory {
public Archiver create(OutputStream out) {
return new ZipArchiver(out, true);
}
private static final long serialVersionUID = 1L;
}
private static final long serialVersionUID = 1L;
}
@@ -36,6 +36,8 @@ import org.apache.tools.zip.ZipOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.LinkOption;
import java.nio.file.OpenOption;
/**
* {@link FileVisitor} that creates a zip archive.
@@ -45,9 +47,15 @@ import java.io.OutputStream;
final class ZipArchiver extends Archiver {
private final byte[] buf = new byte[8192];
private final ZipOutputStream zip;
private final OpenOption[] openOptions;
ZipArchiver(OutputStream out) {
this(out, false);
}
ZipArchiver(OutputStream out, boolean failOnSymLink) {
zip = new ZipOutputStream(out);
openOptions = failOnSymLink ? new LinkOption[]{LinkOption.NOFOLLOW_LINKS} : new OpenOption[0];
zip.setEncoding(System.getProperty("file.encoding"));
zip.setUseZip64(Zip64Mode.AsNeeded);
}
@@ -73,7 +81,7 @@ final class ZipArchiver extends Archiver {
if (mode!=-1) fileZipEntry.setUnixMode(mode);
fileZipEntry.setTime(f.lastModified());
zip.putNextEntry(fileZipEntry);
try (InputStream in = Files.newInputStream(f.toPath())) {
try (InputStream in = Files.newInputStream(f.toPath(), openOptions)) {
int len;
while((len=in.read(buf))>=0)
zip.write(buf,0,len);
+241 -15
View File
@@ -39,6 +39,7 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Serializable;
import java.net.URI;
import java.net.URL;
@@ -57,6 +58,8 @@ import java.util.stream.Collectors;
import edu.umd.cs.findbugs.annotations.CheckForNull;
import edu.umd.cs.findbugs.annotations.NonNull;
import hudson.util.io.Archiver;
import hudson.util.io.ArchiverFactory;
import jenkins.MasterToSlaveFileCallable;
import jenkins.model.ArtifactManager;
import jenkins.security.MasterToSlaveCallable;
@@ -173,10 +176,44 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
*/
public abstract @NonNull VirtualFile[] list() throws IOException;
/**
* Lists children of this directory. Only one level deep.
*
* This is intended to allow the caller to provide {@link java.nio.file.LinkOption#NOFOLLOW_LINKS} to ignore
* symlinks. However, this cannot be enforced. The base implementation here in VirtualFile ignores the openOptions.
* Some VirtualFile subclasses may not be able to provide
* an implementation in which NOFOLLOW_LINKS is used or makes sense. Implementations are free
* to ignore openOptions. Some subclasses of VirtualFile may not have a concept of symlinks.
* @param noFollowLinks if true then do not follow links.
* @return a list of children (files and subdirectories); empty for a file or nonexistent directory
* @throws IOException if it could not be opened
*/
@Restricted(NoExternalUse.class)
public @NonNull VirtualFile[] list(boolean noFollowLinks) throws IOException {
return list();
}
@Restricted(NoExternalUse.class)
public boolean supportsQuickRecursiveListing() {
return false;
}
/**
* Determines when a VirtualFile has a recognized symlink.
* A recognized symlink can be the file itself or any containing directory between
* it and the optional root directory. If there is no provided root directory then
* only the file itself is considered.
*
* This base implementation ignores the existence of symlinks.
* @param noFollowLinks if true, then do not follow links.
* @return True if the file is a symlink or is referenced within a containing symlink
* directory before reaching the root directory.
* @throws IOException If there is a problem accessing the file.
*/
@Restricted(NoExternalUse.class)
public boolean hasSymlink(boolean noFollowLinks) throws IOException {
return false;
}
/**
* Lists only the children that are descendant of the root directory (not necessarily the current VirtualFile).
@@ -218,6 +255,30 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
* @since 2.118
*/
public @NonNull Collection<String> list(@NonNull String includes, @CheckForNull String excludes, boolean useDefaultExcludes) throws IOException {
return list(includes, excludes, useDefaultExcludes, false);
}
/**
* Lists recursive files of this directory with pattern matching.
*
* <p>The default implementation calls {@link #list()} recursively inside {@link #run} and applies filtering to the result.
* Implementations may wish to override this more efficiently.
* This method allows the user to specify that symlinks should not be followed by passing
* noFollowLinks as true. However, some implementations may not be able to reliably
* prevent link following. The base implementation here in VirtualFile ignores this parameter.
* @param includes comma-separated Ant-style globs as per {@link Util#createFileSet(File, String, String)} using {@code /} as a path separator;
* the empty string means <em>no matches</em> (use {@link SelectorUtils#DEEP_TREE_MATCH} if you want to match everything except some excludes)
* @param excludes optional excludes in similar format to {@code includes}
* @param useDefaultExcludes as per {@link AbstractFileSet#setDefaultexcludes}
* @param noFollowLinks if true then do not follow links.
* @return a list of {@code /}-separated relative names of children (files directly inside or in subdirectories)
* @throws IOException if this is not a directory, or listing was not possible for some other reason
* @since TODO
*/
@Restricted(NoExternalUse.class)
public @NonNull Collection<String> list(@NonNull String includes, @CheckForNull String excludes, boolean useDefaultExcludes,
boolean noFollowLinks) throws IOException {
Collection<String> r = run(new CollectFiles(this));
List<TokenizedPattern> includePatterns = patterns(includes);
List<TokenizedPattern> excludePatterns = patterns(excludes);
@@ -231,6 +292,12 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
return includePatterns.stream().anyMatch(patt -> patt.matchPath(path, true)) && excludePatterns.stream().noneMatch(patt -> patt.matchPath(path, true));
}).collect(Collectors.toSet());
}
@Restricted(NoExternalUse.class)
public boolean containsSymLinkChild(boolean noFollowLinks) throws IOException {
return false;
}
private static final class CollectFiles extends MasterToSlaveCallable<Collection<String>, IOException> {
private static final long serialVersionUID = 1;
private final VirtualFile root;
@@ -266,6 +333,15 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
return r;
}
/**
* @since TODO
*/
@Restricted(NoExternalUse.class)
public int zip(OutputStream outputStream, String includes, String excludes, boolean useDefaultExcludes,
boolean noFollowLinks) throws IOException, UnsupportedOperationException {
throw new UnsupportedOperationException("Not implemented.");
}
/**
* Obtains a child file.
* @param name a relative path, possibly including {@code /} (but not {@code ..})
@@ -312,6 +388,18 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
*/
public abstract InputStream open() throws IOException;
/**
* Opens an input stream on the file so its contents can be read.
*
* @param noFollowLinks if true do not follow links.
* @return an open stream
* @throws IOException if it could not be opened
*/
@Restricted(NoExternalUse.class)
public InputStream open(boolean noFollowLinks) throws IOException {
return open();
}
/**
* Does case-insensitive comparison.
* {@inheritDoc}
@@ -470,6 +558,20 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
return vfs;
}
@NonNull
@Override
public VirtualFile[] list(boolean noFollowLinks) throws IOException {
String rootPath = determineRootPath();
File[] kids = f.listFiles();
List<VirtualFile> contents = new ArrayList<>(kids.length);
for (File child : kids) {
if (!FilePath.isSymlink(child, rootPath, noFollowLinks)) {
contents.add(new FileVF(child, root));
}
}
return contents.toArray(new VirtualFile[0]);
}
@Override public boolean supportsQuickRecursiveListing() {
return true;
}
@@ -499,6 +601,32 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
}
return new Scanner(includes, excludes, useDefaultExcludes).invoke(f, null);
}
@Override
public Collection<String> list(String includes, String excludes, boolean useDefaultExcludes,
boolean noFollowLinks) throws IOException {
String rootPath = determineRootPath();
return new Scanner(includes, excludes, useDefaultExcludes, rootPath, noFollowLinks).invoke(f, null);
}
@Override
public int zip(OutputStream outputStream, String includes, String excludes, boolean useDefaultExcludes,
boolean noFollowLinks) throws IOException {
String rootPath = determineRootPath();
DirScanner.Glob globScanner = new DirScanner.Glob(includes, excludes, useDefaultExcludes, !noFollowLinks);
ArchiverFactory archiverFactory = noFollowLinks ? ArchiverFactory.ZIP_WTHOUT_FOLLOWING_SYMLINKS : ArchiverFactory.ZIP;
try (Archiver archiver = archiverFactory.create(outputStream)) {
globScanner.scan(f, FilePath.ignoringSymlinks(archiver, rootPath, noFollowLinks));
return archiver.countEntries();
}
}
@Override
public boolean hasSymlink(boolean noFollowLinks) throws IOException {
String rootPath = determineRootPath();
return FilePath.isSymlink(f, rootPath, noFollowLinks);
}
@Override public VirtualFile child(String name) {
return new FileVF(new File(f, name), root);
}
@@ -536,6 +664,28 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
throw new IOException(e);
}
}
@Override
public InputStream open(boolean noFollowLinks) throws IOException {
String rootPath = determineRootPath();
InputStream inputStream = FilePath.newInputStreamDenyingSymlinkAsNeeded(f, rootPath, noFollowLinks);
return inputStream;
}
@Override
public boolean containsSymLinkChild(boolean noFollowLinks) {
String rootPath = determineRootPath();
File[] kids = f.listFiles();
for (File child : kids) {
if (FilePath.isSymlink(child, rootPath, noFollowLinks)) {
return true;
}
}
return false;
}
private String determineRootPath() {
return root == null ? null : root.getPath();
}
private boolean isIllegalSymlink() {
try {
@@ -680,11 +830,44 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
@Override public VirtualFile[] list() throws IOException {
try {
List<FilePath> kids = f.list();
VirtualFile[] vfs = new VirtualFile[kids.size()];
for (int i = 0; i < vfs.length; i++) {
vfs[i] = new FilePathVF(kids.get(i), this.root);
}
return vfs;
return convertChildrenToVirtualFile(kids);
} catch (InterruptedException x) {
throw new IOException(x);
}
}
private VirtualFile[] convertChildrenToVirtualFile(List<FilePath> kids) {
VirtualFile[] vfs = new VirtualFile[kids.size()];
for (int i = 0; i < vfs.length; i++) {
vfs[i] = new FilePathVF(kids.get(i), this.root);
}
return vfs;
}
@NonNull
@Override
public VirtualFile[] list(boolean noFollowLinks) throws IOException {
try {
List<FilePath> kids = f.list(root, noFollowLinks);
return convertChildrenToVirtualFile(kids);
} catch (InterruptedException x) {
throw new IOException(x);
}
}
@Override
public boolean containsSymLinkChild(boolean noFollowLinks) throws IOException {
try {
return f.containsSymlink(root, noFollowLinks);
} catch (InterruptedException x) {
throw new IOException(x);
}
}
@Override
public boolean hasSymlink(boolean noFollowLinks) throws IOException {
try {
return f.hasSymlink(root, noFollowLinks);
} catch (InterruptedException x) {
throw new IOException(x);
}
@@ -716,13 +899,37 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
}
}
@Override public Collection<String> list(String includes, String excludes, boolean useDefaultExcludes) throws IOException {
try {
return f.act(new Scanner(includes, excludes, useDefaultExcludes));
} catch (InterruptedException x) {
throw new IOException(x);
@Override public Collection<String> list(String includes, String excludes, boolean useDefaultExcludes) throws IOException {
try {
return f.act(new Scanner(includes, excludes, useDefaultExcludes));
} catch (InterruptedException x) {
throw new IOException(x);
}
}
}
@Override
public Collection<String> list(String includes, String excludes, boolean useDefaultExcludes,
boolean noFollowLinks) throws IOException {
try {
String rootPath = root == null ? null : root.getRemote();
return f.act(new Scanner(includes, excludes, useDefaultExcludes, rootPath, noFollowLinks));
} catch (InterruptedException x) {
throw new IOException(x);
}
}
@Override
public int zip(OutputStream outputStream, String includes, String excludes, boolean useDefaultExcludes,
boolean noFollowLinks) throws IOException {
try {
String rootPath = root == null ? null : root.getRemote();
DirScanner.Glob globScanner = new DirScanner.Glob(includes, excludes, useDefaultExcludes, !noFollowLinks);
return f.zip(outputStream, globScanner, rootPath, noFollowLinks);
} catch (InterruptedException x) {
throw new IOException(x);
}
}
@Override public VirtualFile child(String name) {
return new FilePathVF(f.child(name), this.root);
}
@@ -761,6 +968,13 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
throw new IOException(x);
}
}
@Override public InputStream open(boolean noFollowLinks) throws IOException {
try {
return f.read(root == null ? null : root, noFollowLinks);
} catch (InterruptedException x) {
throw new IOException(x);
}
}
@Override public <V> V run(Callable<V,IOException> callable) throws IOException {
try {
return f.act(callable);
@@ -841,22 +1055,34 @@ public abstract class VirtualFile implements Comparable<VirtualFile>, Serializab
private static final class Scanner extends MasterToSlaveFileCallable<List<String>> {
private final String includes, excludes;
private final boolean useDefaultExcludes;
Scanner(String includes, String excludes, boolean useDefaultExcludes) {
private final String verificationRoot;
private final boolean noFollowLinks;
Scanner(String includes, String excludes, boolean useDefaultExcludes, String verificationRoot, boolean noFollowLinks) {
this.includes = includes;
this.excludes = excludes;
this.useDefaultExcludes = useDefaultExcludes;
this.verificationRoot = verificationRoot;
this.noFollowLinks = noFollowLinks;
}
public Scanner(String includes, String excludes, boolean useDefaultExcludes) {
this(includes, excludes, useDefaultExcludes, null, false);
}
@Override public List<String> invoke(File f, VirtualChannel channel) throws IOException {
if (includes.isEmpty()) { // see Glob class Javadoc, and list(String, String, boolean) note
return Collections.emptyList();
}
final List<String> paths = new ArrayList<>();
new DirScanner.Glob(includes, excludes, useDefaultExcludes).scan(f, new FileVisitor() {
FileVisitor listing = new FileVisitor() {
@Override
public void visit(File f, String relativePath) throws IOException {
public void visit(File f, String relativePath) {
paths.add(relativePath.replace('\\', '/'));
}
});
};
DirScanner.Glob globScanner = new DirScanner.Glob(includes, excludes, useDefaultExcludes, !noFollowLinks);
globScanner.scan(f, FilePath.ignoringSymlinks(listing, verificationRoot, noFollowLinks));
return paths;
}