OpenSource/jenkins
1
0 Fork 0
mirror of https://github.com/jenkinsci/jenkins.git synced 2026-11-04 15:45:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3841 from jvz/util-delete-regression

Browse Source 
[JENKINS-55448] Util delete regression
This commit is contained in:
Jeff Thompson authored and GitHub committed 2019-01-18 16:24:19 +00:00
2 parents c6d6d25ef0 f433a5db9e
commit bf3f0fbf7e
11 files changed
+1153 -582
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

core/src/main/java/hudson/Util.java
+37 -238
View File
@@ -23,14 +23,13 @@
*/
package hudson;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import hudson.model.TaskListener;
import jenkins.util.MemoryReductionUtil;
import hudson.util.QuotedStringTokenizer;
import hudson.util.VariableResolver;
import jenkins.util.SystemProperties;
import jenkins.util.io.PathRemover;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.io.output.NullOutputStream;
@@ -62,13 +61,11 @@ import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.InvalidPathException;
import java.nio.file.LinkOption;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.attribute.PosixFileAttributes;
import java.nio.file.attribute.DosFileAttributes;
import java.nio.file.attribute.PosixFilePermission;
import java.nio.file.attribute.PosixFilePermissions;
import java.security.DigestInputStream;
import java.security.MessageDigest;
@@ -249,16 +246,18 @@ public class Util {
* if the operation fails.
*/
public static void deleteContentsRecursive(@Nonnull File file) throws IOException {
for( int numberOfAttempts=1 ; ; numberOfAttempts++ ) {
try {
tryOnceDeleteContentsRecursive(file);
break; // success
} catch (IOException ex) {
boolean threadWasInterrupted = pauseBetweenDeletes(numberOfAttempts);
if( numberOfAttempts>= DELETION_MAX || threadWasInterrupted)
throw new IOException(deleteFailExceptionMessage(file, numberOfAttempts, threadWasInterrupted), ex);
}
}
deleteContentsRecursive(fileToPath(file), PathRemover.PathChecker.ALLOW_ALL);
}
/**
* Deletes the given directory contents (but not the directory itself) recursively using a PathChecker.
* @param path a directory to delete
* @param pathChecker a security check to validate a path before deleting
* @throws IOException if the operation fails
*/
@Restricted(NoExternalUse.class)
public static void deleteContentsRecursive(@Nonnull Path path, @Nonnull PathRemover.PathChecker pathChecker) throws IOException {
newPathRemover(pathChecker).forceRemoveDirectoryContents(path);
}
/**
@@ -269,88 +268,7 @@ public class Util {
* @throws IOException if it exists but could not be successfully deleted
*/
public static void deleteFile(@Nonnull File f) throws IOException {
for( int numberOfAttempts=1 ; ; numberOfAttempts++ ) {
try {
tryOnceDeleteFile(f);
break; // success
} catch (IOException ex) {
boolean threadWasInterrupted = pauseBetweenDeletes(numberOfAttempts);
if( numberOfAttempts>= DELETION_MAX || threadWasInterrupted)
throw new IOException(deleteFailExceptionMessage(f, numberOfAttempts, threadWasInterrupted), ex);
}
}
}
/**
* Deletes this file, working around most problems which might make
* this difficult.
*
* @param f
* What to delete. If a directory, it'll need to be empty.
* @throws IOException if it exists but could not be successfully deleted,
* or if it represents an invalid {@link Path}.
*/
private static void tryOnceDeleteFile(File f) throws IOException {
Path path = fileToPath(f);
try {
Files.deleteIfExists(path);
} catch (IOException e) {
// perhaps this file is read-only?
makeWritable(path);
/*
on Unix both the file and the directory that contains it has to be writable
for a file deletion to be successful. (Confirmed on Solaris 9)
$ ls -la
total 6
dr-xr-sr-x 2 hudson hudson 512 Apr 18 14:41 .
dr-xr-sr-x 3 hudson hudson 512 Apr 17 19:36 ..
-r--r--r-- 1 hudson hudson 469 Apr 17 19:36 manager.xml
-rw-r--r-- 1 hudson hudson 0 Apr 18 14:41 x
$ rm x
rm: x not removed: Permission denied
*/
Path parent = path.getParent();
if (parent != null) {
makeWritable(parent);
}
try {
Files.deleteIfExists(path);
} catch (IOException e2) {
// see https://java.net/projects/hudson/lists/users/archive/2008-05/message/357
// I suspect other processes putting files in this directory
File[] files = f.listFiles();
if(files!=null && files.length>0)
throw new IOException("Unable to delete " + f.getPath()+" - files in dir: "+Arrays.asList(files), e2);
throw e2;
}
}
}
/**
* Makes the file at the given path writable by any means possible.
*/
private static void makeWritable(@Nonnull Path path) throws IOException {
if (!Functions.isWindows()) {
try {
PosixFileAttributes attrs = Files.readAttributes(path, PosixFileAttributes.class);
Set<PosixFilePermission> newPermissions = attrs.permissions();
newPermissions.add(PosixFilePermission.OWNER_WRITE);
Files.setPosixFilePermissions(path, newPermissions);
return;
} catch (NoSuchFileException e) {
return;
} catch (UnsupportedOperationException e) {
// PosixFileAttributes not supported, fall back to old IO.
}
}
/**
* We intentionally do not check the return code of setWritable, because if it
* is false we prefer to rethrow the exception thrown by Files.deleteIfExists,
* which will have a more useful message than something we make up here.
*/
path.toFile().setWritable(true);
newPathRemover(PathRemover.PathChecker.ALLOW_ALL).forceRemoveFile(fileToPath(f));
}
/**
@@ -362,137 +280,18 @@ public class Util {
* if the operation fails.
*/
public static void deleteRecursive(@Nonnull File dir) throws IOException {
for( int numberOfAttempts=1 ; ; numberOfAttempts++ ) {
try {
tryOnceDeleteRecursive(dir);
break; // success
} catch (IOException ex) {
boolean threadWasInterrupted = pauseBetweenDeletes(numberOfAttempts);
if( numberOfAttempts>= DELETION_MAX || threadWasInterrupted)
throw new IOException(deleteFailExceptionMessage(dir, numberOfAttempts, threadWasInterrupted), ex);
}
}
deleteRecursive(fileToPath(dir), PathRemover.PathChecker.ALLOW_ALL);
}
/**
* Deletes a file or folder, throwing the first exception encountered, but
* having a go at deleting everything. i.e. it does not <em>stop</em> on the
* first exception, but tries (to delete) everything once.
*
* @param dir
* What to delete. If a directory, the contents will be deleted
* too.
* @throws The first exception encountered.
* Deletes the given directory and contents recursively using a filter.
* @param dir a directory to delete
* @param pathChecker a security check to validate a path before deleting
* @throws IOException if the operation fails
*/
private static void tryOnceDeleteRecursive(File dir) throws IOException {
if(!isSymlink(dir))
tryOnceDeleteContentsRecursive(dir);
tryOnceDeleteFile(dir);
}
/**
* Deletes a folder's contents, throwing the first exception encountered,
* but having a go at deleting everything. i.e. it does not <em>stop</em>
* on the first exception, but tries (to delete) everything once.
*
* @param directory
* The directory whose contents will be deleted.
* @throws The first exception encountered.
*/
private static void tryOnceDeleteContentsRecursive(File directory) throws IOException {
File[] directoryContents = directory.listFiles();
if(directoryContents==null)
return; // the directory didn't exist in the first place
IOException firstCaught = null;
for (File child : directoryContents) {
try {
tryOnceDeleteRecursive(child);
} catch (IOException justCaught) {
if( firstCaught==null) {
firstCaught = justCaught;
}
}
}
if( firstCaught!=null )
throw firstCaught;
}
/**
* Pauses between delete attempts, and says if it's ok to try again.
* This does not wait if the wait time is zero or if we have tried
* too many times already.
* <p>
* See {@link #WAIT_BETWEEN_DELETION_RETRIES} for details of
* the pause duration.<br/>
* See {@link #GC_AFTER_FAILED_DELETE} for when {@link System#gc()} is called.
*
* @return false if it is ok to continue trying to delete things, true if
* we were interrupted (and should stop now).
*/
@SuppressFBWarnings(value = "DM_GC", justification = "Garbage collection happens only when "
+ "GC_AFTER_FAILED_DELETE is true. It's an experimental feature in Jenkins.")
private static boolean pauseBetweenDeletes(int numberOfAttemptsSoFar) {
long delayInMs;
if( numberOfAttemptsSoFar>=DELETION_MAX ) return false;
/* If the Jenkins process had the file open earlier, and it has not
* closed it then Windows won't let us delete it until the Java object
* with the open stream is Garbage Collected, which can result in builds
* failing due to "file in use" on Windows despite working perfectly
* well on other OSs. */
if (GC_AFTER_FAILED_DELETE) {
System.gc();
}
if (WAIT_BETWEEN_DELETION_RETRIES>=0) {
delayInMs = WAIT_BETWEEN_DELETION_RETRIES;
} else {
delayInMs = -numberOfAttemptsSoFar*WAIT_BETWEEN_DELETION_RETRIES;
}
if (delayInMs<=0)
return Thread.interrupted();
try {
Thread.sleep(delayInMs);
return false;
} catch (InterruptedException e) {
return true;
}
}
/**
* Creates a "couldn't delete file" message that explains how hard we tried.
* See {@link #DELETION_MAX}, {@link #WAIT_BETWEEN_DELETION_RETRIES}
* and {@link #GC_AFTER_FAILED_DELETE} for more details.
*/
private static String deleteFailExceptionMessage(File whatWeWereTryingToRemove, int retryCount, boolean wasInterrupted) {
StringBuilder sb = new StringBuilder();
sb.append("Unable to delete '");
sb.append(whatWeWereTryingToRemove);
sb.append("'. Tried ");
sb.append(retryCount);
sb.append(" time");
if( retryCount!=1 ) sb.append('s');
if( DELETION_MAX>1 ) {
sb.append(" (of a maximum of ");
sb.append(DELETION_MAX);
sb.append(')');
if( GC_AFTER_FAILED_DELETE )
sb.append(" garbage-collecting");
if( WAIT_BETWEEN_DELETION_RETRIES!=0 && GC_AFTER_FAILED_DELETE )
sb.append(" and");
if( WAIT_BETWEEN_DELETION_RETRIES!=0 ) {
sb.append(" waiting ");
sb.append(getTimeSpanString(Math.abs(WAIT_BETWEEN_DELETION_RETRIES)));
if( WAIT_BETWEEN_DELETION_RETRIES<0 ) {
sb.append("-");
sb.append(getTimeSpanString(Math.abs(WAIT_BETWEEN_DELETION_RETRIES)*DELETION_MAX));
}
}
if( WAIT_BETWEEN_DELETION_RETRIES!=0 || GC_AFTER_FAILED_DELETE)
sb.append(" between attempts");
}
if( wasInterrupted )
sb.append(". The delete operation was interrupted before it completed successfully");
sb.append('.');
return sb.toString();
@Restricted(NoExternalUse.class)
public static void deleteRecursive(@Nonnull Path dir, @Nonnull PathRemover.PathChecker pathChecker) throws IOException {
newPathRemover(pathChecker).forceRemoveRecursive(dir);
}
/*
@@ -511,9 +310,16 @@ public class Util {
* limitations under the License.
*/
/**
* Checks if the given file represents a symlink.
* Checks if the given file represents a symlink. Unlike {@link Files#isSymbolicLink(Path)}, this method also
* considers <a href="https://en.wikipedia.org/wiki/NTFS_junction_point">NTFS junction points</a> as symbolic
* links.
*/
public static boolean isSymlink(@Nonnull File file) throws IOException {
return isSymlink(fileToPath(file));
}
@Restricted(NoExternalUse.class)
public static boolean isSymlink(@Nonnull Path path) {
/*
* Windows Directory Junctions are effectively the same as Linux symlinks to directories.
* Unfortunately, the Java 7 NIO2 API function isSymbolicLink does not treat them as such.
@@ -527,20 +333,9 @@ public class Util {
* calling readAttributes.
*/
try {
Path path = fileToPath(file);
BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS);
if (attrs.isSymbolicLink()) {
return true;
} else if (attrs instanceof DosFileAttributes) {
/* Returns true for non-symbolic link reparse points and devices. We could call
* WindowsFileAttributes#isReparsePoint with reflection instead to exclude devices,
* but as mentioned in the above comment this does not appear to be an issue.
*/
return attrs.isOther();
} else {
return false;
}
} catch (NoSuchFileException e) {
return attrs.isSymbolicLink() || (attrs instanceof DosFileAttributes && attrs.isOther());
} catch (IOException ignored) {
return false;
}
}
@@ -1812,6 +1607,10 @@ public class Util {
@Restricted(value = NoExternalUse.class)
static boolean GC_AFTER_FAILED_DELETE = SystemProperties.getBoolean(Util.class.getName() + ".performGCOnFailedDelete");
private static PathRemover newPathRemover(@Nonnull PathRemover.PathChecker pathChecker) {
return PathRemover.newFilteredRobustRemover(pathChecker, DELETION_MAX - 1, GC_AFTER_FAILED_DELETE, WAIT_BETWEEN_DELETION_RETRIES);
}
/**
* If this flag is true, native implementations of {@link FilePath#chmod}
* and {@link hudson.util.IOUtils#mode} are used instead of NIO.