OpenSource/jenkins
1
0 Fork 0
mirror of https://github.com/jenkinsci/jenkins.git synced 2026-11-14 04:22:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge commit '711b3325adf629585d60aa5a2bcac1d49b8324ae'

Browse Source
This commit is contained in:
CloudBees DEV@Cloud committed 2014-12-23 16:59:01 +00:00
2 parents 78cb49b86a 711b3325ad
commit 4c474cea6f
24 files changed
+858 -1428
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

core/src/main/java/hudson/model/Job.java
+18 -3
View File
@@ -105,7 +105,10 @@ import javax.annotation.Nonnull;
import static javax.servlet.http.HttpServletResponse.*;
import jenkins.model.ModelObjectWithChildren;
import jenkins.model.RunIdMigrator;
import jenkins.model.lazy.LazyBuildMixIn;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.NoExternalUse;
/**
* A job is an runnable entity under the monitoring of Hudson.
@@ -162,6 +165,9 @@ public abstract class Job<JobT extends Job<JobT, RunT>, RunT extends Run<JobT, R
// this should have been DescribableList but now it's too late
protected CopyOnWriteList<JobProperty<? super JobT>> properties = new CopyOnWriteList<JobProperty<? super JobT>>();
@Restricted(NoExternalUse.class)
public transient RunIdMigrator runIdMigrator;
protected Job(ItemGroup parent, String name) {
super(parent, name);
}
@@ -172,11 +178,21 @@ public abstract class Job<JobT extends Job<JobT, RunT>, RunT extends Run<JobT, R
holdOffBuildUntilSave = holdOffBuildUntilUserSave;
}
@Override public void onCreatedFromScratch() {
super.onCreatedFromScratch();
runIdMigrator = new RunIdMigrator();
runIdMigrator.created(getBuildDir());
}
@Override
public void onLoad(ItemGroup<? extends Item> parent, String name)
throws IOException {
super.onLoad(parent, name);
File buildDir = getBuildDir();
runIdMigrator = new RunIdMigrator();
runIdMigrator.migrate(buildDir, Jenkins.getInstance().getRootDir());
TextFile f = getNextBuildNumberFile();
if (f.exists()) {
// starting 1.28, we store nextBuildNumber in a separate file.
@@ -188,7 +204,7 @@ public abstract class Job<JobT extends Job<JobT, RunT>, RunT extends Run<JobT, R
}
} catch (NumberFormatException e) {
// try to infer the value of the next build number from the existing build records. See JENKINS-11563
File[] folders = this.getBuildDir().listFiles(new FileFilter() {
File[] folders = buildDir.listFiles(new FileFilter() {
public boolean accept(File file) {
return file.isDirectory() && file.getName().matches("[0-9]+");
}
@@ -638,8 +654,7 @@ public abstract class Job<JobT extends Job<JobT, RunT>, RunT extends Run<JobT, R
File oldBuildDir = getBuildDir();
super.movedTo(destination, newItem, destDir);
File newBuildDir = getBuildDir();
if (oldBuildDir.isDirectory() && !newBuildDir.isDirectory()) {
FileUtils.forceMkdir(destDir.getParentFile());
if (oldBuildDir.isDirectory()) {
FileUtils.moveDirectory(oldBuildDir, newBuildDir);
}
}
core/src/main/java/hudson/model/Run.java
+19 -62
View File
@@ -47,6 +47,7 @@ import hudson.model.Descriptor.FormException;
import hudson.model.Run.RunExecution;
import hudson.model.listeners.RunListener;
import hudson.model.listeners.SaveableListener;
import hudson.model.queue.SubTask;
import hudson.search.SearchIndexBuilder;
import hudson.security.ACL;
import hudson.security.AccessControlled;
@@ -74,7 +75,6 @@ import java.io.StringWriter;
import java.io.Writer;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
@@ -108,6 +108,7 @@ import jenkins.model.PeepholePermalink;
import jenkins.model.RunAction2;
import jenkins.model.StandardArtifactManager;
import jenkins.model.lazy.BuildReference;
import jenkins.model.lazy.LazyBuildMixIn;
import jenkins.util.VirtualFile;
import jenkins.util.io.OnMaster;
import net.sf.json.JSONObject;
@@ -146,7 +147,7 @@ public abstract class Run <JobT extends Job<JobT,RunT>,RunT extends Run<JobT,Run
* In earlier versions &lt; 1.24, this number is not unique nor continuous,
* but going forward, it will, and this really replaces the build id.
*/
public /*final*/ int number;
public transient /*final*/ int number;
/**
* Previous build. Can be null.
@@ -172,10 +173,13 @@ public abstract class Run <JobT extends Job<JobT,RunT>,RunT extends Run<JobT,Run
*/
/* does not compile on JDK 7: private*/ volatile transient RunT previousBuildInProgress;
/** ID as used for historical build records; otherwise null. */
private @CheckForNull String id;
/**
* When the build is scheduled.
*/
protected transient final long timestamp;
protected /*final*/ long timestamp;
/**
* When the build has started running.
@@ -266,23 +270,12 @@ public abstract class Run <JobT extends Job<JobT,RunT>,RunT extends Run<JobT,Run
*/
private @CheckForNull ArtifactManager artifactManager;
private static final SimpleDateFormat CANONICAL_ID_FORMATTER = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss");
public static final ThreadLocal<SimpleDateFormat> ID_FORMATTER = new IDFormatterProvider();
private static final class IDFormatterProvider extends ThreadLocal<SimpleDateFormat> {
@Override
protected SimpleDateFormat initialValue() {
synchronized (CANONICAL_ID_FORMATTER) {
return (SimpleDateFormat) CANONICAL_ID_FORMATTER.clone();
}
}
};
/**
* Creates a new {@link Run}.
* @param job Owner job
*/
protected Run(@Nonnull JobT job) throws IOException {
this(job, new GregorianCalendar());
this(job, System.currentTimeMillis());
this.number = project.assignBuildNumber();
LOGGER.log(FINER, "new {0} @{1}", new Object[] {this, hashCode()});
}
@@ -290,24 +283,29 @@ public abstract class Run <JobT extends Job<JobT,RunT>,RunT extends Run<JobT,Run
/**
* Constructor for creating a {@link Run} object in
* an arbitrary state.
* {@link #number} must be set manually.
* <p>May be used in a {@link SubTask#createExecutable} (instead of calling {@link LazyBuildMixIn#newBuild}).
* For example, {@code MatrixConfiguration.newBuild} does this
* so that the {@link #timestamp} as well as {@link #number} are shared with the parent build.
*/
protected Run(@Nonnull JobT job, @Nonnull Calendar timestamp) {
this(job,timestamp.getTimeInMillis());
}
/** @see #Run(Job, Calendar) */
protected Run(@Nonnull JobT job, long timestamp) {
this.project = job;
this.timestamp = timestamp;
this.state = State.NOT_STARTED;
getRootDir().mkdirs();
}
/**
* Loads a run from a log file.
*/
protected Run(@Nonnull JobT project, @Nonnull File buildDir) throws IOException {
this(project, parseTimestampFromBuildDir(buildDir));
this.project = project;
this.previousBuildInProgress = _this(); // loaded builds are always completed
number = Integer.parseInt(buildDir.getName());
reload();
}
@@ -390,33 +388,6 @@ public abstract class Run <JobT extends Job<JobT,RunT>,RunT extends Run<JobT,Run
}
}
static class InvalidDirectoryNameException extends IOException {
InvalidDirectoryNameException(File buildDir) {
super("Invalid directory name " + buildDir);
}
}
/*package*/ static long parseTimestampFromBuildDir(@Nonnull File buildDir)
throws IOException, InvalidDirectoryNameException {
try {
if(Util.isSymlink(buildDir)) {
// "Util.resolveSymlink(file)" resolves NTFS symlinks.
File target = Util.resolveSymlinkToFile(buildDir);
if(target != null)
buildDir = target;
}
// canonicalization to ensure we are looking at the ID in the directory name
// as opposed to build numbers which are used in symlinks
// (just in case the symlink check above did not work)
buildDir = buildDir.getCanonicalFile();
return ID_FORMATTER.get().parse(buildDir.getName()).getTime();
} catch (ParseException e) {
throw new InvalidDirectoryNameException(buildDir);
} catch (InterruptedException e) {
throw new IOException("Interrupted while resolving symlink directory "+buildDir,e);
}
}
/**
* Obtains 'this' in a more type safe signature.
*/
@@ -1000,21 +971,13 @@ public abstract class Run <JobT extends Job<JobT,RunT>,RunT extends Run<JobT,Run
/**
* Unique ID of this build.
* Usually the decimal form of {@link #number}, but may be a formatted timestamp for historical builds.
*/
@Exported
public @Nonnull String getId() {
return ID_FORMATTER.get().format(new Date(timestamp));
return id != null ? id : Integer.toString(number);
}
/**
* Get the date formatter used to convert the directory name in to a timestamp.
* This is nasty exposure of private data, but needed all the time the directory
* containing the build is used as it's timestamp.
*/
public static @Nonnull DateFormat getIDFormatter() {
return ID_FORMATTER.get();
}
@Override
public @CheckForNull Descriptor getDescriptorByName(String className) {
return Jenkins.getInstance().getDescriptorByName(className);
@@ -1027,7 +990,7 @@ public abstract class Run <JobT extends Job<JobT,RunT>,RunT extends Run<JobT,Run
*/
@Override
public @Nonnull File getRootDir() {
return new File(project.getBuildDir(),getId());
return new File(project.getBuildDir(), Integer.toString(number));
}
/**
@@ -1502,10 +1465,6 @@ public abstract class Run <JobT extends Job<JobT,RunT>,RunT extends Run<JobT,Run
RunListener.fireDeleted(this);
synchronized (this) { // avoid holding a lock while calling plugin impls of onDeleted
// if we have a symlink, delete it, too
File link = new File(project.getBuildDir(), String.valueOf(getNumber()));
link.delete();
File tmp = new File(rootDir.getParentFile(),'.'+rootDir.getName());
if (tmp.exists()) {
@@ -1832,8 +1791,7 @@ public abstract class Run <JobT extends Job<JobT,RunT>,RunT extends Run<JobT,Run
}
/**
* Creates a symlink from build number to ID.
* Also makes sure that {@code lastSuccessful} and {@code lastStable} legacy links in the project’s root directory exist.
* Makes sure that {@code lastSuccessful} and {@code lastStable} legacy links in the project’s root directory exist.
* Normally you do not need to call this explicitly, since {@link #execute} does so,
* but this may be needed if you are creating synthetic {@link Run}s as part of a container project (such as Maven builds in a module set).
* You should also ensure that {@link RunListener#fireStarted} and {@link RunListener#fireCompleted} are called.
@@ -1842,7 +1800,6 @@ public abstract class Run <JobT extends Job<JobT,RunT>,RunT extends Run<JobT,Run
* @since 1.530
*/
public final void updateSymlinks(@Nonnull TaskListener listener) throws InterruptedException {
Util.createSymlink(getParent().getBuildDir(), getId(), String.valueOf(getNumber()), listener);
createSymlink(listener, "lastSuccessful", PermalinkProjectAction.Permalink.LAST_SUCCESSFUL_BUILD);
createSymlink(listener, "lastStable", PermalinkProjectAction.Permalink.LAST_STABLE_BUILD);
}
core/src/main/java/hudson/model/RunMap.java
+30 -39
View File
@@ -24,10 +24,7 @@
package hudson.model;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
@@ -38,10 +35,14 @@ import java.util.logging.Level;
import static java.util.logging.Level.*;
import java.util.logging.Logger;
import jenkins.model.RunIdMigrator;
import jenkins.model.lazy.AbstractLazyLoadRunMap;
import static jenkins.model.lazy.AbstractLazyLoadRunMap.Direction.*;
import jenkins.model.lazy.BuildReference;
import jenkins.model.lazy.LazyBuildMixIn;
import org.apache.commons.collections.comparators.ReverseComparator;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.NoExternalUse;
/**
* {@link Map} from build number to {@link Run}.
@@ -61,6 +62,10 @@ public final class RunMap<R extends Run<?,R>> extends AbstractLazyLoadRunMap<R>
private Constructor<R> cons;
/** Normally overwritten by {@link LazyBuildMixIn#onLoad} or {@link LazyBuildMixIn#onCreatedFromScratch}, in turn created during {@link Job#onLoad}. */
@Restricted(NoExternalUse.class)
public RunIdMigrator runIdMigrator = new RunIdMigrator();
// TODO: before first complete build
// patch up next/previous build link
@@ -118,6 +123,7 @@ public final class RunMap<R extends Run<?,R>> extends AbstractLazyLoadRunMap<R>
@Override
public boolean removeValue(R run) {
run.dropLinks();
runIdMigrator.delete(dir, run.getId());
return super.removeValue(run);
}
@@ -169,11 +175,32 @@ public final class RunMap<R extends Run<?,R>> extends AbstractLazyLoadRunMap<R>
return r.getId();
}
/**
* Add a <em>new</em> build to the map.
* Do not use when loading existing builds (use {@link #put(Integer, Object)}).
*/
@Override
public R put(R r) {
// Defense against JENKINS-23152 and its ilk.
File rootDir = r.getRootDir();
if (rootDir.isDirectory()) {
throw new IllegalStateException(rootDir + " already existed; will not overwite with " + r);
}
proposeNewNumber(r.getNumber());
rootDir.mkdirs();
return super._put(r);
}
@Override public R getById(String id) {
int n;
try {
n = Integer.parseInt(id);
} catch (NumberFormatException x) {
n = runIdMigrator.findNumber(id);
}
return getByNumber(n);
}
/**
* Reuses the same reference as much as we can.
* <p>
@@ -185,31 +212,6 @@ public final class RunMap<R extends Run<?,R>> extends AbstractLazyLoadRunMap<R>
return r.createReference();
}
@Override
protected FilenameFilter createDirectoryFilter() {
final SimpleDateFormat formatter = Run.ID_FORMATTER.get();
return new FilenameFilter() {
@Override public boolean accept(File dir, String name) {
if (name.startsWith("0000")) {
// JENKINS-1461 sometimes create bogus data directories with impossible dates, such as year 0, April 31st,
// or August 0th. Date object doesn't roundtrip those, so we eventually fail to load this data.
// Don't even bother trying.
return false;
}
try {
if (formatter.format(formatter.parse(name)).equals(name)) {
return true;
}
} catch (ParseException e) {
// fall through
}
LOGGER.log(FINE, "Skipping {0} in {1}", new Object[] {name, dir});
return false;
}
};
}
@Override
protected R retrieve(File d) throws IOException {
if(new File(d,"build.xml").exists()) {
@@ -221,17 +223,6 @@ public final class RunMap<R extends Run<?,R>> extends AbstractLazyLoadRunMap<R>
LOGGER.log(FINEST, "Loaded " + b.getFullDisplayName() + " in " + Thread.currentThread().getName(), new ThisIsHowItsLoaded());
}
return b;
} catch (Run.InvalidDirectoryNameException x) {
Level lvl;
try {
Integer.parseInt(d.getName());
// JENKINS-15587: just an mangled symlink
lvl = Level.FINE;
} catch (NumberFormatException x2) {
// potentially a real build dir, maybe a bug
lvl = Level.WARNING;
}
LOGGER.log(lvl, "skipping non-build directory {0}", d);
} catch (IOException e) {
LOGGER.log(Level.WARNING, "could not load " + d, e);
} catch (InstantiationError e) {
core/src/main/java/jenkins/diagnostics/ooom/BuildPtr.java
-93
View File
@@ -1,93 +0,0 @@
package jenkins.diagnostics.ooom;
import hudson.FilePath;
import hudson.model.Job;
import hudson.model.TaskListener;
import java.io.File;
import java.io.IOException;
/**
* ID and build number of one build.
*/
public final class BuildPtr implements Comparable<BuildPtr> {
final Job job;
final File buildDir;
/**
* Timestamp build ID.
*/
public final String id;
/**
* Build number found from the disk.
*/
public final int n;
/**
* Position of this build according to the ordering induced by {@link #n}
*/
int posByN;
/**
* Position of this build according to the ordering induced by {@link #id}
*/
int posByID;
BuildPtr(Job job, File buildDir, int n) {
this.job = job;
this.n = n;
this.id = buildDir.getName();
this.buildDir = buildDir;
}
@Override
public String toString() {
return buildDir.toString()+":#"+n;
}
/**
* If this build and that build are inconsistent, in that
* their numbers and timestamps are ordering in the wrong direction.
*/
public boolean isInconsistentWith(BuildPtr that) {
return signOfCompare(this.posByN,that.posByN) * signOfCompare(this.posByID,that.posByID) < 0;
}
/**
* sign of (a-b).
*/
private static int signOfCompare(int a, int b) {
if (a>b) return 1;
if (a<b) return -1;
return 0;
}
/**
* Fix the problem by moving the out of order builds into a place that Jenkins won't look at.
*
* TODO: another way to fix this is by adjusting the ID and pretend that the build happened
* at a different timestamp.
*/
public void fix(TaskListener listener) throws IOException, InterruptedException {
File dir = new File(job.getRootDir(), "outOfOrderBuilds");
dir.mkdirs();
File dst = new File(dir, buildDir.getName());
listener.getLogger().println("Renaming "+buildDir);
listener.getLogger().println(" -> "+dst);
if (!buildDir.renameTo(dst)) {
FilePath bd = new FilePath(buildDir);
bd.copyRecursiveTo(new FilePath(dst));
bd.deleteRecursive();
}
// if there's a symlink delete it
new File(buildDir.getParentFile(),String.valueOf(n)).delete();
}
@Override
public int compareTo(BuildPtr that) {
return this.id.compareTo(that.id);
}
}
core/src/main/java/jenkins/diagnostics/ooom/OutOfOrderBuildDetector.java
-58
View File
@@ -1,58 +0,0 @@
package jenkins.diagnostics.ooom;
import hudson.Extension;
import hudson.model.AsyncPeriodicWork;
import hudson.model.Job;
import hudson.model.TaskListener;
import hudson.util.TimeUnit2;
import jenkins.model.Jenkins;
import javax.inject.Inject;
import java.io.IOException;
/**
* Discovers {@link Problem}s periodically in the background and
* pass them on to {@link OutOfOrderBuildMonitor}.
*
* @author Kohsuke Kawaguchi
*/
@Extension
public class OutOfOrderBuildDetector extends AsyncPeriodicWork {
@Inject
private OutOfOrderBuildMonitor monitor;
public OutOfOrderBuildDetector() {
super("Out of order build detection");
}
@Override
protected void execute(TaskListener listener) throws IOException, InterruptedException {
execute(listener, 10*1000);
}
/**
* Performs the check synchronously.
*
* @param delay
* delay in the number of milli-seconds to reduce the load on I/O.
*/
public void execute(TaskListener listener, int delay) throws InterruptedException {
for (Job j : Jenkins.getInstance().getAllItems(Job.class)) {
listener.getLogger().println("Scanning " + j.getFullName());
Problem p = Problem.find(j);
if (p!=null) {
monitor.addProblem(p);
listener.getLogger().println(" found problems: "+p);
}
Thread.sleep(delay);
}
}
@Override
public long getRecurrencePeriod() {
return TimeUnit2.DAYS.toMillis(1);
}
}
core/src/main/java/jenkins/diagnostics/ooom/OutOfOrderBuildMonitor.java
-70
View File
@@ -1,70 +0,0 @@
package jenkins.diagnostics.ooom;
import hudson.Extension;
import hudson.model.TaskListener;
import hudson.util.HttpResponses;
import jenkins.management.AsynchronousAdministrativeMonitor;
import org.kohsuke.stapler.HttpResponse;
import org.kohsuke.stapler.interceptor.RequirePOST;
import java.io.File;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.Set;
/**
* Reports any {@link Problem}s found and report them in the "Manage Jenkins" page.
*
* @author Kohsuke Kawaguchi
*/
@Extension
public class OutOfOrderBuildMonitor extends AsynchronousAdministrativeMonitor {
private final Set<Problem> problems = Collections.synchronizedSet(new LinkedHashSet<Problem>());
@Override
public boolean isActivated() {
return !problems.isEmpty() || getLogFile().exists();
}
void addProblem(Problem p) {
problems.add(p);
}
public Set<Problem> getProblems() {
return Collections.unmodifiableSet(new LinkedHashSet<Problem>(problems));
}
@RequirePOST
public HttpResponse doFix() {
start(false);
return HttpResponses.forwardToPreviousPage();
}
/**
* Discards the current log file so that the "stuff is completed" message will be gone.
*/
@RequirePOST
public HttpResponse doDismiss() {
getLogFile().delete();
return HttpResponses.forwardToPreviousPage();
}
@Override
public String getDisplayName() {
return "Fix Out-of-order Builds";
}
@Override
public File getLogFile() {
return super.getLogFile();
}
@Override
protected void fix(TaskListener listener) throws Exception {
Set<Problem> problems = getProblems();
for (Problem problem : problems) {
problem.fix(listener);
}
this.problems.removeAll(problems);
}
}
core/src/main/java/jenkins/diagnostics/ooom/Problem.java
-302
View File
@@ -1,302 +0,0 @@
package jenkins.diagnostics.ooom;
import hudson.AbortException;
import hudson.Util;
import hudson.model.AbstractProject;
import hudson.model.Job;
import hudson.model.Run;
import hudson.model.TaskListener;
import jenkins.model.lazy.AbstractLazyLoadRunMap;
import org.xml.sax.InputSource;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;
import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.logging.Logger;
import static java.util.logging.Level.*;
/**
* Look at build numbers in build.xml and compare them with build IDs (timestamps).
*
* When they are inconsistent (newer build timestamp-wise has an older build number),
* it'll confuse the binary search in {@link AbstractLazyLoadRunMap}, so detect them and report them.
*
* @author Kohsuke Kawaguchi
*/
public final class Problem {
public final Job job;
/**
* A smallest set of builds whose removals would correct the order
* inconsistency problem.
*/
private final Set<BuildPtr> offenders = new TreeSet<BuildPtr>();
/**
* Scans the inconsistencies and return the information about it.
*
* @return null if no problems were found.
*/
public static Problem find(Job j) {
Problem p = new Problem(j);
if (p.countInconsistencies()==0) return null;
return p;
}
private Problem(Job j) {
this.job = j;
new Inspector().inspect();
}
public Set<BuildPtr> getOffenders() {
return Collections.unmodifiableSet(offenders);
}
/**
* Number of inconsistencies, which is the number of builds whose IDs
* have to be messed around on disk to collect problems.
*/
public int countInconsistencies() {
return offenders.size();
}
/**
* Equality is based on the job.
*/
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
Problem problem = (Problem) o;
return job.equals(problem.job);
}
@Override
public int hashCode() {
return job.hashCode();
}
@Override
public String toString() {
return job.getFullName() + " " + Util.join(offenders);
}
public void fix(TaskListener listener) throws IOException, InterruptedException {
listener.getLogger().println("Fixing problems in " + job.getFullName());
for (BuildPtr o : offenders) {
o.fix(listener);
}
if (job instanceof AbstractProject) {
// let all the current references go and build a new one
AbstractProject a = (AbstractProject) job;
a._getRuns().purgeCache();
}
}
/**
* Finds the problems and builds up the data model of {@link Problem}.
*/
class Inspector {
/**
* All the builds sorted by their {@link BuildPtr#n}
*/
private List<BuildPtr> byN;
/**
* All the builds sorted by their {@link BuildPtr#id}
*/
private List<BuildPtr> byId;
private final XPathExpression xpath;
Inspector() {
try {
xpath = XPathFactory.newInstance().newXPath().compile("/*/number/text()");
} catch (XPathExpressionException e) {
throw new AssertionError(e);
}
}
/**
* Simply report inconsistencies in the ordering.
*/
protected void inspect() {
Map<Integer, BuildPtr> builds = scan();
byN = new ArrayList<BuildPtr>(builds.values());
// this is already sorted by BuildPtr.n
int i=0;
for (BuildPtr b : byN) {
b.posByN = i++;
}
byId = new ArrayList<BuildPtr>(byN);
Collections.sort(byId);
i=0;
for (BuildPtr b : byId) {
b.posByID = i++;
}
while (true) {
BuildPtr b = pick();
if (b==null)
break;
offenders.add(b);
}
}
/**
* Find the most inconsistent build, a build whose removal
* would reduce the # of inconsistencies by the most.
*
* This process takes {@link #offenders} into account.
*
* @return null if there's no more build to remove.
*/
private BuildPtr pick() {
BuildPtr worst=null;
int worstScore=0;
for (BuildPtr b : byN) {
if (offenders.contains(b))
continue;
int score = score(b);
if (score>worstScore) {
worst = b;
worstScore = score;
}
}
return worst;
}
/**
* Count the number of other builds the given build is inconsistent with,
* excluding inconsistencies with {@link #offenders} (since those inconsistencies
* are already effectively resolved by fixing offenders.)
*/
private int score(BuildPtr b) {
int i=0;
for (BuildPtr a : byN) {
if (offenders.contains(a))
continue;
if (a.isInconsistentWith(b))
i++;
}
return i;
}
/**
* Looks at the builds directory of the given job and builds up the full map of build number to its ID.
*/
protected SortedMap<Integer,BuildPtr> scan() {
LOGGER.fine("Inspecting "+job);
SortedMap<Integer,BuildPtr> builds = new TreeMap<Integer,BuildPtr>();
File[] files = job.getBuildDir().listFiles();
if (files==null) return builds;
for (File build : files) {
try {
LOGGER.finer("Inspecting " + build);
if (isInt(build.getName())) {
// if this is a number, then it must be a build number
String s = loadBuildNumberFromBuildXml(build);
if (!s.equals(build.getName())) {
LOGGER.warning(build+" contains build number "+s);
// this index is invalid.
if (build.delete()) {
// index should be a symlink, and if so we can just delete it without losing data.
LOGGER.info("Removed problematic index "+build);
} else {
// the deltion will fail if 'build' isn't just a symlink but an actual directory.
// That is good, as we don't want to delete any real data.
LOGGER.warning("Couldn't delete " + build);
}
}
continue;
}
if (isID(build.getName())) {
String bn = loadBuildNumberFromBuildXml(build);
if (bn==null) {
LOGGER.log(WARNING, "Failed to parse "+build);
continue;
}
int n;
try {
n = Integer.parseInt(bn);
} catch (NumberFormatException e) {
LOGGER.log(WARNING, "Expected number in " + build + " but found " + bn, e);
continue;
}
BuildPtr b = new BuildPtr(job,build,n);
BuildPtr o = builds.put(n, b);
if (o != null) {
LOGGER.log(WARNING, "Multiple builds have the same number: {0} vs. {1}", new Object[] {o, b});
offenders.add(b.compareTo(o) > 0 ? o : b);
}
}
} catch (XPathExpressionException e) {
LOGGER.log(WARNING, "Failed to inspect "+build, e);
} catch (AbortException e) {
LOGGER.log(WARNING, "Failed to inspect "+build+": "+e.getMessage());
} catch (IOException e) {
LOGGER.log(WARNING, "Failed to inspect "+build, e);
}
}
return builds;
}
private boolean isInt(String s) {
try {
Integer.parseInt(s);
return true;
} catch (NumberFormatException e) {
return false;
}
}
private boolean isID(String s) {
try {
Run.ID_FORMATTER.get().parse(s);
return true;
} catch (ParseException e) {
return false;
}
}
private String loadBuildNumberFromBuildXml(File dir) throws XPathExpressionException, IOException {
File buildXml = new File(dir, "build.xml");
if (!buildXml.exists())
throw new AbortException(buildXml+" doesn't exist");
String systemId = buildXml.toURI().toURL().toExternalForm();
return (String)xpath.evaluate(new InputSource(systemId), XPathConstants.STRING);
}
}
private static final Logger LOGGER = Logger.getLogger(Problem.class.getName());
}
core/src/main/java/jenkins/model/RunIdMigrator.java
+407
View File
@@ -0,0 +1,407 @@
/*
* The MIT License
*
* Copyright 2014 Jesse Glick.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
package jenkins.model;
import hudson.Extension;
import hudson.Util;
import hudson.model.Job;
import hudson.model.RootAction;
import hudson.model.Run;
import hudson.util.AtomicFileWriter;
import hudson.util.StreamTaskListener;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
import org.apache.commons.io.Charsets;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.time.FastDateFormat;
import org.apache.tools.ant.BuildException;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.NoExternalUse;
import org.kohsuke.stapler.HttpResponse;
import org.kohsuke.stapler.StaplerProxy;
import org.kohsuke.stapler.framework.io.WriterOutputStream;
import static java.util.logging.Level.*;
/**
* Converts legacy {@code builds} directories to the current format.
*
* There would be one instance associated with each {@link Job}, to retain ID -> build# mapping.
*
* The {@link Job#getBuildDir} is passed to every method call (rather than being cached) in case it is moved.
*/
@Restricted(NoExternalUse.class)
public final class RunIdMigrator {
static final Logger LOGGER = Logger.getLogger(RunIdMigrator.class.getName());
private static final DateFormat LEGACY_ID_FORMATTER = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss");
private static final String MAP_FILE = "legacyIds";
/** avoids wasting a map for new jobs */
private static final Map<String,Integer> EMPTY = new TreeMap<String,Integer>();
/**
* Did we record "unmigrate" instruction for this $JENKINS_HOME? Yes if it's in the set.
*/
private static final Set<File> offeredToUnmigrate = Collections.synchronizedSet(new HashSet<File>());
private @Nonnull Map<String,Integer> idToNumber = EMPTY;
public RunIdMigrator() {}
/**
* @return whether there was a file to load
*/
private boolean load(File dir) {
File f = new File(dir, MAP_FILE);
if (!f.isFile()) {
return false;
}
if (f.length() == 0) {
return true;
}
idToNumber = new TreeMap<String,Integer>();
try {
for (String line : FileUtils.readLines(f)) {
int i = line.indexOf(' ');
idToNumber.put(line.substring(0, i), Integer.parseInt(line.substring(i + 1)));
}
} catch (Exception x) { // IOException, IndexOutOfBoundsException, NumberFormatException
LOGGER.log(WARNING, "could not read from " + f, x);
}
return true;
}
private void save(File dir) {
File f = new File(dir, MAP_FILE);
try {
AtomicFileWriter w = new AtomicFileWriter(f);
try {
for (Map.Entry<String,Integer> entry : idToNumber.entrySet()) {
w.write(entry.getKey() + ' ' + entry.getValue() + '\n');
}
w.commit();
} finally {
w.abort();
}
} catch (IOException x) {
LOGGER.log(WARNING, "could not save changes to " + f, x);
}
}
/**
* Called when a job is first created.
* Just saves an empty marker indicating that this job needs no migration.
* @param dir as in {@link Job#getBuildDir}
*/
public void created(File dir) {
save(dir);
}
/**
* Perform one-time migration if this has not been done already.
* Where previously there would be a {@code 2014-01-02_03-04-05/build.xml} specifying {@code <number>99</number>} plus a symlink {@code 99 → 2014-01-02_03-04-05},
* after migration there will be just {@code 99/build.xml} specifying {@code <id>2014-01-02_03-04-05</id>} and {@code <timestamp>…</timestamp>} according to local time zone at time of migration.
* Newly created builds are untouched.
* Does not throw {@link IOException} since we make a best effort to migrate but do not consider it fatal to job loading if we cannot.
* @param dir as in {@link Job#getBuildDir}
* @param jenkinsHome root directory of Jenkins (for logging only)
* @return true if migration was performed
*/
public synchronized boolean migrate(File dir, @CheckForNull File jenkinsHome) {
if (load(dir)) {
LOGGER.log(FINER, "migration already performed for {0}", dir);
return false;
}
if (!dir.isDirectory()) {
LOGGER.log(/* normal during Job.movedTo */FINE, "{0} was unexpectedly missing", dir);
return false;
}
LOGGER.log(INFO, "Migrating build records in {0}", dir);
doMigrate(dir);
save(dir);
if (jenkinsHome != null && offeredToUnmigrate.add(jenkinsHome))
LOGGER.log(WARNING, "Build record migration is one-way. If you need to downgrade Jenkins, run: {0}", getUnmigrationCommandLine(jenkinsHome));
return true;
}
private static String getUnmigrationCommandLine(File jenkinsHome) {
StringBuilder cp = new StringBuilder();
for (Class<?> c : new Class<?>[] {RunIdMigrator.class, /* TODO how to calculate transitive dependencies automatically? */Charsets.class, WriterOutputStream.class, BuildException.class, FastDateFormat.class}) {
URL location = c.getProtectionDomain().getCodeSource().getLocation();
String locationS = location.toString();
if (location.getProtocol().equals("file")) {
try {
locationS = new File(location.toURI()).getAbsolutePath();
} catch (URISyntaxException x) {
// never mind
}
}
if (cp.length() > 0) {
cp.append(File.pathSeparator);
}
cp.append(locationS);
}
return String.format("java -classpath \"%s\" %s \"%s\"", cp, RunIdMigrator.class.getName(), jenkinsHome);
}
private static final Pattern NUMBER_ELT = Pattern.compile("(?m)^ <number>(\\d+)</number>(\r?\n)");
private void doMigrate(File dir) {
idToNumber = new TreeMap<String,Integer>();
File[] kids = dir.listFiles();
// Need to process symlinks first so we can rename to them.
List<File> kidsList = new ArrayList<File>(Arrays.asList(kids));
Iterator<File> it = kidsList.iterator();
while (it.hasNext()) {
File kid = it.next();
String name = kid.getName();
try {
String link = Util.resolveSymlink(kid);
if (link == null) {
continue;
}
try {
Integer.parseInt(name);
if (kid.delete()) {
LOGGER.log(FINE, "deleted build number symlink {0} → {1}", new Object[] {name, link});
} else {
LOGGER.log(WARNING, "could not delete build number symlink {0} → {1}", new Object[] {name, link});
}
} catch (NumberFormatException x) {
LOGGER.log(FINE, "skipping other symlink {0} → {1}", new Object[] {name, link});
}
it.remove();
} catch (Exception x) {
LOGGER.log(WARNING, "failed to process " + kid, x);
}
}
it = kidsList.iterator();
while (it.hasNext()) {
File kid = it.next();
try {
String name = kid.getName();
try {
Integer.parseInt(name);
LOGGER.log(FINE, "skipping new build dir {0}", name);
continue;
} catch (NumberFormatException x) {
// OK, next…
}
if (!kid.isDirectory()) {
LOGGER.log(FINE, "skipping non-directory {0}", name);
continue;
}
long timestamp;
try {
synchronized (LEGACY_ID_FORMATTER) {
timestamp = LEGACY_ID_FORMATTER.parse(name).getTime();
}
} catch (ParseException x) {
LOGGER.log(WARNING, "found unexpected dir {0}", name);
continue;
}
File buildXml = new File(kid, "build.xml");
if (!buildXml.isFile()) {
LOGGER.log(WARNING, "found no build.xml in {0}", name);
continue;
}
String xml = FileUtils.readFileToString(buildXml, Charsets.UTF_8);
Matcher m = NUMBER_ELT.matcher(xml);
if (!m.find()) {
LOGGER.log(WARNING, "could not find <number> in {0}/build.xml", name);
continue;
}
int number = Integer.parseInt(m.group(1));
String nl = m.group(2);
xml = m.replaceFirst(" <id>" + name + "</id>" + nl + " <timestamp>" + timestamp + "</timestamp>" + nl);
File newKid = new File(dir, Integer.toString(number));
if (!kid.renameTo(newKid)) {
LOGGER.log(WARNING, "failed to rename {0} to {1}", new Object[] {name, number});
continue;
}
FileUtils.writeStringToFile(new File(newKid, "build.xml"), xml, Charsets.UTF_8);
LOGGER.log(FINE, "fully processed {0} → {1}", new Object[] {name, number});
idToNumber.put(name, number);
} catch (Exception x) {
LOGGER.log(WARNING, "failed to process " + kid, x);
}
}
}
/**
* Look up a historical run by ID.
* @param id a nonnumeric ID which may be a valid {@link Run#getId}
* @return the corresponding {@link Run#number}, or 0 if unknown
*/
public synchronized int findNumber(@Nonnull String id) {
Integer number = idToNumber.get(id);
return number != null ? number : 0;
}
/**
* Delete the record of a build.
* @param dir as in {@link Job#getBuildDir}
* @param id a {@link Run#getId}
*/
public synchronized void delete(File dir, String id) {
if (idToNumber.remove(id) != null) {
save(dir);
}
}
/**
* Reverses the migration, in case you want to revert to the older format.
* @param args one parameter, {@code $JENKINS_HOME}
*/
public static void main(String... args) throws Exception {
if (args.length != 1) {
throw new Exception("pass one parameter, $JENKINS_HOME");
}
File root = new File(args[0]);
File jobs = new File(root, "jobs");
if (!jobs.isDirectory()) {
throw new FileNotFoundException("no such $JENKINS_HOME " + root);
}
unmigrateJobsDir(jobs);
}
private static void unmigrateJobsDir(File jobs) throws Exception {
for (File job : jobs.listFiles()) {
File[] kids = job.listFiles();
if (kids == null) {
continue;
}
for (File kid : kids) {
if (!kid.isDirectory()) {
continue;
}
if (kid.getName().equals("builds")) {
unmigrateBuildsDir(kid);
} else {
// Might be jobs, modules, promotions, etc.; we assume an ItemGroup.getRootDirFor implementation returns grandchildren.
unmigrateJobsDir(kid);
}
}
}
}
private static final Pattern ID_ELT = Pattern.compile("(?m)^ <id>([0-9_-]+)</id>(\r?\n)");
private static final Pattern TIMESTAMP_ELT = Pattern.compile("(?m)^ <timestamp>(\\d+)</timestamp>(\r?\n)");
/** Inverse of {@link #doMigrate}. */
private static void unmigrateBuildsDir(File builds) throws Exception {
File mapFile = new File(builds, MAP_FILE);
if (!mapFile.isFile()) {
System.err.println(builds + " does not look to have been migrated yet; skipping");
return;
}
for (File build : builds.listFiles()) {
int number;
try {
number = Integer.parseInt(build.getName());
} catch (NumberFormatException x) {
continue;
}
File buildXml = new File(build, "build.xml");
if (!buildXml.isFile()) {
System.err.println(buildXml + " did not exist");
continue;
}
String xml = FileUtils.readFileToString(buildXml, Charsets.UTF_8);
Matcher m = TIMESTAMP_ELT.matcher(xml);
if (!m.find()) {
System.err.println(buildXml + " did not contain <timestamp> as expected");
continue;
}
long timestamp = Long.parseLong(m.group(1));
String nl = m.group(2);
xml = m.replaceFirst(" <number>" + number + "</number>" + nl);
m = ID_ELT.matcher(xml);
String id;
if (m.find()) {
id = m.group(1);
xml = m.replaceFirst("");
} else {
// Post-migration build. We give it a new ID based on its timestamp.
id = LEGACY_ID_FORMATTER.format(new Date(timestamp));
}
FileUtils.write(buildXml, xml, Charsets.UTF_8);
if (!build.renameTo(new File(builds, id))) {
System.err.println(build + " could not be renamed");
}
Util.createSymlink(builds, id, Integer.toString(number), StreamTaskListener.fromStderr());
}
Util.deleteFile(mapFile);
System.err.println(builds + " has been restored to its original format");
}
/**
* Expose unmigration instruction to the user.
*/
@Extension
public static class UnmigrationInstruction implements RootAction, StaplerProxy {
@Override
public String getIconFileName() {
return null;
}
@Override
public String getDisplayName() {
return null;
}
@Override
public String getUrlName() {
return "JENKINS-24380";
}
@Override
public Object getTarget() {
Jenkins.getInstance().checkPermission(Jenkins.ADMINISTER);
return this;
}
public String getCommand() {
return RunIdMigrator.getUnmigrationCommandLine(Jenkins.getInstance().getRootDir());
}
}
}
core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java
+72 -304
View File
@@ -27,7 +27,6 @@ import hudson.model.Job;
import hudson.model.Run;
import hudson.model.RunMap;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.util.AbstractMap;
import java.util.ArrayList;
@@ -44,8 +43,6 @@ import java.util.logging.Logger;
import javax.annotation.CheckForNull;
import static jenkins.model.lazy.AbstractLazyLoadRunMap.Direction.*;
import static jenkins.model.lazy.Boundary.*;
import org.apache.commons.collections.keyvalue.DefaultMapEntry;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.NoExternalUse;
@@ -60,26 +57,16 @@ import org.kohsuke.accmod.restrictions.NoExternalUse;
* requested, this class {@link #retrieve(File) retrieves them} on demand, one by one.
*
* <p>
* The lookup is primarily done by using the build number as the key (hence the key type is {@link Integer}),
* but this class also provides look up based on {@linkplain #getIdOf(Object) the build ID}.
* The lookup is done by using the build number as the key (hence the key type is {@link Integer}).
*
* <p>
* This class makes the following assumption about the on-disk layout of the data:
*
* <ul>
* <li>Every build is stored in a directory, named after its ID.
* <li>ID and build number are in the consistent order. That is,
* if there are two builds #M and #N, {@code M>N <=> M.id > N.id}.
* <li>Every build is stored in a directory, named after its number.
* </ul>
*
* <p>
* On certain platforms, there are symbolic links named after build numbers that link to the build ID.
* If these are available, they are used as a hint to speed up the lookup. Otherwise
* we rely on the assumption above and perform a binary search to locate the build.
* (notice that we'll have to do linear search if we don't have the consistent ordering assumption,
* which robs the whole point of doing lazy loading.)
*
* <p>
* Some of the {@link SortedMap} operations are weakly implemented. For example,
* {@link #size()} may be inaccurate because we only count the number of directories that look like
* build records, without checking if they are loadable. But these weaknesses aren't distinguishable
@@ -93,7 +80,7 @@ import org.kohsuke.accmod.restrictions.NoExternalUse;
* <p>
* Object lock of {@code this} is used to make sure mutation occurs sequentially.
* That is, ensure that only one thread is actually calling {@link #retrieve(File)} and
* updating {@link jenkins.model.lazy.AbstractLazyLoadRunMap.Index#byNumber} and {@link jenkins.model.lazy.AbstractLazyLoadRunMap.Index#byId}.
* updating {@link jenkins.model.lazy.AbstractLazyLoadRunMap.Index#byNumber}.
*
* @author Kohsuke Kawaguchi
* @since 1.485
@@ -112,8 +99,10 @@ public abstract class AbstractLazyLoadRunMap<R> extends AbstractMap<Integer,R> i
private LazyLoadRunMapEntrySet<R> entrySet = new LazyLoadRunMapEntrySet<R>(this);
/**
* Pair of two maps into a single class, so that the changes can be made visible atomically,
* and updates can happen concurrently to read.
* Historical holder for map.
*
* TODO all this mess including {@link #numberOnDisk} could probably be simplified to a single {@code TreeMap<Integer,BuildReference<R>>}
* where a null value means not yet loaded and a broken entry just uses {@code NoHolder}.
*
* The idiom is that you put yourself in a synchronized block, {@linkplain #copy() make a copy of this},
* update the copy, then set it to {@link #index}.
@@ -121,63 +110,24 @@ public abstract class AbstractLazyLoadRunMap<R> extends AbstractMap<Integer,R> i
private class Index {
/**
* Stores the mapping from build number to build, for builds that are already loaded.
*/
private final TreeMap<Integer,BuildReference<R>> byNumber;
/**
* Stores the build ID to build number for builds that we already know.
*
* If we have known load failure of the given ID, we record that in the map
* by using the null value (not to be confused with a non-null {@link BuildReference}
* with null referent, which just means the record was GCed.)
*/
private final TreeMap<String,BuildReference<R>> byId;
private final TreeMap<Integer,BuildReference<R>> byNumber;
private Index() {
byId = new TreeMap<String,BuildReference<R>>();
byNumber = new TreeMap<Integer,BuildReference<R>>(COMPARATOR);
}
private Index(Index rhs) {
byId = new TreeMap<String, BuildReference<R>>(rhs.byId);
byNumber = new TreeMap<Integer,BuildReference<R>>(rhs.byNumber);
}
/**
* Returns the build record #M (<=n)
*/
private Map.Entry<Integer,BuildReference<R>> ceilingEntry(int n) {
// switch to this once we depend on JDK6
// return byNumber.ceilingEntry(n);
Set<Entry<Integer, BuildReference<R>>> s = byNumber.tailMap(n).entrySet();
if (s.isEmpty()) return null;
else return s.iterator().next();
}
/**
* Returns the build record #M (>=n)
*/
// >= and not <= because byNumber is in the descending order
private Map.Entry<Integer,BuildReference<R>> floorEntry(int n) {
// switch to this once we depend on JDK6
// return byNumber.floorEntry(n);
SortedMap<Integer, BuildReference<R>> sub = byNumber.headMap(n);
if (sub.isEmpty()) return null;
Integer k = sub.lastKey();
return new DefaultMapEntry(k,sub.get(k));
}
}
/**
* Build IDs found as directories, in the ascending order.
*/
// copy on write
private volatile SortedList<String> idOnDisk = new SortedList<String>(Collections.<String>emptyList());
/**
* Build number shortcuts found on disk, in the ascending order.
* Build numbers found on disk, in the ascending order.
*/
// copy on write
private volatile SortedIntList numberOnDisk = new SortedIntList(0);
@@ -188,8 +138,9 @@ public abstract class AbstractLazyLoadRunMap<R> extends AbstractMap<Integer,R> i
* because the compatibility requires that we make it settable
* in the first call after the constructor.
*/
private File dir;
protected File dir;
@Restricted(NoExternalUse.class) // subclassing other than by RunMap does not guarantee compatibility
protected AbstractLazyLoadRunMap(File dir) {
initBaseDir(dir);
}
@@ -199,7 +150,7 @@ public abstract class AbstractLazyLoadRunMap<R> extends AbstractMap<Integer,R> i
assert this.dir==null;
this.dir = dir;
if (dir!=null)
loadIdOnDisk();
loadNumberOnDisk();
}
/**
@@ -230,31 +181,23 @@ public abstract class AbstractLazyLoadRunMap<R> extends AbstractMap<Integer,R> i
public synchronized void purgeCache() {
index = new Index();
fullyLoaded = false;
loadIdOnDisk();
loadNumberOnDisk();
}
private void loadIdOnDisk() {
private void loadNumberOnDisk() {
String[] kids = dir.list();
if (kids == null) {
// the job may have just been created
kids = EMPTY_STRING_ARRAY;
}
List<String> buildDirs = new ArrayList<String>();
FilenameFilter buildDirFilter = createDirectoryFilter();
SortedIntList list = new SortedIntList(kids.length / 2);
for (String s : kids) {
if (buildDirFilter.accept(dir, s)) {
buildDirs.add(s);
} else {
try {
list.add(Integer.parseInt(s));
} catch (NumberFormatException e) {
// this isn't a shortcut
}
try {
list.add(Integer.parseInt(s));
} catch (NumberFormatException e) {
// this isn't a build dir
}
}
Collections.sort(buildDirs);
idOnDisk = new SortedList<String>(buildDirs);
list.sort();
numberOnDisk = list;
}
@@ -263,13 +206,9 @@ public abstract class AbstractLazyLoadRunMap<R> extends AbstractMap<Integer,R> i
return COMPARATOR;
}
/**
* If we have non-zero R in memory, we can return false right away.
* If we have zero R in memory, try loading one and see if we can find something.
*/
@Override
public boolean isEmpty() {
return index.byId.isEmpty() && search(Integer.MAX_VALUE, DESC)==null;
return search(Integer.MAX_VALUE, DESC)==null;
}
@Override
@@ -349,7 +288,7 @@ public abstract class AbstractLazyLoadRunMap<R> extends AbstractMap<Integer,R> i
}
public R get(int n) {
return search(n, Direction.EXACT);
return getByNumber(n);
}
/**
@@ -367,184 +306,60 @@ public abstract class AbstractLazyLoadRunMap<R> extends AbstractMap<Integer,R> i
* If DESC, finds the closest #M that satisfies M&lt;=N.
*/
public @CheckForNull R search(final int n, final Direction d) {
Entry<Integer, BuildReference<R>> c = index.ceilingEntry(n);
if (c!=null && c.getKey()== n) {
R r = c.getValue().get();
if (r!=null)
return r; // found the exact #n
}
// at this point we know that we don't have #n loaded yet
{// check numberOnDisk as a cache to see if we can find it there
int npos = numberOnDisk.find(n);
if (npos>=0) {// found exact match
R r = load(numberOnDisk.get(npos), null);
if (r!=null)
return r;
}
switch (d) {
case ASC:
case DESC:
// didn't find the exact match, but what's the nearest ascending value in the cache?
int neighbor = (d==ASC?HIGHER:LOWER).apply(npos);
if (numberOnDisk.isInRange(neighbor)) {
R r = getByNumber(numberOnDisk.get(neighbor));
if (r!=null) {
// make sure that the cache is accurate by looking at the previous ID
// and it actually satisfies the constraint
int prev = (d==ASC?LOWER:HIGHER).apply(idOnDisk.find(getIdOf(r)));
if (idOnDisk.isInRange(prev)) {
R pr = getById(idOnDisk.get(prev));
// sign*sign is making sure that #pr and #r sandwiches #n.
if (pr!=null && signOfCompare(getNumberOf(pr),n)*signOfCompare(n,getNumberOf(r))>0)
return r;
else {
// cache is lying. there's something fishy.
// ignore the cache and do the slow search
}
} else {
// r is the build with youngest ID
return r;
}
} else {
// cache says we should have a build but we didn't.
// ignore the cache and do the slow search
}
}
break;
case EXACT:
// fall through
}
// didn't find it in the cache, but don't give up yet
// maybe the cache just doesn't exist.
// so fall back to the slow search
}
// capture the snapshot and work off with it since it can be overwritten by other threads
SortedList<String> idOnDisk = this.idOnDisk;
boolean clonedIdOnDisk = false; // if we modify idOnDisk we need to write it back. this flag is set to true when we overwrit idOnDisk local var
// slow path: we have to find the build from idOnDisk by guessing ID of the build.
// first, narrow down the candidate IDs to try by using two known number-to-ID mapping
if (idOnDisk.isEmpty()) return null;
Entry<Integer, BuildReference<R>> f = index.floorEntry(n);
// if bound is null, use a sentinel value
String cid = c==null ? "\u0000" : c.getValue().id;
String fid = f==null ? "\uFFFF" : f.getValue().id;
// at this point, #n must be in (cid,fid)
// We know that the build we are looking for exists in [lo,hi) --- it's "hi)" and not "hi]" because we do +1.
// we will narrow this down via binary search
final int initialSize = idOnDisk.size();
int lo = idOnDisk.higher(cid);
int hi = idOnDisk.lower(fid)+1;
final int initialLo = lo, initialHi = hi;
if (!(0<=lo && lo<=hi && hi<=idOnDisk.size())) {
// assertion error, but we are so far unable to get to the bottom of this bug.
// but don't let this kill the loading the hard way
String msg = String.format(
"JENKINS-15652 Assertion error #1: failing to load %s #%d %s: lo=%d,hi=%d,size=%d,size2=%d",
dir, n, d, lo, hi, idOnDisk.size(), initialSize);
LOGGER.log(Level.WARNING, msg);
return null;
}
while (lo<hi) {
final int pivot = (lo+hi)/2;
if (!(0<=lo && lo<=pivot && pivot<hi && hi<=idOnDisk.size())) {
// assertion error, but we are so far unable to get to the bottom of this bug.
// but don't let this kill the loading the hard way
String msg = String.format(
"JENKINS-15652 Assertion error #2: failing to load %s #%d %s: lo=%d,hi=%d,pivot=%d,size=%d (initial:lo=%d,hi=%d,size=%d)",
dir, n, d, lo, hi, pivot, idOnDisk.size(), initialLo, initialHi, initialSize);
LOGGER.log(Level.WARNING, msg);
return null;
}
R r = load(idOnDisk.get(pivot), null);
if (r==null) {
// this ID isn't valid. get rid of that and retry pivot
hi--;
if (!clonedIdOnDisk) {// if we are making an edit, we need to own a copy
idOnDisk = new SortedList<String>(idOnDisk);
clonedIdOnDisk = true;
}
idOnDisk.remove(pivot);
continue;
}
int found = getNumberOf(r);
if (found==n)
return r; // exact match
if (found<n) lo = pivot+1; // the pivot was too small. look in the upper half
else hi = pivot; // the pivot was too big. look in the lower half
}
if (clonedIdOnDisk)
this.idOnDisk = idOnDisk; // feedback the modified result atomically
assert lo==hi;
// didn't find the exact match
// both lo and hi point to the insertion point on idOnDisk
switch (d) {
case ASC:
if (hi==idOnDisk.size()) return null;
return getById(idOnDisk.get(hi));
case DESC:
if (lo<=0) return null;
if (lo-1>=idOnDisk.size()) {
// assertion error, but we are so far unable to get to the bottom of this bug.
// but don't let this kill the loading the hard way
LOGGER.log(Level.WARNING, String.format(
"JENKINS-15652 Assertion error #3: failing to load %s #%d %s: lo=%d,hi=%d,size=%d (initial:lo=%d,hi=%d,size=%d)",
dir, n,d,lo,hi,idOnDisk.size(), initialLo,initialHi,initialSize));
return null;
}
return getById(idOnDisk.get(lo-1));
case EXACT:
if (hi<=0) return null;
R r = getById(idOnDisk.get(hi-1));
if (r==null) return null;
int found = getNumberOf(r);
if (found==n)
return r; // exact match
return getByNumber(n);
case ASC:
for (int m : numberOnDisk) {
if (m < n) {
// TODO could be made more efficient with numberOnDisk.find
continue;
}
R r = getByNumber(m);
if (r != null) {
return r;
}
}
return null;
case DESC:
// TODO again could be made more efficient
List<Integer> reversed = new ArrayList<Integer>(numberOnDisk);
Collections.reverse(reversed);
for (int m : reversed) {
if (m > n) {
continue;
}
R r = getByNumber(m);
if (r != null) {
return r;
}
}
return null;
default:
throw new AssertionError();
}
}
/**
* sign of (a-b).
*/
private static int signOfCompare(int a, int b) {
if (a>b) return 1;
if (a<b) return -1;
return 0;
public R getById(String id) {
return getByNumber(Integer.parseInt(id));
}
public R getById(String id) {
public R getByNumber(int n) {
Index snapshot = index;
if (snapshot.byId.containsKey(id)) {
BuildReference<R> ref = snapshot.byId.get(id);
if (snapshot.byNumber.containsKey(n)) {
BuildReference<R> ref = snapshot.byNumber.get(n);
if (ref==null) return null; // known failure
R v = unwrap(ref);
if (v!=null) return v; // already in memory
// otherwise fall through to load
}
return load(id,null);
return load(n, null);
}
public R getByNumber(int n) {
return search(n, Direction.EXACT);
protected final synchronized void proposeNewNumber(int number) throws IllegalStateException {
if (numberOnDisk.isInRange(numberOnDisk.ceil(number))) {
throw new IllegalStateException("cannot create a build with number " + number + " since that (or higher) is already in use among " + numberOnDisk);
}
}
public R put(R value) {
@@ -557,27 +372,13 @@ public abstract class AbstractLazyLoadRunMap<R> extends AbstractMap<Integer,R> i
@Override
public synchronized R put(Integer key, R r) {
String id = getIdOf(r);
int n = getNumberOf(r);
Index copy = copy();
BuildReference<R> ref = createReference(r);
BuildReference<R> old = copy.byId.put(id,ref);
copy.byNumber.put(n,ref);
BuildReference<R> old = copy.byNumber.put(n,ref);
index = copy;
/*
search relies on the fact that every object added via
put() method be available in the xyzOnDisk index, so I'm adding them here
however, this is awfully inefficient. I wonder if there's any better way to do this?
*/
if (!idOnDisk.contains(id)) {
ArrayList<String> a = new ArrayList<String>(idOnDisk);
a.add(id);
Collections.sort(a);
idOnDisk = new SortedList<String>(a);
}
if (!numberOnDisk.contains(n)) {
SortedIntList a = new SortedIntList(numberOnDisk);
a.add(n);
@@ -598,9 +399,7 @@ public abstract class AbstractLazyLoadRunMap<R> extends AbstractMap<Integer,R> i
public synchronized void putAll(Map<? extends Integer,? extends R> rhs) {
Index copy = copy();
for (R r : rhs.values()) {
String id = getIdOf(r);
BuildReference<R> ref = createReference(r);
copy.byId.put(id,ref);
copy.byNumber.put(getNumberOf(r),ref);
}
index = copy;
@@ -620,9 +419,9 @@ public abstract class AbstractLazyLoadRunMap<R> extends AbstractMap<Integer,R> i
synchronized (this) {
if (!fullyLoaded) {
Index copy = copy();
for (String id : idOnDisk) {
if (!copy.byId.containsKey(id))
load(id,copy);
for (Integer number : numberOnDisk) {
if (!copy.byNumber.containsKey(number))
load(number, copy);
}
index = copy;
fullyLoaded = true;
@@ -640,42 +439,18 @@ public abstract class AbstractLazyLoadRunMap<R> extends AbstractMap<Integer,R> i
}
/**
* Tries to load the record #N by using the shortcut.
* Tries to load the record #N.
*
* @return null if the data failed to load.
*/
protected R load(int n, Index editInPlace) {
R r = null;
File shortcut = new File(dir,String.valueOf(n));
if (shortcut.isDirectory()) {
synchronized (this) {
r = load(shortcut,editInPlace);
// make sure what we actually loaded is #n,
// because the shortcuts can lie.
if (r!=null && getNumberOf(r)!=n)
r = null;
if (r==null) {
// if failed to locate, record that fact
SortedIntList update = new SortedIntList(numberOnDisk);
update.removeValue(n);
numberOnDisk = update;
}
}
}
return r;
}
protected R load(String id, Index editInPlace) {
assert dir != null;
R v = load(new File(dir, id), editInPlace);
R v = load(new File(dir, String.valueOf(n)), editInPlace);
if (v==null && editInPlace!=null) {
// remember the failure.
// if editInPlace==null, we can create a new copy for this, but not sure if it's worth doing,
// given that we also update idOnDisk anyway.
editInPlace.byId.put(id,null);
// TODO should we also update numberOnDisk?
editInPlace.byNumber.put(n, null);
}
return v;
}
@@ -692,9 +467,7 @@ public abstract class AbstractLazyLoadRunMap<R> extends AbstractMap<Integer,R> i
Index copy = editInPlace!=null ? editInPlace : new Index(index);
String id = getIdOf(r);
BuildReference<R> ref = createReference(r);
copy.byId.put(id,ref);
copy.byNumber.put(getNumberOf(r),ref);
if (editInPlace==null) index = copy;
@@ -710,10 +483,13 @@ public abstract class AbstractLazyLoadRunMap<R> extends AbstractMap<Integer,R> i
* Subtype to provide {@link Run#getNumber()} so that this class doesn't have to depend on it.
*/
protected abstract int getNumberOf(R r);
/**
* Subtype to provide {@link Run#getId()} so that this class doesn't have to depend on it.
*/
protected abstract String getIdOf(R r);
protected String getIdOf(R r) {
return String.valueOf(getNumberOf(r));
}
/**
* Allow subtype to capture a reference.
@@ -737,11 +513,10 @@ public abstract class AbstractLazyLoadRunMap<R> extends AbstractMap<Integer,R> i
public synchronized boolean removeValue(R run) {
Index copy = copy();
int n = getNumberOf(run);
copy.byNumber.remove(n);
BuildReference<R> old = copy.byNumber.remove(n);
SortedIntList a = new SortedIntList(numberOnDisk);
a.removeValue(n);
numberOnDisk = a;
BuildReference<R> old = copy.byId.remove(getIdOf(run));
this.index = copy;
entrySet.clearCache();
@@ -755,9 +530,7 @@ public abstract class AbstractLazyLoadRunMap<R> extends AbstractMap<Integer,R> i
public synchronized void reset(TreeMap<Integer,R> builds) {
Index index = new Index();
for (R r : builds.values()) {
String id = getIdOf(r);
BuildReference<R> ref = createReference(r);
index.byId.put(id,ref);
index.byNumber.put(getNumberOf(r),ref);
}
@@ -774,14 +547,9 @@ public abstract class AbstractLazyLoadRunMap<R> extends AbstractMap<Integer,R> i
return o==this;
}
/**
* Lists the actual data directory
*/
protected abstract FilenameFilter createDirectoryFilter();
private static final Comparator<Comparable> COMPARATOR = new Comparator<Comparable>() {
public int compare(Comparable o1, Comparable o2) {
return -o1.compareTo(o2);
private static final Comparator<Integer> COMPARATOR = new Comparator<Integer>() {
@Override public int compare(Integer o1, Integer o2) {
return o2 - o1;
}
};
core/src/main/java/jenkins/model/lazy/LazyBuildMixIn.java
+10 -19
View File
@@ -47,6 +47,7 @@ import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.DoNotUse;
import static java.util.logging.Level.FINER;
import jenkins.model.RunIdMigrator;
/**
* Makes it easier to use a lazy {@link RunMap} from a {@link Job} implementation.
@@ -63,9 +64,6 @@ public abstract class LazyBuildMixIn<JobT extends Job<JobT,RunT> & Queue.Task &
@SuppressWarnings("deprecation") // [JENKINS-15156] builds accessed before onLoad or onCreatedFromScratch called
private @Nonnull RunMap<RunT> builds = new RunMap<RunT>();
// keep track of the previous time we started a build
private long lastBuildStartTime;
/**
* Initializes this mixin.
* Call this from a constructor and {@link AbstractItem#onLoad} to make sure it is always initialized.
@@ -92,14 +90,14 @@ public abstract class LazyBuildMixIn<JobT extends Job<JobT,RunT> & Queue.Task &
}
/**
* Something to be called from {@link AbstractItem#onCreatedFromScratch}.
* Something to be called from {@link Job#onCreatedFromScratch}.
*/
public final void onCreatedFromScratch() {
builds = createBuildRunMap();
}
/**
* Something to be called from {@link AbstractItem#onLoad}.
* Something to be called from {@link Job#onLoad}.
*/
@SuppressWarnings("unchecked")
public void onLoad(ItemGroup<? extends Item> parent, String name) throws IOException {
@@ -123,7 +121,8 @@ public abstract class LazyBuildMixIn<JobT extends Job<JobT,RunT> & Queue.Task &
// if we are reloading, keep all those that are still building intact
for (RunT r : currentBuilds.getLoadedBuilds().values()) {
if (r.isBuilding()) {
_builds.put(r);
// Do not use RunMap.put(Run):
_builds.put(r.getNumber(), r);
}
}
}
@@ -131,11 +130,15 @@ public abstract class LazyBuildMixIn<JobT extends Job<JobT,RunT> & Queue.Task &
}
private RunMap<RunT> createBuildRunMap() {
return new RunMap<RunT>(asJob().getBuildDir(), new RunMap.Constructor<RunT>() {
RunMap<RunT> r = new RunMap<RunT>(asJob().getBuildDir(), new RunMap.Constructor<RunT>() {
@Override public RunT create(File dir) throws IOException {
return loadBuild(dir);
}
});
RunIdMigrator runIdMigrator = asJob().runIdMigrator;
assert runIdMigrator != null;
r.runIdMigrator = runIdMigrator;
return r;
}
/**
@@ -169,19 +172,7 @@ public abstract class LazyBuildMixIn<JobT extends Job<JobT,RunT> & Queue.Task &
* Calls the ({@link Job}) constructor of {@link #getBuildClass}.
* Suitable for {@link SubTask#createExecutable}.
*/
@SuppressWarnings("SleepWhileHoldingLock")
@edu.umd.cs.findbugs.annotations.SuppressWarnings("SWL_SLEEP_WITH_LOCK_HELD")
public final synchronized RunT newBuild() throws IOException {
// make sure we don't start two builds in the same second
// so the build directories will be different too
long timeSinceLast = System.currentTimeMillis() - lastBuildStartTime;
if (timeSinceLast < 1000) {
try {
Thread.sleep(1000 - timeSinceLast);
} catch (InterruptedException e) {
}
}
lastBuildStartTime = System.currentTimeMillis();
try {
RunT lastBuild = getBuildClass().getConstructor(asJob().getClass()).newInstance(asJob());
builds.put(lastBuild);