From ce829c15d0e322f7f785eab27f6562943984c40b Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Thu, 23 Aug 2012 16:04:32 +0900 Subject: [PATCH 01/69] initial source set import --- note.txt | 6 + pom.xml | 7 + src/main/java/AbstractSortedMap.java | 113 +++++++++++++++ src/main/java/Attempt2.java | 199 +++++++++++++++++++++++++++ src/main/java/RunMap.java | 21 +++ src/main/java/SortedStringList.java | 27 ++++ 6 files changed, 373 insertions(+) create mode 100644 note.txt create mode 100644 pom.xml create mode 100644 src/main/java/AbstractSortedMap.java create mode 100644 src/main/java/Attempt2.java create mode 100644 src/main/java/RunMap.java create mode 100644 src/main/java/SortedStringList.java diff --git a/note.txt b/note.txt new file mode 100644 index 0000000000..686e45ec18 --- /dev/null +++ b/note.txt @@ -0,0 +1,6 @@ + +- Perhaps better to create two independent implementations, + one that relies on symlinks (thus mapping from N->R), the other + that uses ID->R mapping. + + It's not clear how we can mix two \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000000..ea67981a07 --- /dev/null +++ b/pom.xml @@ -0,0 +1,7 @@ + + + 4.0.0 + test + lazy-load + 1.1-SNAPSHOT + diff --git a/src/main/java/AbstractSortedMap.java b/src/main/java/AbstractSortedMap.java new file mode 100644 index 0000000000..f4e987973b --- /dev/null +++ b/src/main/java/AbstractSortedMap.java @@ -0,0 +1,113 @@ +import java.util.AbstractMap; +import java.util.Comparator; +import java.util.Map; +import java.util.NavigableMap; +import java.util.NavigableSet; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; + +/** + * Implementing AbstractSortedMap really ends up implelmenting AbstractNavigableMap + * because SortedMap isn't really self contained / closed to itself. + * + * @author Kohsuke Kawaguchi + */ +public abstract class AbstractSortedMap extends AbstractMap implements SortedMap, NavigableMap { + @Override + public abstract NavigableSet> entrySet(); + + public K lowerKey(K key) { + return key(lowerEntry(key)); + } + + public K floorKey(K key) { + return key(floorEntry(key)); + } + + public K ceilingKey(K key) { + return key(ceilingEntry(key)); + } + + public K higherKey(K key) { + return key(higherEntry(key)); + } + + public K firstKey() { + return key(firstEntry()); + } + + public K lastKey() { + return key(lastEntry()); + } + + private K key(Entry e) { + return e!=null ? e.getKey() : null; + } + + public NavigableMap subMap(K fromKey, K toKey) { + return subMap(fromKey,true,toKey,false); + } + + public SortedMap headMap(K toKey) { + return headMap(toKey,false); + } + + public SortedMap tailMap(K fromKey) { + return tailMap(fromKey,true); + } + + public Entry pollFirstEntry() { + return remove(copy(firstEntry())); + } + + public Entry pollLastEntry() { + return remove(copy(lastEntry())); + } + + public NavigableSet navigableKeySet() { + // TODO + throw new UnsupportedOperationException(); + } + + private Entry remove(Entry e) { + if (e != null) + remove(e.getKey()); + return e; + } + + protected Entry copy(Entry e) { + return e!=null ? new SimpleEntry(e.getKey(),e.getValue()) : null; + } + + public NavigableMap subMap(K fromKey, boolean fromInclusive, K toKey, boolean toInclusive) { + final AbstractSortedMap back = this; + return new AbstractSortedMap() { + @Override + public NavigableSet> entrySet() { + return back.entrySet().subSet(fromKey, toKey) + } + + public Comparator comparator() { + // TODO + throw new UnsupportedOperationException(); + } + + public K firstKey() { + // TODO + throw new UnsupportedOperationException(); + } + + public K lastKey() { + // TODO + throw new UnsupportedOperationException(); + } + }; + } + + final int compare(Object k1, Object k2) { + Comparator c = comparator(); + if (c==null) return ((Comparable)k1).compareTo(k2); + else return c.compare(k1,k2); + } +} diff --git a/src/main/java/Attempt2.java b/src/main/java/Attempt2.java new file mode 100644 index 0000000000..7c34e053a0 --- /dev/null +++ b/src/main/java/Attempt2.java @@ -0,0 +1,199 @@ +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.util.AbstractMap; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; + +import static Attempt2.Direction.*; + +/** + * @author Kohsuke Kawaguchi + */ +public abstract class Attempt2 extends AbstractMap implements SortedMap { + + private boolean fullyLoaded; + // copy on write map that stores what's already byNumber + private volatile TreeMap byNumber = new TreeMap(); + + /** + * Stores the build ID to build number for builds that we already know + */ + private final TreeMap byId = new TreeMap(); + + /** + * IDs found as directories, in the ascending order. + */ + private List idOnDisk; + + /** + * Number shortcuts found on disk, in the ascending order. + */ + private List numberOnDisk; + + private final File dir; + + private final Object loadLock = this; + + public Attempt2(File dir) { + this.dir = dir; + } + + public Comparator comparator() { + return COMPARATOR; + } + + @Override + public Set> entrySet() { + return all().entrySet(); + } + + public Integer firstKey() { + R r = search(Integer.MIN_VALUE, Direction.ASC); + if (r==null) throw new NoSuchElementException(); + return getNumberOf(r); + } + + @Override + public R get(Object key) { + if (key instanceof Integer) { + int n = (Integer) key; + if (byNumber.containsKey(n)) + return byNumber.get(n); + + return search(n, EXACT); + } + return super.get(key); + } + + protected R search(int n, Direction d) { + while (true) { + Entry f = byNumber.floorEntry(n); + if (f!=null && f.getKey()== n) return f.getValue(); // found the exact #n + + // at this point we know that we don't have #n + // but maybe we have disk cache? + if (!idOnDisk.isEmpty()) { + int pos = Collections.binarySearch(idOnDisk, n); + if (pos>=0) { + R r = load(n, true); + if (d==EXACT || r!=null) return r; + + } else { + // non-exact match + n = d.candidate(pos); + } + } + + Entry c = byNumber.ceilingEntry(n); + + // bisect the expected record 'start' by two bounds + } + } + + protected int toIdIndex(Entry f, int fallback) { + if (f==null) return fallback; + return getIdOf(f.getValue()); + } + + + /** + * Loads all the build records to fully populate the map. Last resort. + */ + private TreeMap all() { + if (!fullyLoaded) { + synchronized (this) { + if (!fullyLoaded) { + String[] buildDirs = dir.list(createDirectoryFilter()); + if (buildDirs!=null) { + copy(); + for (String id : buildDirs) { + if (!byId.containsKey(id)) + load(id); + } + } + } + } + } + return byNumber; + } + + /** + * Creates a duplicate for the COW data structure in preparation for mutation. + */ + private void copy() { + byNumber = new TreeMap(byNumber); + } + + /** + * Tries to load the record #N by using the shortcut. + * + * @return null if the data failed to load. + */ + protected R load(int n, boolean copy) { + R r = null; + File shortcut = new File(dir,String.valueOf(n)); + if (shortcut.isDirectory()) { + r = load(shortcut,copy); + if (r==null) { + // if failed to locate, record that fact + if (copy) copy(); + byNumber.put(n,null); + } + } + return r; + } + + protected R load(File dataDir, boolean copy) { + try { + R r = retrieve(dataDir); + if (copy) copy(); + byId.put(getIdOf(r),r); + byNumber.put(getNumberOf(r),r); + return r; + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + protected abstract int getNumberOf(R r); + protected abstract String getIdOf(R r); + + protected abstract R retrieve(File dir) throws IOException; + + /** + * Lists the actual data directory + */ + protected abstract FilenameFilter createDirectoryFilter(); + + public static final Comparator COMPARATOR = new Comparator() { + public int compare(Comparable o1, Comparable o2) { + return -o1.compareTo(o2); + } + }; + + enum Direction { + UP(0), DOWN(-1), EXACT(0/*invalid*/); + + private final int binarySearchTrimOffset; + + Direction(int binarySearchTrimOffset) { + this.binarySearchTrimOffset = binarySearchTrimOffset; + } + + /** + * Given the negative result from {@link Collections#binarySearch(List, Object)} + * that indicates no-exact match, determine the position of the element we'll load + * instead that matches with the search order. + */ + private int candidate(int n) { + return -n-1+binarySearchTrimOffset; + } + }; +} diff --git a/src/main/java/RunMap.java b/src/main/java/RunMap.java new file mode 100644 index 0000000000..80db0d6d86 --- /dev/null +++ b/src/main/java/RunMap.java @@ -0,0 +1,21 @@ +import java.util.AbstractMap; +import java.util.Comparator; +import java.util.SortedMap; + +/** + * @author Kohsuke Kawaguchi + */ +public class RunMap extends AbstractMap implements SortedMap { + + public Comparator comparator() { + return COMPARATOR; + } + + + + public static final Comparator COMPARATOR = new Comparator() { + public int compare(Comparable o1, Comparable o2) { + return -o1.compareTo(o2); + } + }; +} diff --git a/src/main/java/SortedStringList.java b/src/main/java/SortedStringList.java new file mode 100644 index 0000000000..a0fa7ea64a --- /dev/null +++ b/src/main/java/SortedStringList.java @@ -0,0 +1,27 @@ +import java.util.AbstractList; +import java.util.Arrays; +import java.util.List; + +/** + * @author Kohsuke Kawaguchi + */ +public class SortedStringList extends AbstractList { + private String[] data; + + public SortedStringList(String[] data) { + this.data = data; + } + + public int find(String probe) { + return Arrays.binarySearch(data,probe); + } + + public String get(int idx) { + return data[idx]; + } + + @Override + public int size() { + return data.length; + } +} From 024a550d1fb7063d18583271034519b79c0ef70a Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Tue, 11 Sep 2012 09:38:27 -0700 Subject: [PATCH 02/69] Checking in to move it to another computer --- src/main/java/AbstractSortedMap.java | 2 +- src/main/java/Attempt2.java | 77 +++++++++++++++++++++------- src/main/java/SortedStringList.java | 45 +++++++++++++--- 3 files changed, 98 insertions(+), 26 deletions(-) diff --git a/src/main/java/AbstractSortedMap.java b/src/main/java/AbstractSortedMap.java index f4e987973b..606aff937e 100644 --- a/src/main/java/AbstractSortedMap.java +++ b/src/main/java/AbstractSortedMap.java @@ -8,7 +8,7 @@ import java.util.SortedMap; import java.util.SortedSet; /** - * Implementing AbstractSortedMap really ends up implelmenting AbstractNavigableMap + * Implementing AbstractSortedMap really ends up implementing AbstractNavigableMap * because SortedMap isn't really self contained / closed to itself. * * @author Kohsuke Kawaguchi diff --git a/src/main/java/Attempt2.java b/src/main/java/Attempt2.java index 7c34e053a0..653c05db55 100644 --- a/src/main/java/Attempt2.java +++ b/src/main/java/Attempt2.java @@ -13,12 +13,17 @@ import java.util.TreeMap; import static Attempt2.Direction.*; /** + * After {@link AbstractSortedMap} is abandoned, this one is more crude implementation. + * + * This implementation is in 2 states. An instance can be {@linkplain #fullyLoaded fully loaded} state, + * where everything that can be loaded gets loaded. Otherwise it's in a partially loaded state. + * * @author Kohsuke Kawaguchi */ public abstract class Attempt2 extends AbstractMap implements SortedMap { private boolean fullyLoaded; - // copy on write map that stores what's already byNumber + // copy on write map that stores what's already loaded private volatile TreeMap byNumber = new TreeMap(); /** @@ -27,14 +32,14 @@ public abstract class Attempt2 extends AbstractMap implements Sort private final TreeMap byId = new TreeMap(); /** - * IDs found as directories, in the ascending order. + * Build IDs found as directories, in the ascending order. */ - private List idOnDisk; + private SortedStringList idOnDisk; /** - * Number shortcuts found on disk, in the ascending order. + * Build bumber shortcuts found on disk, in the ascending order. */ - private List numberOnDisk; + private List numberOnDisk; private final File dir; @@ -59,6 +64,12 @@ public abstract class Attempt2 extends AbstractMap implements Sort return getNumberOf(r); } + public Integer lastKey() { + R r = search(Integer.MAX_VALUE, Direction.DESC); + if (r==null) throw new NoSuchElementException(); + return getNumberOf(r); + } + @Override public R get(Object key) { if (key instanceof Integer) { @@ -71,28 +82,50 @@ public abstract class Attempt2 extends AbstractMap implements Sort return super.get(key); } + /** + * Loads the build #M where M is nearby the given 'n'. + * + * @param n + * the index to start the search from + * @param d + * defines what we mean by "nearby" above. + * If EXACT, find #N or return null. + * If ASC, finds the closest #M that satisfies M>=N. + * If DESC, finds the closest #M that satisfies M<=N. + */ protected R search(int n, Direction d) { while (true) { Entry f = byNumber.floorEntry(n); if (f!=null && f.getKey()== n) return f.getValue(); // found the exact #n - // at this point we know that we don't have #n - // but maybe we have disk cache? - if (!idOnDisk.isEmpty()) { - int pos = Collections.binarySearch(idOnDisk, n); - if (pos>=0) { - R r = load(n, true); - if (d==EXACT || r!=null) return r; + // at this point we know that we don't have #n loaded yet - } else { - // non-exact match - n = d.candidate(pos); - } - } + // TODO: use numberOnDisk to see if we can find it quickly + + // slow path: we have to find the build from idOnDisk. + // first, narrow down the candidate IDs to try by using two known number-to-ID mapping + if (idOnDisk.isEmpty()) return null; Entry c = byNumber.ceilingEntry(n); + // if bound is null, use a sentinel value + String fid = f==null ? "" : getIdOf(f.getValue()); + String cid = c==null ? "z" : getIdOf(c.getValue()); + + idOnDisk.subList(idOnDisk.higher(fid), idOnDisk.lower(cid)); + + int pos = Collections.binarySearch(idOnDisk, n); + if (pos>=0) {// found it + R r = load(n, true); + if (d==EXACT || r!=null) return r; + } else { + // non-exact match + n = d.candidate(pos); + } + + // bisect the expected record 'start' by two bounds + } } @@ -103,7 +136,13 @@ public abstract class Attempt2 extends AbstractMap implements Sort /** - * Loads all the build records to fully populate the map. Last resort. + * Loads all the build records to fully populate the map. + * Calling this method results in eager loading everything, + * so the whole point of this class is to avoid this call as much as possible + * for typical code path. + * + * @return + * fully populated map. */ private TreeMap all() { if (!fullyLoaded) { @@ -179,7 +218,7 @@ public abstract class Attempt2 extends AbstractMap implements Sort }; enum Direction { - UP(0), DOWN(-1), EXACT(0/*invalid*/); + ASC(0), DESC(-1), EXACT(0/*invalid*/); private final int binarySearchTrimOffset; diff --git a/src/main/java/SortedStringList.java b/src/main/java/SortedStringList.java index a0fa7ea64a..1287c07347 100644 --- a/src/main/java/SortedStringList.java +++ b/src/main/java/SortedStringList.java @@ -1,27 +1,60 @@ import java.util.AbstractList; import java.util.Arrays; +import java.util.Collections; import java.util.List; /** * @author Kohsuke Kawaguchi */ -public class SortedStringList extends AbstractList { - private String[] data; +public class SortedStringList extends AbstractList { + private List data; - public SortedStringList(String[] data) { + public SortedStringList(List data) { this.data = data; } public int find(String probe) { - return Arrays.binarySearch(data,probe); + return Collections.binarySearch(data, probe); } public String get(int idx) { - return data[idx]; + return data.get(idx); } @Override public int size() { - return data.length; + return data.size(); } + + /** + * Finds the index of the entry lower than v. + */ + public int lower(String v) { + int r = find(v); + if (r>0) return r-1; + + int ip = -(r+1); + return r-1; + } + + public int higher(String v) { + int r = find(v); + if (r>0) return r+1; + + int ip = -(r+1); + return r; + } + + public int floor(String v) { + throw new UnsupportedOperationException(); + } + + public int ceil(String v) { + throw new UnsupportedOperationException(); + } + + public List safeSubList(int from, int to) { + return subList(inside(from),inside(to)); + } + } From cf85b72036769549a2221a4c81789552d953006e Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sat, 15 Sep 2012 10:13:01 -0700 Subject: [PATCH 03/69] making more progress --- pom.xml | 15 +++ src/main/java/AbstractSortedMap.java | 46 +++++---- src/main/java/Attempt2.java | 144 ++++++++++++++++++--------- src/main/java/RunMap.java | 21 ---- src/main/java/SortedStringList.java | 90 ++++++++++++++--- 5 files changed, 212 insertions(+), 104 deletions(-) delete mode 100644 src/main/java/RunMap.java diff --git a/pom.xml b/pom.xml index ea67981a07..2be2d89bbd 100644 --- a/pom.xml +++ b/pom.xml @@ -4,4 +4,19 @@ test lazy-load 1.1-SNAPSHOT + + + + commons-io + commons-io + 2.0.1 + + + junit + junit + 4.9 + + + + diff --git a/src/main/java/AbstractSortedMap.java b/src/main/java/AbstractSortedMap.java index 606aff937e..a3f26fee6e 100644 --- a/src/main/java/AbstractSortedMap.java +++ b/src/main/java/AbstractSortedMap.java @@ -80,30 +80,32 @@ public abstract class AbstractSortedMap extends AbstractMap implements return e!=null ? new SimpleEntry(e.getKey(),e.getValue()) : null; } - public NavigableMap subMap(K fromKey, boolean fromInclusive, K toKey, boolean toInclusive) { - final AbstractSortedMap back = this; - return new AbstractSortedMap() { - @Override - public NavigableSet> entrySet() { - return back.entrySet().subSet(fromKey, toKey) - } - public Comparator comparator() { - // TODO - throw new UnsupportedOperationException(); - } - public K firstKey() { - // TODO - throw new UnsupportedOperationException(); - } - - public K lastKey() { - // TODO - throw new UnsupportedOperationException(); - } - }; - } +// public NavigableMap subMap(K fromKey, boolean fromInclusive, K toKey, boolean toInclusive) { +// final AbstractSortedMap back = this; +// return new AbstractSortedMap() { +// @Override +// public NavigableSet> entrySet() { +// return back.entrySet().subSet(fromKey, toKey); +// } +// +// public Comparator comparator() { +// // TODO +// throw new UnsupportedOperationException(); +// } +// +// public K firstKey() { +// // TODO +// throw new UnsupportedOperationException(); +// } +// +// public K lastKey() { +// // TODO +// throw new UnsupportedOperationException(); +// } +// }; +// } final int compare(Object k1, Object k2) { Comparator c = comparator(); diff --git a/src/main/java/Attempt2.java b/src/main/java/Attempt2.java index 653c05db55..3dd40d1929 100644 --- a/src/main/java/Attempt2.java +++ b/src/main/java/Attempt2.java @@ -2,6 +2,7 @@ import java.io.File; import java.io.FilenameFilter; import java.io.IOException; import java.util.AbstractMap; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -10,8 +11,6 @@ import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; -import static Attempt2.Direction.*; - /** * After {@link AbstractSortedMap} is abandoned, this one is more crude implementation. * @@ -42,13 +41,23 @@ public abstract class Attempt2 extends AbstractMap implements Sort private List numberOnDisk; private final File dir; - + + /** + * Used to ensure only one thread is actually calling {@link #retrieve(File)} and + * updating {@link #byNumber} and {@link #byId}. + */ private final Object loadLock = this; public Attempt2(File dir) { this.dir = dir; } + private void loadIdOnDisk() { + String[] buildDirs = dir.list(createDirectoryFilter()); + if (buildDirs==null) buildDirs=new String[0]; + idOnDisk = new SortedStringList(Arrays.asList(buildDirs)); + } + public Comparator comparator() { return COMPARATOR; } @@ -58,6 +67,21 @@ public abstract class Attempt2 extends AbstractMap implements Sort return all().entrySet(); } + public SortedMap subMap(Integer fromKey, Integer toKey) { + // TODO: maybe we do want efficient implementation here + return all().subMap(fromKey,toKey); + } + + public SortedMap headMap(Integer toKey) { + // TODO: maybe we do want efficient implementation here + return all().headMap(toKey); + } + + public SortedMap tailMap(Integer fromKey) { + // TODO: maybe we do want efficient implementation here + return all().tailMap(fromKey); + } + public Integer firstKey() { R r = search(Integer.MIN_VALUE, Direction.ASC); if (r==null) throw new NoSuchElementException(); @@ -77,7 +101,7 @@ public abstract class Attempt2 extends AbstractMap implements Sort if (byNumber.containsKey(n)) return byNumber.get(n); - return search(n, EXACT); + return search(n, Direction.EXACT); } return super.get(key); } @@ -93,7 +117,7 @@ public abstract class Attempt2 extends AbstractMap implements Sort * If ASC, finds the closest #M that satisfies M>=N. * If DESC, finds the closest #M that satisfies M<=N. */ - protected R search(int n, Direction d) { + protected R search(final int n, final Direction d) { while (true) { Entry f = byNumber.floorEntry(n); if (f!=null && f.getKey()== n) return f.getValue(); // found the exact #n @@ -112,27 +136,51 @@ public abstract class Attempt2 extends AbstractMap implements Sort String fid = f==null ? "" : getIdOf(f.getValue()); String cid = c==null ? "z" : getIdOf(c.getValue()); - idOnDisk.subList(idOnDisk.higher(fid), idOnDisk.lower(cid)); + // We know that the build we are looking for exists in this range + // we will narrow this down via binary search + int lo = idOnDisk.higher(fid); + int hi = idOnDisk.lower(cid); - int pos = Collections.binarySearch(idOnDisk, n); - if (pos>=0) {// found it - R r = load(n, true); - if (d==EXACT || r!=null) return r; - } else { - // non-exact match - n = d.candidate(pos); + int pivot; + while (true) { + pivot = (lo+hi)/2; + if (hi<=lo) break; // end of search + + R r = load(new File(dir, idOnDisk.get(pivot)), true); + if (r==null) { + // this ID isn't valid. get rid of that and retry pivot + hi--; + idOnDisk.remove(pivot); + continue; + } + + int found = getNumberOf(r); + if (found==n) + return r; // exact match + + if (found f, int fallback) { - if (f==null) return fallback; - return getIdOf(f.getValue()); - } +// protected int toIdIndex(Entry f, int fallback) { +// if (f==null) return fallback; +// return getNumberOf(f.getValue()); +// } /** @@ -146,16 +194,14 @@ public abstract class Attempt2 extends AbstractMap implements Sort */ private TreeMap all() { if (!fullyLoaded) { - synchronized (this) { + synchronized (loadLock) { if (!fullyLoaded) { - String[] buildDirs = dir.list(createDirectoryFilter()); - if (buildDirs!=null) { - copy(); - for (String id : buildDirs) { - if (!byId.containsKey(id)) - load(id); - } + copy(); + for (String id : idOnDisk) { + if (!byId.containsKey(id)) + load(new File(dir,id),false); // copy() called above, so no need to copy inside } + fullyLoaded = true; } } } @@ -178,29 +224,33 @@ public abstract class Attempt2 extends AbstractMap implements Sort R r = null; File shortcut = new File(dir,String.valueOf(n)); if (shortcut.isDirectory()) { - r = load(shortcut,copy); - if (r==null) { - // if failed to locate, record that fact - if (copy) copy(); - byNumber.put(n,null); + synchronized (loadLock) { + r = load(shortcut,copy); + if (r==null) { + // if failed to locate, record that fact + if (copy) copy(); + byNumber.put(n,null); + } } } return r; } - + protected R load(File dataDir, boolean copy) { - try { - R r = retrieve(dataDir); - if (copy) copy(); - byId.put(getIdOf(r),r); - byNumber.put(getNumberOf(r),r); - return r; - } catch (IOException e) { - e.printStackTrace(); + synchronized (loadLock) { + try { + R r = retrieve(dataDir); + if (copy) copy(); + byId.put(getIdOf(r),r); + byNumber.put(getNumberOf(r),r); + return r; + } catch (IOException e) { + e.printStackTrace(); + } + return null; } - return null; } - + protected abstract int getNumberOf(R r); protected abstract String getIdOf(R r); @@ -226,6 +276,10 @@ public abstract class Attempt2 extends AbstractMap implements Sort this.binarySearchTrimOffset = binarySearchTrimOffset; } + public int offset() { + return binarySearchTrimOffset; + } + /** * Given the negative result from {@link Collections#binarySearch(List, Object)} * that indicates no-exact match, determine the position of the element we'll load @@ -234,5 +288,5 @@ public abstract class Attempt2 extends AbstractMap implements Sort private int candidate(int n) { return -n-1+binarySearchTrimOffset; } - }; + } } diff --git a/src/main/java/RunMap.java b/src/main/java/RunMap.java deleted file mode 100644 index 80db0d6d86..0000000000 --- a/src/main/java/RunMap.java +++ /dev/null @@ -1,21 +0,0 @@ -import java.util.AbstractMap; -import java.util.Comparator; -import java.util.SortedMap; - -/** - * @author Kohsuke Kawaguchi - */ -public class RunMap extends AbstractMap implements SortedMap { - - public Comparator comparator() { - return COMPARATOR; - } - - - - public static final Comparator COMPARATOR = new Comparator() { - public int compare(Comparable o1, Comparable o2) { - return -o1.compareTo(o2); - } - }; -} diff --git a/src/main/java/SortedStringList.java b/src/main/java/SortedStringList.java index 1287c07347..caa91ec3d6 100644 --- a/src/main/java/SortedStringList.java +++ b/src/main/java/SortedStringList.java @@ -1,7 +1,9 @@ import java.util.AbstractList; -import java.util.Arrays; +import java.util.AbstractMap.SimpleEntry; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; /** * @author Kohsuke Kawaguchi @@ -12,7 +14,14 @@ public class SortedStringList extends AbstractList { public SortedStringList(List data) { this.data = data; } - + + /** + * Binary search to find the position of the given string. + * + * @return + * -(insertionPoint+1) if the exact string isn't found. + * That is, -1 means the probe would be inserted at the very beginning. + */ public int find(String probe) { return Collections.binarySearch(data, probe); } @@ -30,31 +39,80 @@ public class SortedStringList extends AbstractList { * Finds the index of the entry lower than v. */ public int lower(String v) { - int r = find(v); - if (r>0) return r-1; - - int ip = -(r+1); - return r-1; + return lowerHigherFloorCeil(v,-1,-1); } + /** + * Finds the index of the entry greater than v. + */ public int higher(String v) { - int r = find(v); - if (r>0) return r+1; + return lowerHigherFloorCeil(v,1,0); + } + + /** + * Finds the index of the entry lower or equal to v. + */ + public int floor(String v) { + return lowerHigherFloorCeil(v,0,0); + } + + /** + * Finds the index of the entry grater or equal to v. + */ + public int ceil(String v) { + return lowerHigherFloorCeil(v,0,1); + } + + private int lowerHigherFloorCeil(String probe, int offsetOfExactMatch, int offsetOfInsertionPoint) { + int r = find(probe); + if (r>=0) return r+offsetOfExactMatch; int ip = -(r+1); - return r; + return ip+offsetOfInsertionPoint; } - public int floor(String v) { - throw new UnsupportedOperationException(); + /** + * Smarter {@link #subList(int, int)} that gracefully handles index that's out of range. + */ + public SortedStringList safeSubList(int from, int to) { + return new SortedStringList(subList(inside(from),inside(to))); } - public int ceil(String v) { - throw new UnsupportedOperationException(); + /** + * Brings the index into the legal range. + */ + private int inside(int idx) { + idx = Math.max(0,idx); + idx = Math.min(idx,size()); + return idx; } - public List safeSubList(int from, int to) { - return subList(inside(from),inside(to)); + /** + * Finds the middle value used for binary search pivot. + */ + public Entry middle() { + assert !data.isEmpty(); + int idx = data.size()/2; + return new SimpleEntry(idx,data.get(idx)); } + /** + * Overloaded version of {@link #remove(int)} that takes a pointer instead of the index. + */ + public void remove(Entry ptr) { + remove(ptr.getKey().intValue()); + } + + /** + * Returns the sub list that contains (pivot,end] + */ + public SortedStringList upperHalf() { + int sz = data.size(); + return safeSubList(sz/2+1,sz); + } + + public SortedStringList lowerHalf() { + int sz = data.size(); + return safeSubList(0,sz/2); + } } From b94565cc327a6ea93fda13703b0ed157a585b16b Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sat, 15 Sep 2012 10:17:35 -0700 Subject: [PATCH 04/69] adding a test case --- src/test/java/Attempt2Test.java | 25 ++++++++++++++ src/test/java/FakeMap.java | 54 +++++++++++++++++++++++++++++++ src/test/java/FakeMapBuilder.java | 44 +++++++++++++++++++++++++ 3 files changed, 123 insertions(+) create mode 100644 src/test/java/Attempt2Test.java create mode 100644 src/test/java/FakeMap.java create mode 100644 src/test/java/FakeMapBuilder.java diff --git a/src/test/java/Attempt2Test.java b/src/test/java/Attempt2Test.java new file mode 100644 index 0000000000..e87141a4c6 --- /dev/null +++ b/src/test/java/Attempt2Test.java @@ -0,0 +1,25 @@ +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; + +import java.io.IOException; + +/** + * @author Kohsuke Kawaguchi + */ +public class Attempt2Test extends Assert { + @Rule + public FakeMapBuilder builder = new FakeMapBuilder(); + + @Test + public void testSomething() throws IOException { + FakeMap map = builder.add(1, "A").add(3, "B").add(5, "C").make(); + + map.get(1).asserts(1,"A"); + assertNull(map.get(2)); + assertNull(map.get(4)); + map.get(3).asserts(3,"B"); + map.get(5).asserts(5,"C"); + } + +} diff --git a/src/test/java/FakeMap.java b/src/test/java/FakeMap.java new file mode 100644 index 0000000000..c3a819c1c2 --- /dev/null +++ b/src/test/java/FakeMap.java @@ -0,0 +1,54 @@ +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; + +/** + * @author Kohsuke Kawaguchi + */ +public class FakeMap extends Attempt2 { + public FakeMap(File dir) { + super(dir); + } + + @Override + protected int getNumberOf(Build build) { + return build.n; + } + + @Override + protected String getIdOf(Build build) { + return build.id; + } + + @Override + protected FilenameFilter createDirectoryFilter() { + return new FilenameFilter() { + public boolean accept(File dir, String name) { + return name.startsWith("_"); + } + }; + } + + @Override + protected Build retrieve(File dir) throws IOException { + String n = FileUtils.readFileToString(new File(dir, "n")).trim(); + return new Build(Integer.parseInt(n),dir.getName()); + } +} + +class Build { + final int n; + final String id; + + Build(int n, String id) { + this.n = n; + this.id = id; + } + + public void asserts(int n, String id) { + assert this.n==n; + assert this.id.equals(id); + } +} \ No newline at end of file diff --git a/src/test/java/FakeMapBuilder.java b/src/test/java/FakeMapBuilder.java new file mode 100644 index 0000000000..88f40d4865 --- /dev/null +++ b/src/test/java/FakeMapBuilder.java @@ -0,0 +1,44 @@ +import org.apache.commons.io.FileUtils; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; + +import java.io.File; +import java.io.IOException; + +/** + * @author Kohsuke Kawaguchi + */ +public class FakeMapBuilder implements TestRule { + private File dir; + + public FakeMapBuilder() { + } + + public FakeMapBuilder add(int n, String id) throws IOException { + File build = new File(dir, id); + build.mkdir(); + FileUtils.write(new File(build,"n"),Integer.toString(n)); + return this; + } + + public FakeMap make() { + return new FakeMap(dir); + } + + public Statement apply(final Statement base, Description description) { + return new Statement() { + @Override + public void evaluate() throws Throwable { + dir = File.createTempFile("lazyload","test"); + dir.delete(); + dir.mkdirs(); + try { + base.evaluate(); + } finally { + FileUtils.deleteDirectory(dir); + } + } + }; + } +} From dae7ba2027e3eb11ecf4bbfcbac438353c04d7c0 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sat, 15 Sep 2012 10:35:21 -0700 Subject: [PATCH 05/69] adding a test case and fixed off-by-one bug --- src/main/java/SortedStringList.java | 7 ++-- src/test/java/SortedStringListTest.java | 55 +++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 src/test/java/SortedStringListTest.java diff --git a/src/main/java/SortedStringList.java b/src/main/java/SortedStringList.java index caa91ec3d6..835fb7f63c 100644 --- a/src/main/java/SortedStringList.java +++ b/src/main/java/SortedStringList.java @@ -2,7 +2,6 @@ import java.util.AbstractList; import java.util.AbstractMap.SimpleEntry; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Map.Entry; /** @@ -53,14 +52,14 @@ public class SortedStringList extends AbstractList { * Finds the index of the entry lower or equal to v. */ public int floor(String v) { - return lowerHigherFloorCeil(v,0,0); + return lowerHigherFloorCeil(v,0,-1); } /** - * Finds the index of the entry grater or equal to v. + * Finds the index of the entry greater or equal to v. */ public int ceil(String v) { - return lowerHigherFloorCeil(v,0,1); + return lowerHigherFloorCeil(v,0,0); } private int lowerHigherFloorCeil(String probe, int offsetOfExactMatch, int offsetOfInsertionPoint) { diff --git a/src/test/java/SortedStringListTest.java b/src/test/java/SortedStringListTest.java new file mode 100644 index 0000000000..14cd992d7d --- /dev/null +++ b/src/test/java/SortedStringListTest.java @@ -0,0 +1,55 @@ +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; + +/** + * @author Kohsuke Kawaguchi + */ +public class SortedStringListTest extends Assert { + SortedStringList l = new SortedStringList(Arrays.asList("B","D","F")); + + @Test + public void testCeil() { + assertEquals(0,l.ceil("A")); + assertEquals(0,l.ceil("B")); + assertEquals(1,l.ceil("C")); + assertEquals(1,l.ceil("D")); + assertEquals(2,l.ceil("E")); + assertEquals(2,l.ceil("F")); + assertEquals(3,l.ceil("G")); + } + + @Test + public void testFloor() { + assertEquals(-1,l.floor("A")); + assertEquals(0,l.floor("B")); + assertEquals(0,l.floor("C")); + assertEquals(1,l.floor("D")); + assertEquals(1,l.floor("E")); + assertEquals(2,l.floor("F")); + assertEquals(2,l.floor("G")); + } + + @Test + public void testLower() { + assertEquals(-1,l.lower("A")); + assertEquals(-1,l.lower("B")); + assertEquals(0,l.lower("C")); + assertEquals(0,l.lower("D")); + assertEquals(1,l.lower("E")); + assertEquals(1,l.lower("F")); + assertEquals(2,l.lower("G")); + } + + @Test + public void testHigher() { + assertEquals(0,l.higher("A")); + assertEquals(1,l.higher("B")); + assertEquals(1,l.higher("C")); + assertEquals(2,l.higher("D")); + assertEquals(2,l.higher("E")); + assertEquals(3,l.higher("F")); + assertEquals(3,l.higher("G")); + } +} From 8bcaa021c329dca5e9deaef2fe4c172b50dd2526 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sat, 15 Sep 2012 10:40:48 -0700 Subject: [PATCH 06/69] using a test as a driver to fix bugs --- src/main/java/Attempt2.java | 120 +++++++++++++++++------------- src/test/java/FakeMap.java | 7 +- src/test/java/FakeMapBuilder.java | 9 ++- 3 files changed, 82 insertions(+), 54 deletions(-) diff --git a/src/main/java/Attempt2.java b/src/main/java/Attempt2.java index 3dd40d1929..22cf14f70b 100644 --- a/src/main/java/Attempt2.java +++ b/src/main/java/Attempt2.java @@ -33,11 +33,13 @@ public abstract class Attempt2 extends AbstractMap implements Sort /** * Build IDs found as directories, in the ascending order. */ + // copy on write private SortedStringList idOnDisk; /** * Build bumber shortcuts found on disk, in the ascending order. */ + // copy on write private List numberOnDisk; private final File dir; @@ -50,6 +52,7 @@ public abstract class Attempt2 extends AbstractMap implements Sort public Attempt2(File dir) { this.dir = dir; + loadIdOnDisk(); } private void loadIdOnDisk() { @@ -118,62 +121,65 @@ public abstract class Attempt2 extends AbstractMap implements Sort * If DESC, finds the closest #M that satisfies M<=N. */ protected R search(final int n, final Direction d) { + Entry f = byNumber.floorEntry(n); + if (f!=null && f.getKey()== n) return f.getValue(); // found the exact #n + + // at this point we know that we don't have #n loaded yet + + // TODO: use numberOnDisk to see if we can find it quickly + + // capture the snapshot and work off with it since it can be changed by other threads + SortedStringList idOnDisk = this.idOnDisk; + + // slow path: we have to find the build from idOnDisk. + // first, narrow down the candidate IDs to try by using two known number-to-ID mapping + if (idOnDisk.isEmpty()) return null; + + Entry c = byNumber.ceilingEntry(n); + + // if bound is null, use a sentinel value + String fid = f==null ? "\u0000" : getIdOf(f.getValue()); + String cid = c==null ? "\uFFFF" : getIdOf(c.getValue()); + + // We know that the build we are looking for exists in this range + // we will narrow this down via binary search + int lo = idOnDisk.higher(fid); + int hi = idOnDisk.lower(cid)+1; + + int pivot; while (true) { - Entry f = byNumber.floorEntry(n); - if (f!=null && f.getKey()== n) return f.getValue(); // found the exact #n + pivot = (lo+hi)/2; + if (hi<=lo) break; // end of search - // at this point we know that we don't have #n loaded yet - - // TODO: use numberOnDisk to see if we can find it quickly - - // slow path: we have to find the build from idOnDisk. - // first, narrow down the candidate IDs to try by using two known number-to-ID mapping - if (idOnDisk.isEmpty()) return null; - - Entry c = byNumber.ceilingEntry(n); - - // if bound is null, use a sentinel value - String fid = f==null ? "" : getIdOf(f.getValue()); - String cid = c==null ? "z" : getIdOf(c.getValue()); - - // We know that the build we are looking for exists in this range - // we will narrow this down via binary search - int lo = idOnDisk.higher(fid); - int hi = idOnDisk.lower(cid); - - int pivot; - while (true) { - pivot = (lo+hi)/2; - if (hi<=lo) break; // end of search - - R r = load(new File(dir, idOnDisk.get(pivot)), true); - if (r==null) { - // this ID isn't valid. get rid of that and retry pivot - hi--; - idOnDisk.remove(pivot); - continue; - } - - int found = getNumberOf(r); - if (found==n) - return r; // exact match - - if (found extends AbstractMap implements Sort protected abstract R retrieve(File dir) throws IOException; + @Override + public int hashCode() { + return System.identityHashCode(this); + } + + @Override + public boolean equals(Object o) { + return o==this; + } + /** * Lists the actual data directory */ diff --git a/src/test/java/FakeMap.java b/src/test/java/FakeMap.java index c3a819c1c2..9d39e167be 100644 --- a/src/test/java/FakeMap.java +++ b/src/test/java/FakeMap.java @@ -26,7 +26,12 @@ public class FakeMap extends Attempt2 { protected FilenameFilter createDirectoryFilter() { return new FilenameFilter() { public boolean accept(File dir, String name) { - return name.startsWith("_"); + try { + Integer.parseInt(name); + return false; + } catch (NumberFormatException e) { + return true; + } } }; } diff --git a/src/test/java/FakeMapBuilder.java b/src/test/java/FakeMapBuilder.java index 88f40d4865..35e4b5281e 100644 --- a/src/test/java/FakeMapBuilder.java +++ b/src/test/java/FakeMapBuilder.java @@ -16,7 +16,14 @@ public class FakeMapBuilder implements TestRule { } public FakeMapBuilder add(int n, String id) throws IOException { - File build = new File(dir, id); + try { + Integer.parseInt(id); + throw new IllegalMonitorStateException("ID cannot be a number"); + } catch (NumberFormatException e) { + // OK + } + + File build = new File(dir,id); build.mkdir(); FileUtils.write(new File(build,"n"),Integer.toString(n)); return this; From 953c44d52595d708e9a55fc04946875383920011 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sat, 15 Sep 2012 10:54:08 -0700 Subject: [PATCH 07/69] moving into a package --- src/main/java/{ => ll}/AbstractSortedMap.java | 2 + src/main/java/{ => ll}/Attempt2.java | 8 +- src/main/java/{ => ll}/SortedStringList.java | 2 + src/test/java/Attempt2Test.java | 25 ------ src/test/java/ll/Attempt2Test.java | 79 +++++++++++++++++++ src/test/java/{ => ll}/FakeMap.java | 2 + src/test/java/{ => ll}/FakeMapBuilder.java | 2 + .../java/{ => ll}/SortedStringListTest.java | 3 + 8 files changed, 95 insertions(+), 28 deletions(-) rename src/main/java/{ => ll}/AbstractSortedMap.java (99%) rename src/main/java/{ => ll}/Attempt2.java (98%) rename src/main/java/{ => ll}/SortedStringList.java (99%) delete mode 100644 src/test/java/Attempt2Test.java create mode 100644 src/test/java/ll/Attempt2Test.java rename src/test/java/{ => ll}/FakeMap.java (98%) rename src/test/java/{ => ll}/FakeMapBuilder.java (99%) rename src/test/java/{ => ll}/SortedStringListTest.java (97%) diff --git a/src/main/java/AbstractSortedMap.java b/src/main/java/ll/AbstractSortedMap.java similarity index 99% rename from src/main/java/AbstractSortedMap.java rename to src/main/java/ll/AbstractSortedMap.java index a3f26fee6e..bc5b001f3d 100644 --- a/src/main/java/AbstractSortedMap.java +++ b/src/main/java/ll/AbstractSortedMap.java @@ -1,3 +1,5 @@ +package ll; + import java.util.AbstractMap; import java.util.Comparator; import java.util.Map; diff --git a/src/main/java/Attempt2.java b/src/main/java/ll/Attempt2.java similarity index 98% rename from src/main/java/Attempt2.java rename to src/main/java/ll/Attempt2.java index 22cf14f70b..ccdf27f8c3 100644 --- a/src/main/java/Attempt2.java +++ b/src/main/java/ll/Attempt2.java @@ -1,3 +1,5 @@ +package ll; + import java.io.File; import java.io.FilenameFilter; import java.io.IOException; @@ -120,7 +122,7 @@ public abstract class Attempt2 extends AbstractMap implements Sort * If ASC, finds the closest #M that satisfies M>=N. * If DESC, finds the closest #M that satisfies M<=N. */ - protected R search(final int n, final Direction d) { + public R search(final int n, final Direction d) { Entry f = byNumber.floorEntry(n); if (f!=null && f.getKey()== n) return f.getValue(); // found the exact #n @@ -174,8 +176,8 @@ public abstract class Attempt2 extends AbstractMap implements Sort if (hi==idOnDisk.size()) return null; return byId.get(idOnDisk.get(hi)); case DESC: - if (lo==-1) return null; - return byId.get(idOnDisk.get(lo)); + if (lo<=0) return null; + return byId.get(idOnDisk.get(lo-1)); case EXACT: return null; default: diff --git a/src/main/java/SortedStringList.java b/src/main/java/ll/SortedStringList.java similarity index 99% rename from src/main/java/SortedStringList.java rename to src/main/java/ll/SortedStringList.java index 835fb7f63c..79e7a02a17 100644 --- a/src/main/java/SortedStringList.java +++ b/src/main/java/ll/SortedStringList.java @@ -1,3 +1,5 @@ +package ll; + import java.util.AbstractList; import java.util.AbstractMap.SimpleEntry; import java.util.Collections; diff --git a/src/test/java/Attempt2Test.java b/src/test/java/Attempt2Test.java deleted file mode 100644 index e87141a4c6..0000000000 --- a/src/test/java/Attempt2Test.java +++ /dev/null @@ -1,25 +0,0 @@ -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; - -import java.io.IOException; - -/** - * @author Kohsuke Kawaguchi - */ -public class Attempt2Test extends Assert { - @Rule - public FakeMapBuilder builder = new FakeMapBuilder(); - - @Test - public void testSomething() throws IOException { - FakeMap map = builder.add(1, "A").add(3, "B").add(5, "C").make(); - - map.get(1).asserts(1,"A"); - assertNull(map.get(2)); - assertNull(map.get(4)); - map.get(3).asserts(3,"B"); - map.get(5).asserts(5,"C"); - } - -} diff --git a/src/test/java/ll/Attempt2Test.java b/src/test/java/ll/Attempt2Test.java new file mode 100644 index 0000000000..8fd638a581 --- /dev/null +++ b/src/test/java/ll/Attempt2Test.java @@ -0,0 +1,79 @@ +package ll; + +import ll.Attempt2.Direction; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import java.util.NoSuchElementException; + +/** + * @author Kohsuke Kawaguchi + */ +public class Attempt2Test extends Assert { + @Rule + public FakeMapBuilder aBuilder = new FakeMapBuilder(); + private FakeMap a; + + @Rule + public FakeMapBuilder bBuilder = new FakeMapBuilder(); + private FakeMap b; + + + @Before + public void setUp() throws Exception { + a = aBuilder.add(1, "A").add(3, "B").add(5, "C").make(); + + b = bBuilder.make(); + } + + @Test + public void lookup() { + a.get(1).asserts(1,"A"); + assertNull(a.get(2)); + a.get(3).asserts(3,"B"); + assertNull(a.get(4)); + a.get(5).asserts(5,"C"); + + assertNull(b.get(1)); + assertNull(b.get(3)); + assertNull(b.get(5)); + } + + @Test + public void idempotentLookup() { + for (int i=0; i<5; i++) + a.get(1).asserts(1,"A"); + } + + @Test + public void firstKey() { + assertEquals(1, a.firstKey().intValue()); + + try { + b.firstKey(); + fail(); + } catch (NoSuchElementException e) { + // as expected + } + } + + @Test + public void lastKey() { + assertEquals(5, a.lastKey().intValue()); + try { + b.lastKey(); + fail(); + } catch (NoSuchElementException e) { + // as expected + } + } + + @Test + public void search() { + // searching toward non-existent direction + assertNull(a.search( 99, Direction.ASC)); + assertNull(a.search(-99, Direction.DESC)); + } +} diff --git a/src/test/java/FakeMap.java b/src/test/java/ll/FakeMap.java similarity index 98% rename from src/test/java/FakeMap.java rename to src/test/java/ll/FakeMap.java index 9d39e167be..f777b5239c 100644 --- a/src/test/java/FakeMap.java +++ b/src/test/java/ll/FakeMap.java @@ -1,3 +1,5 @@ +package ll; + import org.apache.commons.io.FileUtils; import java.io.File; diff --git a/src/test/java/FakeMapBuilder.java b/src/test/java/ll/FakeMapBuilder.java similarity index 99% rename from src/test/java/FakeMapBuilder.java rename to src/test/java/ll/FakeMapBuilder.java index 35e4b5281e..6fcfc65860 100644 --- a/src/test/java/FakeMapBuilder.java +++ b/src/test/java/ll/FakeMapBuilder.java @@ -1,3 +1,5 @@ +package ll; + import org.apache.commons.io.FileUtils; import org.junit.rules.TestRule; import org.junit.runner.Description; diff --git a/src/test/java/SortedStringListTest.java b/src/test/java/ll/SortedStringListTest.java similarity index 97% rename from src/test/java/SortedStringListTest.java rename to src/test/java/ll/SortedStringListTest.java index 14cd992d7d..1775297d0c 100644 --- a/src/test/java/SortedStringListTest.java +++ b/src/test/java/ll/SortedStringListTest.java @@ -1,3 +1,6 @@ +package ll; + +import ll.SortedStringList; import org.junit.Assert; import org.junit.Test; From 980a1202b511f92689fee54220ccbccf6e7eb446 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sat, 15 Sep 2012 10:55:50 -0700 Subject: [PATCH 08/69] redundant code --- src/main/java/ll/Attempt2.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/ll/Attempt2.java b/src/main/java/ll/Attempt2.java index ccdf27f8c3..bdb60fd6db 100644 --- a/src/main/java/ll/Attempt2.java +++ b/src/main/java/ll/Attempt2.java @@ -103,9 +103,6 @@ public abstract class Attempt2 extends AbstractMap implements Sort public R get(Object key) { if (key instanceof Integer) { int n = (Integer) key; - if (byNumber.containsKey(n)) - return byNumber.get(n); - return search(n, Direction.EXACT); } return super.get(key); From 9ad5632ed49f1808dd74bfac75931c97559ab516 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sat, 15 Sep 2012 11:07:21 -0700 Subject: [PATCH 09/69] added a reasonable coverage --- src/main/java/ll/Attempt2.java | 4 +++- src/main/java/ll/SortedStringList.java | 10 ++++++++ src/test/java/ll/Attempt2Test.java | 33 ++++++++++++++++++++++++++ src/test/java/ll/FakeMapBuilder.java | 6 +++++ 4 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/main/java/ll/Attempt2.java b/src/main/java/ll/Attempt2.java index bdb60fd6db..608d28e6a0 100644 --- a/src/main/java/ll/Attempt2.java +++ b/src/main/java/ll/Attempt2.java @@ -4,6 +4,7 @@ import java.io.File; import java.io.FilenameFilter; import java.io.IOException; import java.util.AbstractMap; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; @@ -60,7 +61,8 @@ public abstract class Attempt2 extends AbstractMap implements Sort private void loadIdOnDisk() { String[] buildDirs = dir.list(createDirectoryFilter()); if (buildDirs==null) buildDirs=new String[0]; - idOnDisk = new SortedStringList(Arrays.asList(buildDirs)); + // wrap into ArrayList to enable mutation + idOnDisk = new SortedStringList(new ArrayList(Arrays.asList(buildDirs))); } public Comparator comparator() { diff --git a/src/main/java/ll/SortedStringList.java b/src/main/java/ll/SortedStringList.java index 79e7a02a17..f870c15385 100644 --- a/src/main/java/ll/SortedStringList.java +++ b/src/main/java/ll/SortedStringList.java @@ -36,6 +36,16 @@ public class SortedStringList extends AbstractList { return data.size(); } + @Override + public String remove(int index) { + return data.remove(index); + } + + @Override + public boolean remove(Object o) { + return data.remove(o); + } + /** * Finds the index of the entry lower than v. */ diff --git a/src/test/java/ll/Attempt2Test.java b/src/test/java/ll/Attempt2Test.java index 8fd638a581..c682562975 100644 --- a/src/test/java/ll/Attempt2Test.java +++ b/src/test/java/ll/Attempt2Test.java @@ -6,20 +6,26 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; +import java.io.IOException; +import java.util.Map; import java.util.NoSuchElementException; /** * @author Kohsuke Kawaguchi */ public class Attempt2Test extends Assert { + // A=1, B=3, C=5 @Rule public FakeMapBuilder aBuilder = new FakeMapBuilder(); private FakeMap a; + // empty map @Rule public FakeMapBuilder bBuilder = new FakeMapBuilder(); private FakeMap b; + @Rule + public FakeMapBuilder localBuilder = new FakeMapBuilder(); @Before public void setUp() throws Exception { @@ -47,6 +53,13 @@ public class Attempt2Test extends Assert { a.get(1).asserts(1,"A"); } + @Test + public void lookupWithBogusKeyType() { + assertNull(a.get(null)); + assertNull(a.get("foo")); + assertNull(a.get(this)); + } + @Test public void firstKey() { assertEquals(1, a.firstKey().intValue()); @@ -76,4 +89,24 @@ public class Attempt2Test extends Assert { assertNull(a.search( 99, Direction.ASC)); assertNull(a.search(-99, Direction.DESC)); } + + /** + * If load fails, search needs to gracefully handle it + */ + @Test + public void unloadableData() throws IOException { + FakeMap m = localBuilder.add(1, "A").addUnloadable("B").add(5, "C").make(); + + assertNull(m.search(3,Direction.EXACT)); + m.search(3,Direction.DESC).asserts(1,"A"); + m.search(3,Direction.ASC ).asserts(5,"C"); + } + + @Test + public void eagerLoading() throws IOException { + Map.Entry[] b = a.entrySet().toArray(new Map.Entry[3]); + ((Build)b[0].getValue()).asserts(1,"A"); + ((Build)b[1].getValue()).asserts(3,"B"); + ((Build)b[2].getValue()).asserts(5,"C"); + } } diff --git a/src/test/java/ll/FakeMapBuilder.java b/src/test/java/ll/FakeMapBuilder.java index 6fcfc65860..a928f7f207 100644 --- a/src/test/java/ll/FakeMapBuilder.java +++ b/src/test/java/ll/FakeMapBuilder.java @@ -31,6 +31,12 @@ public class FakeMapBuilder implements TestRule { return this; } + public FakeMapBuilder addUnloadable(String id) throws IOException { + File build = new File(dir,id); + build.mkdir(); + return this; + } + public FakeMap make() { return new FakeMap(dir); } From 03a077e80b73ab1aa63b562597fac20ca8e6e8f2 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sat, 15 Sep 2012 11:27:17 -0700 Subject: [PATCH 10/69] externalized algorithm --- src/main/java/ll/Boundary.java | 29 ++++++++ src/main/java/ll/SortedIntList.java | 91 ++++++++++++++++++++++++++ src/main/java/ll/SortedStringList.java | 16 ++--- 3 files changed, 124 insertions(+), 12 deletions(-) create mode 100644 src/main/java/ll/Boundary.java create mode 100644 src/main/java/ll/SortedIntList.java diff --git a/src/main/java/ll/Boundary.java b/src/main/java/ll/Boundary.java new file mode 100644 index 0000000000..b3c7968dcf --- /dev/null +++ b/src/main/java/ll/Boundary.java @@ -0,0 +1,29 @@ +package ll; + +/** + * ceil/floor/lower/higher implementations + * based on the result from a binary search. + * + * @author Kohsuke Kawaguchi + */ +public enum Boundary { + LOWER(-1,-1), + HIGHER(1,0), + FLOOR(0,-1), + CEIL(0,0); + + private final int offsetOfExactMatch, offsetOfInsertionPoint; + + private Boundary(int offsetOfExactMatch, int offsetOfInsertionPoint) { + this.offsetOfExactMatch = offsetOfExactMatch; + this.offsetOfInsertionPoint = offsetOfInsertionPoint; + } + + public int apply(int binarySearchOutput) { + int r = binarySearchOutput; + if (r>=0) return r+offsetOfExactMatch; + + int ip = -(r+1); + return ip+offsetOfInsertionPoint; + } +} diff --git a/src/main/java/ll/SortedIntList.java b/src/main/java/ll/SortedIntList.java new file mode 100644 index 0000000000..2315aba82e --- /dev/null +++ b/src/main/java/ll/SortedIntList.java @@ -0,0 +1,91 @@ +package ll; + +import java.util.AbstractList; +import java.util.Arrays; + +/** + * @author Kohsuke Kawaguchi + */ +public class SortedIntList extends AbstractList { + private int[] data; + private int size; + + public SortedIntList(int capacity) { + this.data = new int[capacity]; + this.size = 0; + } + + public SortedIntList() { + this(16); + } + + /** + * Binary search to find the position of the given string. + * + * @return + * -(insertionPoint+1) if the exact string isn't found. + * That is, -1 means the probe would be inserted at the very beginning. + */ + public int find(int probe) { + return Arrays.binarySearch(data, 0, size, probe); + } + + @Override + public Integer get(int index) { + if (size<=index) throw new IndexOutOfBoundsException(); + return data[index]; + } + + @Override + public int size() { + return size; + } + + @Override + public boolean add(Integer i) { + return add(i.intValue()); + } + + public boolean add(int i) { + ensureCapacity(size+1); + data[size++] = i; + return true; + } + + private void ensureCapacity(int i) { + if (data.length { * Finds the index of the entry lower than v. */ public int lower(String v) { - return lowerHigherFloorCeil(v,-1,-1); + return Boundary.LOWER.apply(find(v)); } /** * Finds the index of the entry greater than v. */ public int higher(String v) { - return lowerHigherFloorCeil(v,1,0); + return Boundary.HIGHER.apply(find(v)); } /** * Finds the index of the entry lower or equal to v. */ public int floor(String v) { - return lowerHigherFloorCeil(v,0,-1); + return Boundary.FLOOR.apply(find(v)); } /** * Finds the index of the entry greater or equal to v. */ public int ceil(String v) { - return lowerHigherFloorCeil(v,0,0); - } - - private int lowerHigherFloorCeil(String probe, int offsetOfExactMatch, int offsetOfInsertionPoint) { - int r = find(probe); - if (r>=0) return r+offsetOfExactMatch; - - int ip = -(r+1); - return ip+offsetOfInsertionPoint; + return Boundary.CEIL.apply(find(v)); } /** From 54223b5f035a7c100da2cdab46e99e44828658e7 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sat, 15 Sep 2012 11:49:09 -0700 Subject: [PATCH 11/69] initial cut of the fast search path that uses build number symlinks --- src/main/java/ll/Attempt2.java | 113 +++++++++++++++++++++++-- src/main/java/ll/SortedIntList.java | 3 + src/main/java/ll/SortedStringList.java | 4 + 3 files changed, 112 insertions(+), 8 deletions(-) diff --git a/src/main/java/ll/Attempt2.java b/src/main/java/ll/Attempt2.java index 608d28e6a0..682ac06c93 100644 --- a/src/main/java/ll/Attempt2.java +++ b/src/main/java/ll/Attempt2.java @@ -14,6 +14,9 @@ import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; +import static ll.Attempt2.Direction.ASC; +import static ll.Boundary.*; + /** * After {@link AbstractSortedMap} is abandoned, this one is more crude implementation. * @@ -43,7 +46,7 @@ public abstract class Attempt2 extends AbstractMap implements Sort * Build bumber shortcuts found on disk, in the ascending order. */ // copy on write - private List numberOnDisk; + private SortedIntList numberOnDisk; private final File dir; @@ -60,9 +63,22 @@ public abstract class Attempt2 extends AbstractMap implements Sort private void loadIdOnDisk() { String[] buildDirs = dir.list(createDirectoryFilter()); - if (buildDirs==null) buildDirs=new String[0]; + if (buildDirs==null) buildDirs=EMPTY_STRING_ARRAY; // wrap into ArrayList to enable mutation idOnDisk = new SortedStringList(new ArrayList(Arrays.asList(buildDirs))); + + // TODO: should we check that shortcuts is a symlink? + String[] shortcuts = dir.list(); + if (shortcuts==null) shortcuts=EMPTY_STRING_ARRAY; + SortedIntList list = new SortedIntList(shortcuts.length/2); + for (String s : shortcuts) { + try { + list.add(Integer.parseInt(s)); + } catch (NumberFormatException e) { + // this isn't a shortcut + } + } + numberOnDisk = list; } public Comparator comparator() { @@ -90,7 +106,7 @@ public abstract class Attempt2 extends AbstractMap implements Sort } public Integer firstKey() { - R r = search(Integer.MIN_VALUE, Direction.ASC); + R r = search(Integer.MIN_VALUE, ASC); if (r==null) throw new NoSuchElementException(); return getNumberOf(r); } @@ -110,6 +126,10 @@ public abstract class Attempt2 extends AbstractMap implements Sort return super.get(key); } + public R get(int n) { + return search(n,Direction.EXACT); + } + /** * Loads the build #M where M is nearby the given 'n'. * @@ -127,7 +147,52 @@ public abstract class Attempt2 extends AbstractMap implements Sort // at this point we know that we don't have #n loaded yet - // TODO: use numberOnDisk to see if we can find it quickly + {// check numberOnDisk as a cache to see if we can find it there + int npos = numberOnDisk.find(n); + if (npos>=0) {// found exact match + R r = load(numberOnDisk.get(npos), true); + if (r!=null) + return r; + } + + switch (d) { + case ASC: + case DESC: + // didn't find the exact match, but what's the nearest ascending value in the cache? + int neighbor = (d==ASC?HIGHER:LOWER).apply(npos); + if (numberOnDisk.isInRange(neighbor)) { + R r = getByNumber(numberOnDisk.get(neighbor)); + if (r!=null) { + // make sure that the cache is accurate by looking at the previous ID + // and it actually satisfies the constraint + int prev = (d==ASC?LOWER:HIGHER).apply(idOnDisk.find(getIdOf(r))); + if (idOnDisk.isInRange(prev)) { + R pr = getById(idOnDisk.get(prev)); + // sign*sign is making sure that #pr and #r sandwiches #n. + if (pr!=null && signOfCompare(getNumberOf(pr),n)*signOfCompare(n,getNumberOf(r))<0) + return r; + else { + // cache is lying. there's something fishy. + // ignore the cache and do the slow search + } + } else { + // r is the build with youngest ID + return r; + } + } else { + // cache says we should have a build but we didn't. + // ignore the cache and do the slow search + } + } + break; + case EXACT: + // fall through + } + + // didn't find it in the cache, but don't give up yet + // maybe the cache just doesn't exist. + // so fall back to the slow search + } // capture the snapshot and work off with it since it can be changed by other threads SortedStringList idOnDisk = this.idOnDisk; @@ -152,7 +217,7 @@ public abstract class Attempt2 extends AbstractMap implements Sort pivot = (lo+hi)/2; if (hi<=lo) break; // end of search - R r = load(new File(dir, idOnDisk.get(pivot)), true); + R r = load(idOnDisk.get(pivot), true); if (r==null) { // this ID isn't valid. get rid of that and retry pivot hi--; @@ -173,10 +238,10 @@ public abstract class Attempt2 extends AbstractMap implements Sort switch (d) { case ASC: if (hi==idOnDisk.size()) return null; - return byId.get(idOnDisk.get(hi)); + return getById(idOnDisk.get(hi)); case DESC: if (lo<=0) return null; - return byId.get(idOnDisk.get(lo-1)); + return getById(idOnDisk.get(lo-1)); case EXACT: return null; default: @@ -184,6 +249,25 @@ public abstract class Attempt2 extends AbstractMap implements Sort } } + /** + * sign of (a-b). + */ + private static int signOfCompare(int a, int b) { + if (a>b) return 1; + if (a f, int fallback) { // if (f==null) return fallback; // return getNumberOf(f.getValue()); @@ -206,7 +290,7 @@ public abstract class Attempt2 extends AbstractMap implements Sort copy(); for (String id : idOnDisk) { if (!byId.containsKey(id)) - load(new File(dir,id),false); // copy() called above, so no need to copy inside + load(id,false); // copy() called above, so no need to copy inside } fullyLoaded = true; } @@ -233,6 +317,12 @@ public abstract class Attempt2 extends AbstractMap implements Sort if (shortcut.isDirectory()) { synchronized (loadLock) { r = load(shortcut,copy); + + // make sure what we actually loaded is #n, + // because the shortcuts can lie. + if (r!=null && getNumberOf(r)!=n) + r = null; + if (r==null) { // if failed to locate, record that fact if (copy) copy(); @@ -243,6 +333,11 @@ public abstract class Attempt2 extends AbstractMap implements Sort return r; } + + protected R load(String id, boolean copy) { + return load(new File(dir,id),copy); + } + protected R load(File dataDir, boolean copy) { synchronized (loadLock) { try { @@ -306,4 +401,6 @@ public abstract class Attempt2 extends AbstractMap implements Sort return -n-1+binarySearchTrimOffset; } } + + private static final String[] EMPTY_STRING_ARRAY = new String[0]; } diff --git a/src/main/java/ll/SortedIntList.java b/src/main/java/ll/SortedIntList.java index 2315aba82e..5af08df3d9 100644 --- a/src/main/java/ll/SortedIntList.java +++ b/src/main/java/ll/SortedIntList.java @@ -88,4 +88,7 @@ public class SortedIntList extends AbstractList { return Boundary.CEIL.apply(find(v)); } + public boolean isInRange(int idx) { + return 0 { int sz = data.size(); return safeSubList(0,sz/2); } + + public boolean isInRange(int idx) { + return 0 Date: Sat, 15 Sep 2012 12:29:10 -0700 Subject: [PATCH 12/69] adding more tests and fixing bugs as they are discovered --- src/main/java/ll/Attempt2.java | 19 ++++-- src/main/java/ll/SortedIntList.java | 23 +++++++- src/main/java/ll/SortedStringList.java | 2 +- src/test/java/ll/Attempt2Test.java | 67 +++++++++++++++++++++- src/test/java/ll/FakeMap.java | 3 +- src/test/java/ll/FakeMapBuilder.java | 45 ++++++++++++++- src/test/java/ll/SortedStringListTest.java | 10 ++++ 7 files changed, 155 insertions(+), 14 deletions(-) diff --git a/src/main/java/ll/Attempt2.java b/src/main/java/ll/Attempt2.java index 682ac06c93..b12b48535e 100644 --- a/src/main/java/ll/Attempt2.java +++ b/src/main/java/ll/Attempt2.java @@ -14,7 +14,7 @@ import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; -import static ll.Attempt2.Direction.ASC; +import static ll.Attempt2.Direction.*; import static ll.Boundary.*; /** @@ -28,7 +28,11 @@ import static ll.Boundary.*; public abstract class Attempt2 extends AbstractMap implements SortedMap { private boolean fullyLoaded; - // copy on write map that stores what's already loaded + + /** + * Stores the mapping from build number to build, for builds that are already loaded. + */ + // copy on write map private volatile TreeMap byNumber = new TreeMap(); /** @@ -65,6 +69,7 @@ public abstract class Attempt2 extends AbstractMap implements Sort String[] buildDirs = dir.list(createDirectoryFilter()); if (buildDirs==null) buildDirs=EMPTY_STRING_ARRAY; // wrap into ArrayList to enable mutation + Arrays.sort(buildDirs); idOnDisk = new SortedStringList(new ArrayList(Arrays.asList(buildDirs))); // TODO: should we check that shortcuts is a symlink? @@ -78,6 +83,7 @@ public abstract class Attempt2 extends AbstractMap implements Sort // this isn't a shortcut } } + list.sort(); numberOnDisk = list; } @@ -121,7 +127,7 @@ public abstract class Attempt2 extends AbstractMap implements Sort public R get(Object key) { if (key instanceof Integer) { int n = (Integer) key; - return search(n, Direction.EXACT); + return get(n); } return super.get(key); } @@ -169,7 +175,7 @@ public abstract class Attempt2 extends AbstractMap implements Sort if (idOnDisk.isInRange(prev)) { R pr = getById(idOnDisk.get(prev)); // sign*sign is making sure that #pr and #r sandwiches #n. - if (pr!=null && signOfCompare(getNumberOf(pr),n)*signOfCompare(n,getNumberOf(r))<0) + if (pr!=null && signOfCompare(getNumberOf(pr),n)*signOfCompare(n,getNumberOf(r))>0) return r; else { // cache is lying. there's something fishy. @@ -325,8 +331,9 @@ public abstract class Attempt2 extends AbstractMap implements Sort if (r==null) { // if failed to locate, record that fact - if (copy) copy(); - byNumber.put(n,null); + SortedIntList update = new SortedIntList(numberOnDisk); + update.removeValue(n); + numberOnDisk = update; } } } diff --git a/src/main/java/ll/SortedIntList.java b/src/main/java/ll/SortedIntList.java index 5af08df3d9..5bcaf7d5b0 100644 --- a/src/main/java/ll/SortedIntList.java +++ b/src/main/java/ll/SortedIntList.java @@ -19,6 +19,14 @@ public class SortedIntList extends AbstractList { this(16); } + /** + * Internal copy constructor. + */ + public SortedIntList(SortedIntList rhs) { + this.data = Arrays.copyOf(rhs.data,rhs.data.length); + this.size = rhs.size; + } + /** * Binary search to find the position of the given string. * @@ -54,7 +62,7 @@ public class SortedIntList extends AbstractList { private void ensureCapacity(int i) { if (data.length { } public boolean isInRange(int idx) { - return 0 { } public boolean isInRange(int idx) { - return 0 { @Override protected Build retrieve(File dir) throws IOException { String n = FileUtils.readFileToString(new File(dir, "n")).trim(); - return new Build(Integer.parseInt(n),dir.getName()); + String id = FileUtils.readFileToString(new File(dir, "id")).trim(); + return new Build(Integer.parseInt(n),id); } } diff --git a/src/test/java/ll/FakeMapBuilder.java b/src/test/java/ll/FakeMapBuilder.java index a928f7f207..a8608403f4 100644 --- a/src/test/java/ll/FakeMapBuilder.java +++ b/src/test/java/ll/FakeMapBuilder.java @@ -9,6 +9,8 @@ import java.io.File; import java.io.IOException; /** + * Builder for creating a {@link FakeMap} + * * @author Kohsuke Kawaguchi */ public class FakeMapBuilder implements TestRule { @@ -18,21 +20,58 @@ public class FakeMapBuilder implements TestRule { } public FakeMapBuilder add(int n, String id) throws IOException { + verifyId(id); + File build = new File(dir,id); + build.mkdir(); + FileUtils.write(new File(build, "n"), Integer.toString(n)); + FileUtils.write(new File(build,"id"),id); + return this; + } + + /** + * Adds a symlink from n to build id. + * + * (in test we should ideally create a symlink, but we fake the test + * by actually making it a directory and staging the same data.) + */ + public FakeMapBuilder addCache(int n, String id) throws IOException { + return addBogusCache(n,n,id); + } + + public FakeMapBuilder addBogusCache(int label, int actual, String id) throws IOException { + verifyId(id); + File build = new File(dir,Integer.toString(label)); + build.mkdir(); + FileUtils.write(new File(build, "n"), Integer.toString(actual)); + FileUtils.write(new File(build,"id"),id); + return this; + } + + public FakeMapBuilder addBoth(int n, String id) throws IOException { + return add(n,id).addCache(n,id); + } + + private void verifyId(String id) { try { Integer.parseInt(id); throw new IllegalMonitorStateException("ID cannot be a number"); } catch (NumberFormatException e) { // OK } + } + /** + * Adds a build record under the givn ID but make it unloadable, + * which will cause a failure when a load is attempted on this build ID. + */ + public FakeMapBuilder addUnloadable(String id) throws IOException { File build = new File(dir,id); build.mkdir(); - FileUtils.write(new File(build,"n"),Integer.toString(n)); return this; } - public FakeMapBuilder addUnloadable(String id) throws IOException { - File build = new File(dir,id); + public FakeMapBuilder addUnloadableCache(int n) throws IOException { + File build = new File(dir,String.valueOf(n)); build.mkdir(); return this; } diff --git a/src/test/java/ll/SortedStringListTest.java b/src/test/java/ll/SortedStringListTest.java index 1775297d0c..de661bedcd 100644 --- a/src/test/java/ll/SortedStringListTest.java +++ b/src/test/java/ll/SortedStringListTest.java @@ -55,4 +55,14 @@ public class SortedStringListTest extends Assert { assertEquals(3,l.higher("F")); assertEquals(3,l.higher("G")); } + + @Test + public void testRange() { + assertTrue(l.isInRange(0)); + assertTrue(l.isInRange(1)); + assertTrue(l.isInRange(2)); + + assertFalse(l.isInRange(-1)); + assertFalse(l.isInRange(3)); + } } From d5f69c7643e1c48e98791d8eaa336af180f5d87b Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sat, 15 Sep 2012 12:30:53 -0700 Subject: [PATCH 13/69] recording TODOs --- note.txt | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/note.txt b/note.txt index 686e45ec18..5fd75f669d 100644 --- a/note.txt +++ b/note.txt @@ -1,6 +1,3 @@ - -- Perhaps better to create two independent implementations, - one that relies on symlinks (thus mapping from N->R), the other - that uses ID->R mapping. - - It's not clear how we can mix two \ No newline at end of file +TODO: + - remove Java6 dependency (or do we really?) + - ship it! \ No newline at end of file From 35e71ade55054993f67ab676403f46c9ebd54cc9 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sat, 15 Sep 2012 12:35:25 -0700 Subject: [PATCH 14/69] Removing unused code --- src/main/java/ll/AbstractSortedMap.java | 117 --------------------- src/main/java/ll/Attempt2.java | 2 +- src/main/java/ll/SortedIntList.java | 4 - src/main/java/ll/SortedStringList.java | 47 --------- src/test/java/ll/SortedStringListTest.java | 14 ++- 5 files changed, 14 insertions(+), 170 deletions(-) delete mode 100644 src/main/java/ll/AbstractSortedMap.java diff --git a/src/main/java/ll/AbstractSortedMap.java b/src/main/java/ll/AbstractSortedMap.java deleted file mode 100644 index bc5b001f3d..0000000000 --- a/src/main/java/ll/AbstractSortedMap.java +++ /dev/null @@ -1,117 +0,0 @@ -package ll; - -import java.util.AbstractMap; -import java.util.Comparator; -import java.util.Map; -import java.util.NavigableMap; -import java.util.NavigableSet; -import java.util.Set; -import java.util.SortedMap; -import java.util.SortedSet; - -/** - * Implementing AbstractSortedMap really ends up implementing AbstractNavigableMap - * because SortedMap isn't really self contained / closed to itself. - * - * @author Kohsuke Kawaguchi - */ -public abstract class AbstractSortedMap extends AbstractMap implements SortedMap, NavigableMap { - @Override - public abstract NavigableSet> entrySet(); - - public K lowerKey(K key) { - return key(lowerEntry(key)); - } - - public K floorKey(K key) { - return key(floorEntry(key)); - } - - public K ceilingKey(K key) { - return key(ceilingEntry(key)); - } - - public K higherKey(K key) { - return key(higherEntry(key)); - } - - public K firstKey() { - return key(firstEntry()); - } - - public K lastKey() { - return key(lastEntry()); - } - - private K key(Entry e) { - return e!=null ? e.getKey() : null; - } - - public NavigableMap subMap(K fromKey, K toKey) { - return subMap(fromKey,true,toKey,false); - } - - public SortedMap headMap(K toKey) { - return headMap(toKey,false); - } - - public SortedMap tailMap(K fromKey) { - return tailMap(fromKey,true); - } - - public Entry pollFirstEntry() { - return remove(copy(firstEntry())); - } - - public Entry pollLastEntry() { - return remove(copy(lastEntry())); - } - - public NavigableSet navigableKeySet() { - // TODO - throw new UnsupportedOperationException(); - } - - private Entry remove(Entry e) { - if (e != null) - remove(e.getKey()); - return e; - } - - protected Entry copy(Entry e) { - return e!=null ? new SimpleEntry(e.getKey(),e.getValue()) : null; - } - - - -// public NavigableMap subMap(K fromKey, boolean fromInclusive, K toKey, boolean toInclusive) { -// final AbstractSortedMap back = this; -// return new AbstractSortedMap() { -// @Override -// public NavigableSet> entrySet() { -// return back.entrySet().subSet(fromKey, toKey); -// } -// -// public Comparator comparator() { -// // TODO -// throw new UnsupportedOperationException(); -// } -// -// public K firstKey() { -// // TODO -// throw new UnsupportedOperationException(); -// } -// -// public K lastKey() { -// // TODO -// throw new UnsupportedOperationException(); -// } -// }; -// } - - final int compare(Object k1, Object k2) { - Comparator c = comparator(); - if (c==null) return ((Comparable)k1).compareTo(k2); - else return c.compare(k1,k2); - } -} diff --git a/src/main/java/ll/Attempt2.java b/src/main/java/ll/Attempt2.java index b12b48535e..b04398758c 100644 --- a/src/main/java/ll/Attempt2.java +++ b/src/main/java/ll/Attempt2.java @@ -18,7 +18,7 @@ import static ll.Attempt2.Direction.*; import static ll.Boundary.*; /** - * After {@link AbstractSortedMap} is abandoned, this one is more crude implementation. + * Lazy-loading map of build records. * * This implementation is in 2 states. An instance can be {@linkplain #fullyLoaded fully loaded} state, * where everything that can be loaded gets loaded. Otherwise it's in a partially loaded state. diff --git a/src/main/java/ll/SortedIntList.java b/src/main/java/ll/SortedIntList.java index 5bcaf7d5b0..a830780dcd 100644 --- a/src/main/java/ll/SortedIntList.java +++ b/src/main/java/ll/SortedIntList.java @@ -15,10 +15,6 @@ public class SortedIntList extends AbstractList { this.size = 0; } - public SortedIntList() { - this(16); - } - /** * Internal copy constructor. */ diff --git a/src/main/java/ll/SortedStringList.java b/src/main/java/ll/SortedStringList.java index fde9e8856e..c205a025dd 100644 --- a/src/main/java/ll/SortedStringList.java +++ b/src/main/java/ll/SortedStringList.java @@ -1,10 +1,8 @@ package ll; import java.util.AbstractList; -import java.util.AbstractMap.SimpleEntry; import java.util.Collections; import java.util.List; -import java.util.Map.Entry; /** * @author Kohsuke Kawaguchi @@ -74,51 +72,6 @@ public class SortedStringList extends AbstractList { return Boundary.CEIL.apply(find(v)); } - /** - * Smarter {@link #subList(int, int)} that gracefully handles index that's out of range. - */ - public SortedStringList safeSubList(int from, int to) { - return new SortedStringList(subList(inside(from),inside(to))); - } - - /** - * Brings the index into the legal range. - */ - private int inside(int idx) { - idx = Math.max(0,idx); - idx = Math.min(idx,size()); - return idx; - } - - /** - * Finds the middle value used for binary search pivot. - */ - public Entry middle() { - assert !data.isEmpty(); - int idx = data.size()/2; - return new SimpleEntry(idx,data.get(idx)); - } - - /** - * Overloaded version of {@link #remove(int)} that takes a pointer instead of the index. - */ - public void remove(Entry ptr) { - remove(ptr.getKey().intValue()); - } - - /** - * Returns the sub list that contains (pivot,end] - */ - public SortedStringList upperHalf() { - int sz = data.size(); - return safeSubList(sz/2+1,sz); - } - - public SortedStringList lowerHalf() { - int sz = data.size(); - return safeSubList(0,sz/2); - } - public boolean isInRange(int idx) { return 0<=idx && idx(Arrays.asList("B","D","F"))); @Test public void testCeil() { @@ -65,4 +66,15 @@ public class SortedStringListTest extends Assert { assertFalse(l.isInRange(-1)); assertFalse(l.isInRange(3)); } + + @Test + public void remove() { + l.remove("nosuchthing"); + assertEquals(3,l.size()); + + l.remove("B"); + assertEquals(2, l.size()); + assertEquals("D",l.get(0)); + assertEquals("F",l.get(1)); + } } From 8b822ce2be96bc0b68851352b78ae147f2618c84 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sat, 15 Sep 2012 12:47:24 -0700 Subject: [PATCH 15/69] added fast submap implementation --- src/main/java/ll/Attempt2.java | 22 ++++++++++++++++------ src/test/java/ll/Attempt2Test.java | 12 ++++++++++++ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/main/java/ll/Attempt2.java b/src/main/java/ll/Attempt2.java index b04398758c..0c6026f64a 100644 --- a/src/main/java/ll/Attempt2.java +++ b/src/main/java/ll/Attempt2.java @@ -97,18 +97,26 @@ public abstract class Attempt2 extends AbstractMap implements Sort } public SortedMap subMap(Integer fromKey, Integer toKey) { - // TODO: maybe we do want efficient implementation here - return all().subMap(fromKey,toKey); + R start = search(fromKey, ASC); + if (start==null) return EMPTY_SORTED_MAP; + + R end = search(toKey-1, DESC); + if (end==null) return EMPTY_SORTED_MAP; + + for (R i=start; i!=end; ) { + i = search(getNumberOf(i)+1,ASC); + assert i!=null; + } + + return byNumber.subMap(fromKey,toKey); } public SortedMap headMap(Integer toKey) { - // TODO: maybe we do want efficient implementation here - return all().headMap(toKey); + return subMap(Integer.MIN_VALUE,toKey); } public SortedMap tailMap(Integer fromKey) { - // TODO: maybe we do want efficient implementation here - return all().tailMap(fromKey); + return subMap(fromKey,Integer.MAX_VALUE); } public Integer firstKey() { @@ -410,4 +418,6 @@ public abstract class Attempt2 extends AbstractMap implements Sort } private static final String[] EMPTY_STRING_ARRAY = new String[0]; + + private static final SortedMap EMPTY_SORTED_MAP = Collections.unmodifiableSortedMap(new TreeMap()); } diff --git a/src/test/java/ll/Attempt2Test.java b/src/test/java/ll/Attempt2Test.java index 478a8b4c2a..50d42e8e9e 100644 --- a/src/test/java/ll/Attempt2Test.java +++ b/src/test/java/ll/Attempt2Test.java @@ -9,6 +9,7 @@ import org.junit.Test; import java.io.IOException; import java.util.Map; import java.util.NoSuchElementException; +import java.util.SortedMap; /** * @author Kohsuke Kawaguchi @@ -174,4 +175,15 @@ public class Attempt2Test extends Assert { assertNull(setup.make().search(0, Direction.DESC)); assertNull(setup.make().search(6, Direction.ASC)); } + + @Test + public void fastSubMap() throws Exception { + SortedMap m = a.subMap(2, 99); + assertEquals(2, m.size()); + + Build[] b = m.values().toArray(new Build[2]); + assertEquals(2,b.length); + b[0].asserts(3,"B"); + b[1].asserts(5,"C"); + } } From e290de6a0e50c10134b4738990413eb3850b7ab7 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sat, 15 Sep 2012 12:50:18 -0700 Subject: [PATCH 16/69] improving coverage --- src/main/java/ll/Attempt2.java | 21 +-------------------- src/test/java/ll/Attempt2Test.java | 8 ++++++++ 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/src/main/java/ll/Attempt2.java b/src/main/java/ll/Attempt2.java index 0c6026f64a..772127da45 100644 --- a/src/main/java/ll/Attempt2.java +++ b/src/main/java/ll/Attempt2.java @@ -395,26 +395,7 @@ public abstract class Attempt2 extends AbstractMap implements Sort }; enum Direction { - ASC(0), DESC(-1), EXACT(0/*invalid*/); - - private final int binarySearchTrimOffset; - - Direction(int binarySearchTrimOffset) { - this.binarySearchTrimOffset = binarySearchTrimOffset; - } - - public int offset() { - return binarySearchTrimOffset; - } - - /** - * Given the negative result from {@link Collections#binarySearch(List, Object)} - * that indicates no-exact match, determine the position of the element we'll load - * instead that matches with the search order. - */ - private int candidate(int n) { - return -n-1+binarySearchTrimOffset; - } + ASC, DESC, EXACT } private static final String[] EMPTY_STRING_ARRAY = new String[0]; diff --git a/src/test/java/ll/Attempt2Test.java b/src/test/java/ll/Attempt2Test.java index 50d42e8e9e..5b17472b88 100644 --- a/src/test/java/ll/Attempt2Test.java +++ b/src/test/java/ll/Attempt2Test.java @@ -186,4 +186,12 @@ public class Attempt2Test extends Assert { b[0].asserts(3,"B"); b[1].asserts(5,"C"); } + + @Test + public void identity() { + assertTrue(a.equals(a)); + assertTrue(!a.equals(b)); + a.hashCode(); + b.hashCode(); + } } From 00b0ebcdf8690fe23febd1217a51302c8bc95bce Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 17 Sep 2012 14:18:23 -0700 Subject: [PATCH 17/69] moving into positions to be merged into the mainline --- .../java/jenkins/model/lazy}/Attempt2.java | 0 .../java/jenkins/model/lazy}/Boundary.java | 0 .../jenkins/model/lazy}/SortedIntList.java | 0 .../jenkins/model/lazy}/SortedStringList.java | 0 .../jenkins/model/lazy}/Attempt2Test.java | 0 .../java/jenkins/model/lazy}/FakeMap.java | 0 .../jenkins/model/lazy}/FakeMapBuilder.java | 0 .../model/lazy}/SortedStringListTest.java | 0 pom.xml | 22 ------------------- 9 files changed, 22 deletions(-) rename {src/main/java/ll => core/src/main/java/jenkins/model/lazy}/Attempt2.java (100%) rename {src/main/java/ll => core/src/main/java/jenkins/model/lazy}/Boundary.java (100%) rename {src/main/java/ll => core/src/main/java/jenkins/model/lazy}/SortedIntList.java (100%) rename {src/main/java/ll => core/src/main/java/jenkins/model/lazy}/SortedStringList.java (100%) rename {src/test/java/ll => core/src/test/java/jenkins/model/lazy}/Attempt2Test.java (100%) rename {src/test/java/ll => core/src/test/java/jenkins/model/lazy}/FakeMap.java (100%) rename {src/test/java/ll => core/src/test/java/jenkins/model/lazy}/FakeMapBuilder.java (100%) rename {src/test/java/ll => core/src/test/java/jenkins/model/lazy}/SortedStringListTest.java (100%) delete mode 100644 pom.xml diff --git a/src/main/java/ll/Attempt2.java b/core/src/main/java/jenkins/model/lazy/Attempt2.java similarity index 100% rename from src/main/java/ll/Attempt2.java rename to core/src/main/java/jenkins/model/lazy/Attempt2.java diff --git a/src/main/java/ll/Boundary.java b/core/src/main/java/jenkins/model/lazy/Boundary.java similarity index 100% rename from src/main/java/ll/Boundary.java rename to core/src/main/java/jenkins/model/lazy/Boundary.java diff --git a/src/main/java/ll/SortedIntList.java b/core/src/main/java/jenkins/model/lazy/SortedIntList.java similarity index 100% rename from src/main/java/ll/SortedIntList.java rename to core/src/main/java/jenkins/model/lazy/SortedIntList.java diff --git a/src/main/java/ll/SortedStringList.java b/core/src/main/java/jenkins/model/lazy/SortedStringList.java similarity index 100% rename from src/main/java/ll/SortedStringList.java rename to core/src/main/java/jenkins/model/lazy/SortedStringList.java diff --git a/src/test/java/ll/Attempt2Test.java b/core/src/test/java/jenkins/model/lazy/Attempt2Test.java similarity index 100% rename from src/test/java/ll/Attempt2Test.java rename to core/src/test/java/jenkins/model/lazy/Attempt2Test.java diff --git a/src/test/java/ll/FakeMap.java b/core/src/test/java/jenkins/model/lazy/FakeMap.java similarity index 100% rename from src/test/java/ll/FakeMap.java rename to core/src/test/java/jenkins/model/lazy/FakeMap.java diff --git a/src/test/java/ll/FakeMapBuilder.java b/core/src/test/java/jenkins/model/lazy/FakeMapBuilder.java similarity index 100% rename from src/test/java/ll/FakeMapBuilder.java rename to core/src/test/java/jenkins/model/lazy/FakeMapBuilder.java diff --git a/src/test/java/ll/SortedStringListTest.java b/core/src/test/java/jenkins/model/lazy/SortedStringListTest.java similarity index 100% rename from src/test/java/ll/SortedStringListTest.java rename to core/src/test/java/jenkins/model/lazy/SortedStringListTest.java diff --git a/pom.xml b/pom.xml deleted file mode 100644 index 2be2d89bbd..0000000000 --- a/pom.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - 4.0.0 - test - lazy-load - 1.1-SNAPSHOT - - - - commons-io - commons-io - 2.0.1 - - - junit - junit - 4.9 - - - - - From 580a463ab81996c8487f42c6a8a6ad94e9dad095 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 17 Sep 2012 14:28:30 -0700 Subject: [PATCH 18/69] Renaming packages --- core/src/main/java/jenkins/model/lazy/Attempt2.java | 9 ++++----- core/src/main/java/jenkins/model/lazy/Boundary.java | 2 +- .../main/java/jenkins/model/lazy/SortedIntList.java | 2 +- .../main/java/jenkins/model/lazy/SortedStringList.java | 2 +- .../src/test/java/jenkins/model/lazy/Attempt2Test.java | 6 +++--- core/src/test/java/jenkins/model/lazy/FakeMap.java | 2 +- .../test/java/jenkins/model/lazy/FakeMapBuilder.java | 10 +++++----- .../java/jenkins/model/lazy/SortedStringListTest.java | 3 +-- 8 files changed, 17 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/jenkins/model/lazy/Attempt2.java b/core/src/main/java/jenkins/model/lazy/Attempt2.java index 772127da45..0d81d0ddb2 100644 --- a/core/src/main/java/jenkins/model/lazy/Attempt2.java +++ b/core/src/main/java/jenkins/model/lazy/Attempt2.java @@ -1,4 +1,4 @@ -package ll; +package jenkins.model.lazy; import java.io.File; import java.io.FilenameFilter; @@ -8,14 +8,13 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; -import java.util.List; import java.util.NoSuchElementException; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; -import static ll.Attempt2.Direction.*; -import static ll.Boundary.*; +import static jenkins.model.lazy.Attempt2.Direction.*; +import static jenkins.model.lazy.Boundary.*; /** * Lazy-loading map of build records. @@ -126,7 +125,7 @@ public abstract class Attempt2 extends AbstractMap implements Sort } public Integer lastKey() { - R r = search(Integer.MAX_VALUE, Direction.DESC); + R r = search(Integer.MAX_VALUE, DESC); if (r==null) throw new NoSuchElementException(); return getNumberOf(r); } diff --git a/core/src/main/java/jenkins/model/lazy/Boundary.java b/core/src/main/java/jenkins/model/lazy/Boundary.java index b3c7968dcf..93560e0315 100644 --- a/core/src/main/java/jenkins/model/lazy/Boundary.java +++ b/core/src/main/java/jenkins/model/lazy/Boundary.java @@ -1,4 +1,4 @@ -package ll; +package jenkins.model.lazy; /** * ceil/floor/lower/higher implementations diff --git a/core/src/main/java/jenkins/model/lazy/SortedIntList.java b/core/src/main/java/jenkins/model/lazy/SortedIntList.java index a830780dcd..3dbb50a7c1 100644 --- a/core/src/main/java/jenkins/model/lazy/SortedIntList.java +++ b/core/src/main/java/jenkins/model/lazy/SortedIntList.java @@ -1,4 +1,4 @@ -package ll; +package jenkins.model.lazy; import java.util.AbstractList; import java.util.Arrays; diff --git a/core/src/main/java/jenkins/model/lazy/SortedStringList.java b/core/src/main/java/jenkins/model/lazy/SortedStringList.java index c205a025dd..c62e475078 100644 --- a/core/src/main/java/jenkins/model/lazy/SortedStringList.java +++ b/core/src/main/java/jenkins/model/lazy/SortedStringList.java @@ -1,4 +1,4 @@ -package ll; +package jenkins.model.lazy; import java.util.AbstractList; import java.util.Collections; diff --git a/core/src/test/java/jenkins/model/lazy/Attempt2Test.java b/core/src/test/java/jenkins/model/lazy/Attempt2Test.java index 5b17472b88..9a7e2f6f06 100644 --- a/core/src/test/java/jenkins/model/lazy/Attempt2Test.java +++ b/core/src/test/java/jenkins/model/lazy/Attempt2Test.java @@ -1,6 +1,6 @@ -package ll; +package jenkins.model.lazy; -import ll.Attempt2.Direction; +import jenkins.model.lazy.Attempt2.Direction; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -102,7 +102,7 @@ public class Attempt2Test extends Assert { assertNull(m.search(3,Direction.EXACT)); m.search(3,Direction.DESC).asserts(1,"A"); - m.search(3,Direction.ASC ).asserts(5,"C"); + m.search(3, Direction.ASC ).asserts(5,"C"); } @Test diff --git a/core/src/test/java/jenkins/model/lazy/FakeMap.java b/core/src/test/java/jenkins/model/lazy/FakeMap.java index 6900ba3ea7..839ceb54be 100644 --- a/core/src/test/java/jenkins/model/lazy/FakeMap.java +++ b/core/src/test/java/jenkins/model/lazy/FakeMap.java @@ -1,4 +1,4 @@ -package ll; +package jenkins.model.lazy; import org.apache.commons.io.FileUtils; diff --git a/core/src/test/java/jenkins/model/lazy/FakeMapBuilder.java b/core/src/test/java/jenkins/model/lazy/FakeMapBuilder.java index a8608403f4..20ab49c095 100644 --- a/core/src/test/java/jenkins/model/lazy/FakeMapBuilder.java +++ b/core/src/test/java/jenkins/model/lazy/FakeMapBuilder.java @@ -1,4 +1,4 @@ -package ll; +package jenkins.model.lazy; import org.apache.commons.io.FileUtils; import org.junit.rules.TestRule; @@ -23,8 +23,8 @@ public class FakeMapBuilder implements TestRule { verifyId(id); File build = new File(dir,id); build.mkdir(); - FileUtils.write(new File(build, "n"), Integer.toString(n)); - FileUtils.write(new File(build,"id"),id); + FileUtils.writeStringToFile(new File(build, "n"), Integer.toString(n)); + FileUtils.writeStringToFile(new File(build,"id"),id); return this; } @@ -42,8 +42,8 @@ public class FakeMapBuilder implements TestRule { verifyId(id); File build = new File(dir,Integer.toString(label)); build.mkdir(); - FileUtils.write(new File(build, "n"), Integer.toString(actual)); - FileUtils.write(new File(build,"id"),id); + FileUtils.writeStringToFile(new File(build, "n"), Integer.toString(actual)); + FileUtils.writeStringToFile(new File(build,"id"),id); return this; } diff --git a/core/src/test/java/jenkins/model/lazy/SortedStringListTest.java b/core/src/test/java/jenkins/model/lazy/SortedStringListTest.java index d836ee3995..5ac482001c 100644 --- a/core/src/test/java/jenkins/model/lazy/SortedStringListTest.java +++ b/core/src/test/java/jenkins/model/lazy/SortedStringListTest.java @@ -1,6 +1,5 @@ -package ll; +package jenkins.model.lazy; -import ll.SortedStringList; import org.junit.Assert; import org.junit.Test; From bc1fdc170eb7131c885d5ddd118435dc9e84d061 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 17 Sep 2012 14:53:10 -0700 Subject: [PATCH 19/69] minor adjustments - doc improvements - SortedStringList actually isn't String specific --- ...empt2.java => AbstractLazyLoadRunMap.java} | 35 +++++- .../java/jenkins/model/lazy/Boundary.java | 25 ++++- .../jenkins/model/lazy/SortedIntList.java | 29 ++++- .../java/jenkins/model/lazy/SortedList.java | 104 ++++++++++++++++++ .../jenkins/model/lazy/SortedStringList.java | 78 ------------- ...t.java => AbstractLazyLoadRunMapTest.java} | 27 ++++- .../test/java/jenkins/model/lazy/FakeMap.java | 25 ++++- .../jenkins/model/lazy/FakeMapBuilder.java | 23 ++++ ...tringListTest.java => SortedListTest.java} | 27 ++++- 9 files changed, 282 insertions(+), 91 deletions(-) rename core/src/main/java/jenkins/model/lazy/{Attempt2.java => AbstractLazyLoadRunMap.java} (88%) create mode 100644 core/src/main/java/jenkins/model/lazy/SortedList.java delete mode 100644 core/src/main/java/jenkins/model/lazy/SortedStringList.java rename core/src/test/java/jenkins/model/lazy/{Attempt2Test.java => AbstractLazyLoadRunMapTest.java} (81%) rename core/src/test/java/jenkins/model/lazy/{SortedStringListTest.java => SortedListTest.java} (59%) diff --git a/core/src/main/java/jenkins/model/lazy/Attempt2.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java similarity index 88% rename from core/src/main/java/jenkins/model/lazy/Attempt2.java rename to core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index 0d81d0ddb2..8a5485500a 100644 --- a/core/src/main/java/jenkins/model/lazy/Attempt2.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -1,3 +1,26 @@ +/* + * The MIT License + * + * Copyright (c) 2012, CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package jenkins.model.lazy; import java.io.File; @@ -13,7 +36,7 @@ import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; -import static jenkins.model.lazy.Attempt2.Direction.*; +import static jenkins.model.lazy.AbstractLazyLoadRunMap.Direction.*; import static jenkins.model.lazy.Boundary.*; /** @@ -24,7 +47,7 @@ import static jenkins.model.lazy.Boundary.*; * * @author Kohsuke Kawaguchi */ -public abstract class Attempt2 extends AbstractMap implements SortedMap { +public abstract class AbstractLazyLoadRunMap extends AbstractMap implements SortedMap { private boolean fullyLoaded; @@ -43,7 +66,7 @@ public abstract class Attempt2 extends AbstractMap implements Sort * Build IDs found as directories, in the ascending order. */ // copy on write - private SortedStringList idOnDisk; + private SortedList idOnDisk; /** * Build bumber shortcuts found on disk, in the ascending order. @@ -59,7 +82,7 @@ public abstract class Attempt2 extends AbstractMap implements Sort */ private final Object loadLock = this; - public Attempt2(File dir) { + public AbstractLazyLoadRunMap(File dir) { this.dir = dir; loadIdOnDisk(); } @@ -69,7 +92,7 @@ public abstract class Attempt2 extends AbstractMap implements Sort if (buildDirs==null) buildDirs=EMPTY_STRING_ARRAY; // wrap into ArrayList to enable mutation Arrays.sort(buildDirs); - idOnDisk = new SortedStringList(new ArrayList(Arrays.asList(buildDirs))); + idOnDisk = new SortedList(new ArrayList(Arrays.asList(buildDirs))); // TODO: should we check that shortcuts is a symlink? String[] shortcuts = dir.list(); @@ -208,7 +231,7 @@ public abstract class Attempt2 extends AbstractMap implements Sort } // capture the snapshot and work off with it since it can be changed by other threads - SortedStringList idOnDisk = this.idOnDisk; + SortedList idOnDisk = this.idOnDisk; // slow path: we have to find the build from idOnDisk. // first, narrow down the candidate IDs to try by using two known number-to-ID mapping diff --git a/core/src/main/java/jenkins/model/lazy/Boundary.java b/core/src/main/java/jenkins/model/lazy/Boundary.java index 93560e0315..e4625b4cbd 100644 --- a/core/src/main/java/jenkins/model/lazy/Boundary.java +++ b/core/src/main/java/jenkins/model/lazy/Boundary.java @@ -1,3 +1,26 @@ +/* + * The MIT License + * + * Copyright (c) 2012, CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package jenkins.model.lazy; /** @@ -6,7 +29,7 @@ package jenkins.model.lazy; * * @author Kohsuke Kawaguchi */ -public enum Boundary { +enum Boundary { LOWER(-1,-1), HIGHER(1,0), FLOOR(0,-1), diff --git a/core/src/main/java/jenkins/model/lazy/SortedIntList.java b/core/src/main/java/jenkins/model/lazy/SortedIntList.java index 3dbb50a7c1..12c7fc61a4 100644 --- a/core/src/main/java/jenkins/model/lazy/SortedIntList.java +++ b/core/src/main/java/jenkins/model/lazy/SortedIntList.java @@ -1,12 +1,39 @@ +/* + * The MIT License + * + * Copyright (c) 2012, CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package jenkins.model.lazy; import java.util.AbstractList; import java.util.Arrays; /** + * {@code ArrayList<Integer>} that uses {@code int} for storage. + * + * Plus a number of binary-search related methods that assume the array is sorted in the ascending order. + * * @author Kohsuke Kawaguchi */ -public class SortedIntList extends AbstractList { +class SortedIntList extends AbstractList { private int[] data; private int size; diff --git a/core/src/main/java/jenkins/model/lazy/SortedList.java b/core/src/main/java/jenkins/model/lazy/SortedList.java new file mode 100644 index 0000000000..9c66d9dd8a --- /dev/null +++ b/core/src/main/java/jenkins/model/lazy/SortedList.java @@ -0,0 +1,104 @@ +/* + * The MIT License + * + * Copyright (c) 2012, CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package jenkins.model.lazy; + +import java.util.AbstractList; +import java.util.Collections; +import java.util.List; + +/** + * {@link List} decorator that provides a number of binary-search related methods + * by assuming that the array is sorted in the ascending order. + * + * @author Kohsuke Kawaguchi + */ +class SortedList> extends AbstractList { + private List data; + + public SortedList(List data) { + this.data = data; + } + + /** + * Binary search to find the position of the given string. + * + * @return + * -(insertionPoint+1) if the exact string isn't found. + * That is, -1 means the probe would be inserted at the very beginning. + */ + public int find(T probe) { + return Collections.binarySearch(data, probe); + } + + public T get(int idx) { + return data.get(idx); + } + + @Override + public int size() { + return data.size(); + } + + @Override + public T remove(int index) { + return data.remove(index); + } + + @Override + public boolean remove(Object o) { + return data.remove(o); + } + + /** + * Finds the index of the entry lower than v. + */ + public int lower(T v) { + return Boundary.LOWER.apply(find(v)); + } + + /** + * Finds the index of the entry greater than v. + */ + public int higher(T v) { + return Boundary.HIGHER.apply(find(v)); + } + + /** + * Finds the index of the entry lower or equal to v. + */ + public int floor(T v) { + return Boundary.FLOOR.apply(find(v)); + } + + /** + * Finds the index of the entry greater or equal to v. + */ + public int ceil(T v) { + return Boundary.CEIL.apply(find(v)); + } + + public boolean isInRange(int idx) { + return 0<=idx && idx { - private List data; - - public SortedStringList(List data) { - this.data = data; - } - - /** - * Binary search to find the position of the given string. - * - * @return - * -(insertionPoint+1) if the exact string isn't found. - * That is, -1 means the probe would be inserted at the very beginning. - */ - public int find(String probe) { - return Collections.binarySearch(data, probe); - } - - public String get(int idx) { - return data.get(idx); - } - - @Override - public int size() { - return data.size(); - } - - @Override - public String remove(int index) { - return data.remove(index); - } - - @Override - public boolean remove(Object o) { - return data.remove(o); - } - - /** - * Finds the index of the entry lower than v. - */ - public int lower(String v) { - return Boundary.LOWER.apply(find(v)); - } - - /** - * Finds the index of the entry greater than v. - */ - public int higher(String v) { - return Boundary.HIGHER.apply(find(v)); - } - - /** - * Finds the index of the entry lower or equal to v. - */ - public int floor(String v) { - return Boundary.FLOOR.apply(find(v)); - } - - /** - * Finds the index of the entry greater or equal to v. - */ - public int ceil(String v) { - return Boundary.CEIL.apply(find(v)); - } - - public boolean isInRange(int idx) { - return 0<=idx && idx { +public class FakeMap extends AbstractLazyLoadRunMap { public FakeMap(File dir) { super(dir); } diff --git a/core/src/test/java/jenkins/model/lazy/FakeMapBuilder.java b/core/src/test/java/jenkins/model/lazy/FakeMapBuilder.java index 20ab49c095..0288094245 100644 --- a/core/src/test/java/jenkins/model/lazy/FakeMapBuilder.java +++ b/core/src/test/java/jenkins/model/lazy/FakeMapBuilder.java @@ -1,3 +1,26 @@ +/* + * The MIT License + * + * Copyright (c) 2012, CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package jenkins.model.lazy; import org.apache.commons.io.FileUtils; diff --git a/core/src/test/java/jenkins/model/lazy/SortedStringListTest.java b/core/src/test/java/jenkins/model/lazy/SortedListTest.java similarity index 59% rename from core/src/test/java/jenkins/model/lazy/SortedStringListTest.java rename to core/src/test/java/jenkins/model/lazy/SortedListTest.java index 5ac482001c..10696be491 100644 --- a/core/src/test/java/jenkins/model/lazy/SortedStringListTest.java +++ b/core/src/test/java/jenkins/model/lazy/SortedListTest.java @@ -1,3 +1,26 @@ +/* + * The MIT License + * + * Copyright (c) 2012, CloudBees, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ package jenkins.model.lazy; import org.junit.Assert; @@ -9,8 +32,8 @@ import java.util.Arrays; /** * @author Kohsuke Kawaguchi */ -public class SortedStringListTest extends Assert { - SortedStringList l = new SortedStringList(new ArrayList(Arrays.asList("B","D","F"))); +public class SortedListTest extends Assert { + SortedList l = new SortedList(new ArrayList(Arrays.asList("B","D","F"))); @Test public void testCeil() { From 2e91bcb99a40e344801551c8721f267330f656aa Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 17 Sep 2012 14:55:02 -0700 Subject: [PATCH 20/69] these methods actually return unmodifiable stuff. --- .../main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index 8a5485500a..2e26986872 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -115,7 +115,7 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i @Override public Set> entrySet() { - return all().entrySet(); + return Collections.unmodifiableSet(all().entrySet()); } public SortedMap subMap(Integer fromKey, Integer toKey) { @@ -130,7 +130,7 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i assert i!=null; } - return byNumber.subMap(fromKey,toKey); + return Collections.unmodifiableSortedMap(byNumber.subMap(fromKey,toKey)); } public SortedMap headMap(Integer toKey) { From ab4b0e1e2081e217c6a5ac5b2f66781dcfef2a84 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 17 Sep 2012 15:08:32 -0700 Subject: [PATCH 21/69] byId needs to be copied, too --- .../jenkins/model/lazy/AbstractLazyLoadRunMap.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index 2e26986872..19b9924484 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -54,13 +54,14 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i /** * Stores the mapping from build number to build, for builds that are already loaded. */ - // copy on write map + // copy on write private volatile TreeMap byNumber = new TreeMap(); /** * Stores the build ID to build number for builds that we already know */ - private final TreeMap byId = new TreeMap(); + // copy on write + private volatile TreeMap byId = new TreeMap(); /** * Build IDs found as directories, in the ascending order. @@ -130,15 +131,15 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i assert i!=null; } - return Collections.unmodifiableSortedMap(byNumber.subMap(fromKey,toKey)); + return Collections.unmodifiableSortedMap(byNumber.subMap(fromKey, toKey)); } public SortedMap headMap(Integer toKey) { - return subMap(Integer.MIN_VALUE,toKey); + return subMap(Integer.MIN_VALUE, toKey); } public SortedMap tailMap(Integer fromKey) { - return subMap(fromKey,Integer.MAX_VALUE); + return subMap(fromKey, Integer.MAX_VALUE); } public Integer firstKey() { @@ -339,6 +340,7 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i * Creates a duplicate for the COW data structure in preparation for mutation. */ private void copy() { + byId = new TreeMap(byId); byNumber = new TreeMap(byNumber); } From 8fc8915f6fbbf31e07d1320d09c98b59919b6faf Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 17 Sep 2012 15:33:23 -0700 Subject: [PATCH 22/69] added mutation methods --- .../model/lazy/AbstractLazyLoadRunMap.java | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index 19b9924484..d3898ef54d 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -31,6 +31,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; +import java.util.Map; import java.util.NoSuchElementException; import java.util.Set; import java.util.SortedMap; @@ -305,11 +306,30 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i return search(n,Direction.EXACT); } -// protected int toIdIndex(Entry f, int fallback) { -// if (f==null) return fallback; -// return getNumberOf(f.getValue()); -// } + public R put(R value) { + return put(getNumberOf(value),value); + } + @Override + public R put(Integer key, R r) { + synchronized (loadLock) { + copy(); + R old = byId.put(getIdOf(r),r); + byNumber.put(getNumberOf(r),r); + return old; + } + } + + @Override + public void putAll(Map rhs) { + synchronized (loadLock) { + copy(); + for (R r : rhs.values()) { + byId.put(getIdOf(r),r); + byNumber.put(getNumberOf(r),r); + } + } + } /** * Loads all the build records to fully populate the map. @@ -412,13 +432,13 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i */ protected abstract FilenameFilter createDirectoryFilter(); - public static final Comparator COMPARATOR = new Comparator() { + private static final Comparator COMPARATOR = new Comparator() { public int compare(Comparable o1, Comparable o2) { return -o1.compareTo(o2); } }; - enum Direction { + public enum Direction { ASC, DESC, EXACT } From fb3fea037f907c09fea5b0a3dbb493f94f4316ab Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 17 Sep 2012 16:03:13 -0700 Subject: [PATCH 23/69] adding remove() because RunMap needs it --- .../model/lazy/AbstractLazyLoadRunMap.java | 50 +++++++++++++++++-- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index d3898ef54d..e737e99c2b 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -23,6 +23,9 @@ */ package jenkins.model.lazy; +import org.kohsuke.accmod.Restricted; +import org.kohsuke.accmod.restrictions.NoExternalUse; + import java.io.File; import java.io.FilenameFilter; import java.io.IOException; @@ -36,6 +39,8 @@ import java.util.NoSuchElementException; import java.util.Set; import java.util.SortedMap; import java.util.TreeMap; +import java.util.logging.Level; +import java.util.logging.Logger; import static jenkins.model.lazy.AbstractLazyLoadRunMap.Direction.*; import static jenkins.model.lazy.Boundary.*; @@ -76,7 +81,13 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i // copy on write private SortedIntList numberOnDisk; - private final File dir; + /** + * Base directory for data. + * In effect this is treated as a final field, but can't mark it final + * because the compatibility requires that we make it settable + * in the first call after the constructor. + */ + private File dir; /** * Used to ensure only one thread is actually calling {@link #retrieve(File)} and @@ -84,9 +95,17 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i */ private final Object loadLock = this; - public AbstractLazyLoadRunMap(File dir) { + protected AbstractLazyLoadRunMap(File dir) { this.dir = dir; - loadIdOnDisk(); + initBaseDir(dir); + } + + @Restricted(NoExternalUse.class) + protected void initBaseDir(File dir) { + assert this.dir==null; + this.dir = dir; + if (dir!=null) + loadIdOnDisk(); } private void loadIdOnDisk() { @@ -401,12 +420,14 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i synchronized (loadLock) { try { R r = retrieve(dataDir); + if (r==null) return null; + if (copy) copy(); byId.put(getIdOf(r),r); byNumber.put(getNumberOf(r),r); return r; } catch (IOException e) { - e.printStackTrace(); + LOGGER.log(Level.WARNING, "Failed to load "+dataDir,e); } return null; } @@ -415,8 +436,27 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i protected abstract int getNumberOf(R r); protected abstract String getIdOf(R r); + /** + * Parses {@code R} instance from data in the specified directory. + * + * @return + * null if the parsing failed. + * @throws IOException + * if the parsing failed. This is just like returning null + * except the caller will catch the exception and report it. + */ protected abstract R retrieve(File dir) throws IOException; + public boolean remove(R run) { + synchronized (loadLock) { + copy(); + byNumber.remove(getNumberOf(run)); + R old = byId.remove(getIdOf(run)); + return old!=null; + } + } + + @Override public int hashCode() { return System.identityHashCode(this); @@ -445,4 +485,6 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i private static final String[] EMPTY_STRING_ARRAY = new String[0]; private static final SortedMap EMPTY_SORTED_MAP = Collections.unmodifiableSortedMap(new TreeMap()); + + private static final Logger LOGGER = Logger.getLogger(AbstractLazyLoadRunMap.class.getName()); } From 9e86629c76b2c8772cb955d5b5ca66296df3b83f Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 17 Sep 2012 16:18:08 -0700 Subject: [PATCH 24/69] Use 'this' as a lock to remain compatible with RunMap --- .../model/lazy/AbstractLazyLoadRunMap.java | 88 ++++++++++--------- 1 file changed, 46 insertions(+), 42 deletions(-) diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index e737e99c2b..77378649c1 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -51,6 +51,11 @@ import static jenkins.model.lazy.Boundary.*; * This implementation is in 2 states. An instance can be {@linkplain #fullyLoaded fully loaded} state, * where everything that can be loaded gets loaded. Otherwise it's in a partially loaded state. * + *

+ * Object lock of {@code this} is used to make sure mutation occurs sequentially. + * That is, ensure that only one thread is actually calling {@link #retrieve(File)} and + * updating {@link #byNumber} and {@link #byId}. + * * @author Kohsuke Kawaguchi */ public abstract class AbstractLazyLoadRunMap extends AbstractMap implements SortedMap { @@ -89,12 +94,6 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i */ private File dir; - /** - * Used to ensure only one thread is actually calling {@link #retrieve(File)} and - * updating {@link #byNumber} and {@link #byId}. - */ - private final Object loadLock = this; - protected AbstractLazyLoadRunMap(File dir) { this.dir = dir; initBaseDir(dir); @@ -330,23 +329,19 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i } @Override - public R put(Integer key, R r) { - synchronized (loadLock) { - copy(); - R old = byId.put(getIdOf(r),r); - byNumber.put(getNumberOf(r),r); - return old; - } + public synchronized R put(Integer key, R r) { + copy(); + R old = byId.put(getIdOf(r),r); + byNumber.put(getNumberOf(r),r); + return old; } @Override - public void putAll(Map rhs) { - synchronized (loadLock) { - copy(); - for (R r : rhs.values()) { - byId.put(getIdOf(r),r); - byNumber.put(getNumberOf(r),r); - } + public synchronized void putAll(Map rhs) { + copy(); + for (R r : rhs.values()) { + byId.put(getIdOf(r),r); + byNumber.put(getNumberOf(r),r); } } @@ -361,7 +356,7 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i */ private TreeMap all() { if (!fullyLoaded) { - synchronized (loadLock) { + synchronized (this) { if (!fullyLoaded) { copy(); for (String id : idOnDisk) { @@ -392,7 +387,7 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i R r = null; File shortcut = new File(dir,String.valueOf(n)); if (shortcut.isDirectory()) { - synchronized (loadLock) { + synchronized (this) { r = load(shortcut,copy); // make sure what we actually loaded is #n, @@ -416,21 +411,19 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i return load(new File(dir,id),copy); } - protected R load(File dataDir, boolean copy) { - synchronized (loadLock) { - try { - R r = retrieve(dataDir); - if (r==null) return null; + protected synchronized R load(File dataDir, boolean copy) { + try { + R r = retrieve(dataDir); + if (r==null) return null; - if (copy) copy(); - byId.put(getIdOf(r),r); - byNumber.put(getNumberOf(r),r); - return r; - } catch (IOException e) { - LOGGER.log(Level.WARNING, "Failed to load "+dataDir,e); - } - return null; + if (copy) copy(); + byId.put(getIdOf(r),r); + byNumber.put(getNumberOf(r),r); + return r; + } catch (IOException e) { + LOGGER.log(Level.WARNING, "Failed to load "+dataDir,e); } + return null; } protected abstract int getNumberOf(R r); @@ -447,15 +440,26 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i */ protected abstract R retrieve(File dir) throws IOException; - public boolean remove(R run) { - synchronized (loadLock) { - copy(); - byNumber.remove(getNumberOf(run)); - R old = byId.remove(getIdOf(run)); - return old!=null; - } + public synchronized boolean remove(R run) { + copy(); + byNumber.remove(getNumberOf(run)); + R old = byId.remove(getIdOf(run)); + return old!=null; } + /** + * Replaces all the current loaded Rs with the given ones. + */ + public synchronized void reset(TreeMap builds) { + TreeMap byNumber = new TreeMap(builds); + TreeMap byId = new TreeMap(); + for (R r : builds.values()) { + byId.put(getIdOf(r),r); + } + + this.byNumber = byNumber; + this.byId = byId; + } @Override public int hashCode() { From b2fecdf8604491ca829d316e053fb197ad3ea7fb Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 17 Sep 2012 16:18:24 -0700 Subject: [PATCH 25/69] merged up with AbstractLazyLoadRunMap --- core/src/main/java/hudson/model/RunMap.java | 224 ++++++-------------- 1 file changed, 65 insertions(+), 159 deletions(-) diff --git a/core/src/main/java/hudson/model/RunMap.java b/core/src/main/java/hudson/model/RunMap.java index b622da0191..6b962b00f0 100644 --- a/core/src/main/java/hudson/model/RunMap.java +++ b/core/src/main/java/hudson/model/RunMap.java @@ -23,22 +23,23 @@ */ package hudson.model; -import com.google.common.collect.Maps; +import jenkins.model.lazy.AbstractLazyLoadRunMap; +import org.apache.commons.collections.comparators.ReverseComparator; + import java.io.File; import java.io.FilenameFilter; import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.util.AbstractMap; import java.util.Collections; import java.util.Comparator; import java.util.Map; -import java.util.Set; import java.util.SortedMap; -import java.util.TreeMap; import java.util.logging.Level; import java.util.logging.Logger; +import static jenkins.model.lazy.AbstractLazyLoadRunMap.Direction.*; + /** * {@link Map} from build number to {@link Run}. * @@ -49,86 +50,30 @@ import java.util.logging.Logger; * * @author Kohsuke Kawaguchi */ -public final class RunMap> extends AbstractMap implements SortedMap { - // copy-on-write map - private transient volatile SortedMap builds = - new TreeMap(COMPARATOR); - +public final class RunMap> extends AbstractLazyLoadRunMap { /** * Read-only view of this map. */ private final SortedMap view = Collections.unmodifiableSortedMap(this); - public Set> entrySet() { - // since the map is copy-on-write, make sure no one modifies it - return Collections.unmodifiableSet(builds.entrySet()); - } + private Constructor cons; - public synchronized R put(R value) { - return put(value.getNumber(),value); + // TODO: before first complete build + // patch up next/previous build link + + + public RunMap() { + super(null); // will be set later } @Override - public synchronized R put(Integer key, R value) { - // copy-on-write update - TreeMap m = new TreeMap(builds); - - R r = update(m, key, value); - - this.builds = m; - return r; - } - - @Override - public synchronized void putAll(Map rhs) { - // copy-on-write update - TreeMap m = new TreeMap(builds); - - for (Map.Entry e : rhs.entrySet()) - update(m, e.getKey(), e.getValue()); - - this.builds = m; - } - - private R update(TreeMap m, Integer key, R value) { - // things are bit tricky because this map is order so that the newest one comes first, - // yet 'nextBuild' refers to the newer build. - R first = m.isEmpty() ? null : m.get(m.firstKey()); - R r = m.put(key, value); - SortedMap head = m.headMap(key); - if(!head.isEmpty()) { - R prev = m.get(head.lastKey()); - value.previousBuild = prev.previousBuild; - value.nextBuild = prev; - if(value.previousBuild!=null) - value.previousBuild.nextBuild = value; - prev.previousBuild=value; - } else { - value.previousBuild = first; - value.nextBuild = null; - if(first!=null) - first.nextBuild = value; - } - return r; - } - - public synchronized boolean remove(R run) { + public boolean remove(R run) { if(run.nextBuild!=null) run.nextBuild.previousBuild = run.previousBuild; if(run.previousBuild!=null) - run.previousBuild.nextBuild = run.nextBuild; + run.previousBuild.nextBuild = run.nextBuild; - // copy-on-write update - TreeMap m = new TreeMap(builds); - R r = m.remove(run.getNumber()); - this.builds = m; - - return r!=null; - } - - public synchronized void reset(TreeMap builds) { - this.builds = new TreeMap(COMPARATOR); - putAll(builds); + return super.remove(run); } /** @@ -138,51 +83,24 @@ public final class RunMap> extends AbstractMap imp return view; } -// -// SortedMap delegation -// - public Comparator comparator() { - return builds.comparator(); - } - - public SortedMap subMap(Integer fromKey, Integer toKey) { - return builds.subMap(fromKey, toKey); - } - - public SortedMap headMap(Integer toKey) { - return builds.headMap(toKey); - } - - public SortedMap tailMap(Integer fromKey) { - return builds.tailMap(fromKey); - } - - public Integer firstKey() { - return builds.firstKey(); - } - - public Integer lastKey() { - return builds.lastKey(); - } - /** * This is the newest build (with the biggest build number) */ public R newestValue() { - SortedMap _builds = builds; - if (_builds.isEmpty()) return null; - return _builds.get(firstKey()); + return search(Integer.MAX_VALUE, DESC); } /** * This is the oldest build (with the smallest build number) */ public R oldestValue() { - SortedMap _builds = builds; - if (_builds.isEmpty()) return null; - return _builds.get(lastKey()); + return search(Integer.MIN_VALUE, ASC); } + /** + * @deprecated + * Use {@link ReverseComparator} + */ public static final Comparator COMPARATOR = new Comparator() { public int compare(Comparable o1, Comparable o2) { return -o1.compareTo(o2); @@ -196,21 +114,21 @@ public final class RunMap> extends AbstractMap imp R create(File dir) throws IOException; } - /** - * Fills in {@link RunMap} by loading build records from the file system. - * - * @param job - * Job that owns this map. - * @param cons - * Used to create new instance of {@link Run}. - */ - public synchronized void load(Job job, Constructor cons) { + @Override + protected final int getNumberOf(R r) { + return r.getNumber(); + } + + @Override + protected final String getIdOf(R r) { + return r.getId(); + } + + @Override + protected FilenameFilter createDirectoryFilter() { final SimpleDateFormat formatter = Run.ID_FORMATTER.get(); - TreeMap builds = new TreeMap(RunMap.COMPARATOR); - File buildDir = job.getBuildDir(); - buildDir.mkdirs(); - String[] buildDirs = buildDir.list(new FilenameFilter() { + return new FilenameFilter() { public boolean accept(File dir, String name) { // JENKINS-1461 sometimes create bogus data directories with impossible dates, such as year 0, April 31st, // or August 0th. Date object doesn't roundtrip those, so we eventually fail to load this data. @@ -231,52 +149,40 @@ public final class RunMap> extends AbstractMap imp } return false; } - }); + }; + } - // keep all those that are building intact. - Map building = Maps.newHashMap(); - for (R b=newestValue(); b!=null && b.isBuilding(); b=b.getPreviousBuild()) { - building.put(b.getNumber(), b); - } - - for( String build : buildDirs ) { - File d = new File(buildDir,build); - if(new File(d,"build.xml").exists()) { - // if the build result file isn't in the directory, ignore it. - try { - R b = cons.create(d); - R existing = builds.put(b.getNumber(), b); - if (existing != null) { - LOGGER.log(Level.WARNING, "multiple runs claiming to be #{0}; using run from {1}", new Object[] {b.getNumber(), d}); - } - } catch (IOException e) { - LOGGER.log(Level.WARNING, "could not load " + d, e); - } catch (InstantiationError e) { - LOGGER.log(Level.WARNING, "could not load " + d, e); - } + @Override + protected R retrieve(File d) throws IOException { + if(new File(d,"build.xml").exists()) { + // if the build result file isn't in the directory, ignore it. + try { + R b = cons.create(d); + b.onLoad(); + return b; + } catch (IOException e) { + LOGGER.log(Level.WARNING, "could not load " + d, e); + } catch (InstantiationError e) { + LOGGER.log(Level.WARNING, "could not load " + d, e); } } + return null; + } - // overlay what's currently building on top of what's loaded - builds.putAll(building); - /* - // we probably aren't saving every little changes during the build to disk, - // so it's risky to reload these from disk. - for (R b : building.values()) { - try { - b.reload(); - } catch (IOException e) { - e.printStackTrace(); - } catch (InstantiationError e) { - e.printStackTrace(); - } - } - */ - - reset(builds); - - for (R r : builds.values()) - r.onLoad(); + /** + * Backward compatibility method that notifies {@link RunMap} of who the owner is. + * + * Traditionally, this method blocked and loaded all the build records on the disk, + * but now all the actual loading happens lazily. + * + * @param job + * Job that owns this map. + * @param cons + * Used to create new instance of {@link Run}. + */ + public synchronized void load(Job job, Constructor cons) { + this.cons = cons; + initBaseDir(job.getBuildDir()); } private static final Logger LOGGER = Logger.getLogger(RunMap.class.getName()); From 6b5a2f6a8f819ac0b066507d95d1fb49cb56ec59 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 17 Sep 2012 16:35:52 -0700 Subject: [PATCH 26/69] initialize with empty index to allow operation without ever calling initBaseDir() --- .../main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index 77378649c1..fff288b27c 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -78,13 +78,13 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i * Build IDs found as directories, in the ascending order. */ // copy on write - private SortedList idOnDisk; + private SortedList idOnDisk = new SortedList(Collections.emptyList()); /** * Build bumber shortcuts found on disk, in the ascending order. */ // copy on write - private SortedIntList numberOnDisk; + private SortedIntList numberOnDisk = new SortedIntList(0); /** * Base directory for data. From eec63febb9f4232ad948722b1c84d1d10dd96c4e Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 17 Sep 2012 16:36:25 -0700 Subject: [PATCH 27/69] for now don't let plugins consume this --- .../src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index fff288b27c..ae185c0b99 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -94,6 +94,7 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i */ private File dir; + @Restricted(NoExternalUse.class) // for now protected AbstractLazyLoadRunMap(File dir) { this.dir = dir; initBaseDir(dir); From 7741f96ba3771035ab69034376e6d2a6bc8a7aaf Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 17 Sep 2012 16:45:26 -0700 Subject: [PATCH 28/69] fix up next/previous build links --- .../main/java/hudson/model/AbstractBuild.java | 21 ++++++++++++++++++ .../java/hudson/model/AbstractProject.java | 6 ++--- core/src/main/java/hudson/model/Run.java | 22 +++++++++---------- core/src/main/java/hudson/model/RunMap.java | 17 +++++++++++++- .../model/lazy/AbstractLazyLoadRunMap.java | 1 - 5 files changed, 50 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/hudson/model/AbstractBuild.java b/core/src/main/java/hudson/model/AbstractBuild.java index 2d1bc35d4e..72ea53596c 100644 --- a/core/src/main/java/hudson/model/AbstractBuild.java +++ b/core/src/main/java/hudson/model/AbstractBuild.java @@ -62,6 +62,7 @@ import hudson.util.Iterators; import hudson.util.LogTaskListener; import hudson.util.VariableResolver; import jenkins.model.Jenkins; +import jenkins.model.lazy.AbstractLazyLoadRunMap.Direction; import org.kohsuke.stapler.Stapler; import org.kohsuke.stapler.StaplerRequest; import org.kohsuke.stapler.StaplerResponse; @@ -170,6 +171,26 @@ public abstract class AbstractBuild

,R extends Abs return getParent(); } + private boolean previousBuildComputed, nextBuildComputed; + + @Override + public R getPreviousBuild() { + if (previousBuild==null && !previousBuildComputed) { + previousBuild = getParent().builds.search(number-1, Direction.DESC); + previousBuildComputed = true; + } + return previousBuild; + } + + @Override + public R getNextBuild() { + if (nextBuild==null && !nextBuildComputed) { + nextBuild = getParent().builds.search(number+1, Direction.ASC); + nextBuildComputed = true; + } + return nextBuild; + } + /** * Returns a {@link Slave} on which this build was done. * diff --git a/core/src/main/java/hudson/model/AbstractProject.java b/core/src/main/java/hudson/model/AbstractProject.java index 046e3fc7bf..aa039688f8 100644 --- a/core/src/main/java/hudson/model/AbstractProject.java +++ b/core/src/main/java/hudson/model/AbstractProject.java @@ -160,7 +160,7 @@ public abstract class AbstractProject

,R extends A * {@link Run#getPreviousBuild()} */ @Restricted(NoExternalUse.class) - protected transient /*almost final*/ RunMap builds = new RunMap(); + protected transient RunMap builds = new RunMap(); /** * The quiet period. Null to delegate to the system default. @@ -269,9 +269,7 @@ public abstract class AbstractProject

,R extends A public void onLoad(ItemGroup parent, String name) throws IOException { super.onLoad(parent, name); - if (this.builds==null) - this.builds = new RunMap(); - this.builds.load(this,new Constructor() { + this.builds = new RunMap(getBuildDir(),new Constructor() { public R create(File dir) throws IOException { return loadBuild(dir); } diff --git a/core/src/main/java/hudson/model/Run.java b/core/src/main/java/hudson/model/Run.java index eb44e7c2f4..9c17f98518 100644 --- a/core/src/main/java/hudson/model/Run.java +++ b/core/src/main/java/hudson/model/Run.java @@ -44,7 +44,6 @@ import hudson.model.Descriptor.FormException; import hudson.model.listeners.RunListener; import hudson.model.listeners.SaveableListener; import hudson.security.PermissionScope; -import jenkins.model.Jenkins.MasterComputer; import hudson.search.SearchIndexBuilder; import hudson.security.ACL; import hudson.security.AccessControlled; @@ -640,10 +639,11 @@ public abstract class Run ,RunT extends Run,RunT extends Run,RunT extends Run,RunT extends Run,RunT extends Run> extends AbstractLazyLoadRunMap // patch up next/previous build link + /** + * @deprecated + * Use {@link #RunMap(File, Constructor)}. + */ public RunMap() { super(null); // will be set later } + /** + * @param cons + * Used to create new instance of {@link Run}. + */ + public RunMap(File baseDir, Constructor cons) { + super(baseDir); + this.cons = cons; + } + @Override public boolean remove(R run) { if(run.nextBuild!=null) @@ -179,8 +192,10 @@ public final class RunMap> extends AbstractLazyLoadRunMap * Job that owns this map. * @param cons * Used to create new instance of {@link Run}. + * @deprecated + * Use {@link #RunMap(File, Constructor)} */ - public synchronized void load(Job job, Constructor cons) { + public void load(Job job, Constructor cons) { this.cons = cons; initBaseDir(job.getBuildDir()); } diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index ae185c0b99..fff288b27c 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -94,7 +94,6 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i */ private File dir; - @Restricted(NoExternalUse.class) // for now protected AbstractLazyLoadRunMap(File dir) { this.dir = dir; initBaseDir(dir); From 456f14ce52410a9127261434f827ef1a0b7e9207 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 17 Sep 2012 16:48:20 -0700 Subject: [PATCH 29/69] remove(R) collides with remove(Object) at AbstractLazyLoadRunMap. So renaming a method. RunMap.remove(R) needs to be still called remove(R) to remain compatible --- core/src/main/java/hudson/model/RunMap.java | 8 ++++++-- .../java/jenkins/model/lazy/AbstractLazyLoadRunMap.java | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/hudson/model/RunMap.java b/core/src/main/java/hudson/model/RunMap.java index edf5864f5d..e87a7feed4 100644 --- a/core/src/main/java/hudson/model/RunMap.java +++ b/core/src/main/java/hudson/model/RunMap.java @@ -79,14 +79,18 @@ public final class RunMap> extends AbstractLazyLoadRunMap this.cons = cons; } - @Override public boolean remove(R run) { + return removeValue(run); + } + + @Override + public boolean removeValue(R run) { if(run.nextBuild!=null) run.nextBuild.previousBuild = run.previousBuild; if(run.previousBuild!=null) run.previousBuild.nextBuild = run.nextBuild; - return super.remove(run); + return super.removeValue(run); } /** diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index fff288b27c..cc1b5bb0be 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -440,7 +440,7 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i */ protected abstract R retrieve(File dir) throws IOException; - public synchronized boolean remove(R run) { + public synchronized boolean removeValue(R run) { copy(); byNumber.remove(getNumberOf(run)); R old = byId.remove(getIdOf(run)); From 7f72bf1a2247b5a59263f027b2c73d4d9b3a319f Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 17 Sep 2012 17:03:18 -0700 Subject: [PATCH 30/69] otherwise the assert in initBaseDir will fail right away --- .../src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index cc1b5bb0be..867bf85ad0 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -95,7 +95,6 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i private File dir; protected AbstractLazyLoadRunMap(File dir) { - this.dir = dir; initBaseDir(dir); } From b0541e4bbe63b9edc88e97aef864657ed4999cdd Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 17 Sep 2012 17:04:11 -0700 Subject: [PATCH 31/69] Having two neighbors pointing to each other is important to make RunMap.removeValue work. Why, you may ask? Think of neighbors A, B. If A points to B but B doesn't point to A, when B is removed A will still end up pointing B as the next build. --- core/src/main/java/hudson/model/AbstractBuild.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/hudson/model/AbstractBuild.java b/core/src/main/java/hudson/model/AbstractBuild.java index 72ea53596c..02f9152cbe 100644 --- a/core/src/main/java/hudson/model/AbstractBuild.java +++ b/core/src/main/java/hudson/model/AbstractBuild.java @@ -176,7 +176,11 @@ public abstract class AbstractBuild

,R extends Abs @Override public R getPreviousBuild() { if (previousBuild==null && !previousBuildComputed) { - previousBuild = getParent().builds.search(number-1, Direction.DESC); + // having two neighbors pointing to each other is important to make RunMap.removeValue work + R previousBuild = getParent().builds.search(number-1, Direction.DESC); + if (previousBuild!=null) + previousBuild.nextBuild = (R)this; + this.previousBuild = previousBuild; previousBuildComputed = true; } return previousBuild; @@ -185,7 +189,11 @@ public abstract class AbstractBuild

,R extends Abs @Override public R getNextBuild() { if (nextBuild==null && !nextBuildComputed) { - nextBuild = getParent().builds.search(number+1, Direction.ASC); + // having two neighbors pointing to each other is important to make RunMap.removeValue work + R nextBuild = getParent().builds.search(number+1, Direction.ASC); + if (nextBuild!=null) + nextBuild.previousBuild = (R)this; + this.nextBuild = nextBuild; nextBuildComputed = true; } return nextBuild; From 64ab2ca2001cd771cbe3b3c6390c8f1f620017e1 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 17 Sep 2012 17:40:32 -0700 Subject: [PATCH 32/69] Added efficient contains() method --- .../java/jenkins/model/lazy/SortedIntList.java | 15 ++++++++++++++- .../main/java/jenkins/model/lazy/SortedList.java | 7 ++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/jenkins/model/lazy/SortedIntList.java b/core/src/main/java/jenkins/model/lazy/SortedIntList.java index 12c7fc61a4..b9d8af14f9 100644 --- a/core/src/main/java/jenkins/model/lazy/SortedIntList.java +++ b/core/src/main/java/jenkins/model/lazy/SortedIntList.java @@ -46,7 +46,7 @@ class SortedIntList extends AbstractList { * Internal copy constructor. */ public SortedIntList(SortedIntList rhs) { - this.data = Arrays.copyOf(rhs.data,rhs.data.length); + this.data = Arrays.copyOf(rhs.data,rhs.data.length+8); this.size = rhs.size; } @@ -61,6 +61,15 @@ class SortedIntList extends AbstractList { return Arrays.binarySearch(data, 0, size, probe); } + @Override + public boolean contains(Object o) { + return o instanceof Integer && contains(((Integer)o).intValue()); + } + + public boolean contains(int i) { + return find(i)>=0; + } + @Override public Integer get(int index) { if (size<=index) throw new IndexOutOfBoundsException(); @@ -127,6 +136,10 @@ class SortedIntList extends AbstractList { Arrays.sort(data,0,size); } + public void copyInto(int[] dest) { + System.arraycopy(data,0,dest,0,size); + } + public void removeValue(int n) { int idx = find(n); if (idx<0) return; diff --git a/core/src/main/java/jenkins/model/lazy/SortedList.java b/core/src/main/java/jenkins/model/lazy/SortedList.java index 9c66d9dd8a..37139cafe1 100644 --- a/core/src/main/java/jenkins/model/lazy/SortedList.java +++ b/core/src/main/java/jenkins/model/lazy/SortedList.java @@ -50,7 +50,12 @@ class SortedList> extends AbstractList { public int find(T probe) { return Collections.binarySearch(data, probe); } - + + @Override + public boolean contains(Object o) { + return find((T)o)>=0; + } + public T get(int idx) { return data.get(idx); } From bbe74db9209aa0c6a0c48f60972fde67be08690d Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 17 Sep 2012 17:40:53 -0700 Subject: [PATCH 33/69] when new objects are added programmatically, disk index needs to be updated or else search() will not find them --- .../model/lazy/AbstractLazyLoadRunMap.java | 43 +++++++++++++++++-- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index 867bf85ad0..885eb693ee 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -132,6 +132,15 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i return COMPARATOR; } + /** + * If we have non-zero R in memory, we can return false right away. + * If we have zero R in memory, try loading one and see if we can find something. + */ + @Override + public boolean isEmpty() { + return byId.isEmpty() && search(Integer.MAX_VALUE, DESC)==null; + } + @Override public Set> entrySet() { return Collections.unmodifiableSet(all().entrySet()); @@ -186,7 +195,10 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i } /** - * Loads the build #M where M is nearby the given 'n'. + * Finds the build #M where M is nearby the given 'n'. + * + *

+ * * * @param n * the index to start the search from @@ -323,15 +335,38 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i return search(n,Direction.EXACT); } - public R put(R value) { + public final R put(R value) { return put(getNumberOf(value),value); } @Override public synchronized R put(Integer key, R r) { + String id = getIdOf(r); + int n = getNumberOf(r); + copy(); - R old = byId.put(getIdOf(r),r); - byNumber.put(getNumberOf(r),r); + R old = byId.put(id,r); + byNumber.put(n,r); + + /* + search relies on the fact that every objet added via + put() method be available in the xyzOnDisk index, so I'm adding them here + however, this is awfully inefficient. I wonder if there's any better way to do this? + */ + if (!idOnDisk.contains(id)) { + ArrayList a = new ArrayList(idOnDisk); + a.add(id); + Collections.sort(a); + idOnDisk = new SortedList(a); + } + + if (!numberOnDisk.contains(n)) { + SortedIntList a = new SortedIntList(numberOnDisk); + a.add(n); + a.sort(); + numberOnDisk = a; + } + return old; } From 25ed3c5818592a2e51620d1d55654e36412be4f2 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Mon, 17 Sep 2012 18:37:01 -0700 Subject: [PATCH 34/69] RunMap should be in the descending order --- .../model/lazy/AbstractLazyLoadRunMap.java | 43 +++++++++++-------- .../lazy/AbstractLazyLoadRunMapTest.java | 14 +++--- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index 885eb693ee..ea92ac2a95 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -23,6 +23,7 @@ */ package jenkins.model.lazy; +import hudson.model.Run; import org.kohsuke.accmod.Restricted; import org.kohsuke.accmod.restrictions.NoExternalUse; @@ -46,7 +47,12 @@ import static jenkins.model.lazy.AbstractLazyLoadRunMap.Direction.*; import static jenkins.model.lazy.Boundary.*; /** - * Lazy-loading map of build records. + * {@link SortedMap} that keeps build records by their build numbers, in the descending order + * (newer ones first.) + * + * The main thing about this class is that it encapsulates the lazy loading logic. + * That is, + * think of subtype of {@link Run} * * This implementation is in 2 states. An instance can be {@linkplain #fullyLoaded fully loaded} state, * where everything that can be loaded gets loaded. Otherwise it's in a partially loaded state. @@ -66,7 +72,7 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i * Stores the mapping from build number to build, for builds that are already loaded. */ // copy on write - private volatile TreeMap byNumber = new TreeMap(); + private volatile TreeMap byNumber = new TreeMap(COMPARATOR); /** * Stores the build ID to build number for builds that we already know @@ -147,14 +153,14 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i } public SortedMap subMap(Integer fromKey, Integer toKey) { - R start = search(fromKey, ASC); + R start = search(fromKey, DESC); if (start==null) return EMPTY_SORTED_MAP; - R end = search(toKey-1, DESC); + R end = search(toKey-1, ASC); if (end==null) return EMPTY_SORTED_MAP; for (R i=start; i!=end; ) { - i = search(getNumberOf(i)+1,ASC); + i = search(getNumberOf(i)-1,DESC); assert i!=null; } @@ -162,21 +168,21 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i } public SortedMap headMap(Integer toKey) { - return subMap(Integer.MIN_VALUE, toKey); + return subMap(Integer.MAX_VALUE, toKey); } public SortedMap tailMap(Integer fromKey) { - return subMap(fromKey, Integer.MAX_VALUE); + return subMap(fromKey, Integer.MIN_VALUE); } public Integer firstKey() { - R r = search(Integer.MIN_VALUE, ASC); + R r = search(Integer.MAX_VALUE, DESC); if (r==null) throw new NoSuchElementException(); return getNumberOf(r); } public Integer lastKey() { - R r = search(Integer.MAX_VALUE, DESC); + R r = search(Integer.MIN_VALUE, ASC); if (r==null) throw new NoSuchElementException(); return getNumberOf(r); } @@ -209,8 +215,8 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i * If DESC, finds the closest #M that satisfies M<=N. */ public R search(final int n, final Direction d) { - Entry f = byNumber.floorEntry(n); - if (f!=null && f.getKey()== n) return f.getValue(); // found the exact #n + Entry c = byNumber.ceilingEntry(n); + if (c!=null && c.getKey()== n) return c.getValue(); // found the exact #n // at this point we know that we don't have #n loaded yet @@ -268,16 +274,16 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i // first, narrow down the candidate IDs to try by using two known number-to-ID mapping if (idOnDisk.isEmpty()) return null; - Entry c = byNumber.ceilingEntry(n); + Entry f = byNumber.floorEntry(n); // if bound is null, use a sentinel value - String fid = f==null ? "\u0000" : getIdOf(f.getValue()); - String cid = c==null ? "\uFFFF" : getIdOf(c.getValue()); + String cid = c==null ? "\u0000" : getIdOf(c.getValue()); + String fid = f==null ? "\uFFFF" : getIdOf(f.getValue()); // We know that the build we are looking for exists in this range // we will narrow this down via binary search - int lo = idOnDisk.higher(fid); - int hi = idOnDisk.lower(cid)+1; + int lo = idOnDisk.higher(cid); + int hi = idOnDisk.lower(fid)+1; int pivot; while (true) { @@ -485,10 +491,11 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i * Replaces all the current loaded Rs with the given ones. */ public synchronized void reset(TreeMap builds) { - TreeMap byNumber = new TreeMap(builds); - TreeMap byId = new TreeMap(); + TreeMap byNumber = new TreeMap(COMPARATOR); + TreeMap byId = new TreeMap(COMPARATOR); for (R r : builds.values()) { byId.put(getIdOf(r),r); + byNumber.put(getNumberOf(r),r); } this.byNumber = byNumber; diff --git a/core/src/test/java/jenkins/model/lazy/AbstractLazyLoadRunMapTest.java b/core/src/test/java/jenkins/model/lazy/AbstractLazyLoadRunMapTest.java index 9be2daa8e5..3177c6d59c 100644 --- a/core/src/test/java/jenkins/model/lazy/AbstractLazyLoadRunMapTest.java +++ b/core/src/test/java/jenkins/model/lazy/AbstractLazyLoadRunMapTest.java @@ -88,7 +88,7 @@ public class AbstractLazyLoadRunMapTest extends Assert { @Test public void firstKey() { - assertEquals(1, a.firstKey().intValue()); + assertEquals(5, a.firstKey().intValue()); try { b.firstKey(); @@ -100,7 +100,7 @@ public class AbstractLazyLoadRunMapTest extends Assert { @Test public void lastKey() { - assertEquals(5, a.lastKey().intValue()); + assertEquals(1, a.lastKey().intValue()); try { b.lastKey(); fail(); @@ -131,9 +131,9 @@ public class AbstractLazyLoadRunMapTest extends Assert { @Test public void eagerLoading() throws IOException { Map.Entry[] b = a.entrySet().toArray(new Map.Entry[3]); - ((Build)b[0].getValue()).asserts(1,"A"); + ((Build)b[0].getValue()).asserts(5,"C"); ((Build)b[1].getValue()).asserts(3,"B"); - ((Build)b[2].getValue()).asserts(5,"C"); + ((Build)b[2].getValue()).asserts(1,"A"); } @Test @@ -201,13 +201,13 @@ public class AbstractLazyLoadRunMapTest extends Assert { @Test public void fastSubMap() throws Exception { - SortedMap m = a.subMap(2, 99); + SortedMap m = a.subMap(99, 2); assertEquals(2, m.size()); Build[] b = m.values().toArray(new Build[2]); assertEquals(2,b.length); - b[0].asserts(3,"B"); - b[1].asserts(5,"C"); + b[0].asserts(5,"C"); + b[1].asserts(3,"B"); } @Test From 93321854f2fe308d25e27bf000b700c48311f998 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Tue, 18 Sep 2012 09:39:48 -0700 Subject: [PATCH 35/69] off by one bug. toKey-1 won't work as intended when toKey is Integer.MIN_VALUE. --- .../java/jenkins/model/lazy/AbstractLazyLoadRunMap.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index ea92ac2a95..83475402ff 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -152,11 +152,17 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i return Collections.unmodifiableSet(all().entrySet()); } + /** + * @param fromKey + * Biggest build number to be in the returned set. + * @param toKey + * Smallest build number-1 to be in the returned set (-1 because this is exclusive) + */ public SortedMap subMap(Integer fromKey, Integer toKey) { R start = search(fromKey, DESC); if (start==null) return EMPTY_SORTED_MAP; - R end = search(toKey-1, ASC); + R end = search(toKey, ASC); if (end==null) return EMPTY_SORTED_MAP; for (R i=start; i!=end; ) { From b35678d5c1b548fc0752822c4e209b4ad078fd13 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Wed, 19 Sep 2012 14:31:13 -0700 Subject: [PATCH 36/69] fixed a regression in a test. --- core/src/main/java/hudson/model/AbstractProject.java | 11 +++++++++-- .../jenkins/model/lazy/AbstractLazyLoadRunMap.java | 7 +++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/hudson/model/AbstractProject.java b/core/src/main/java/hudson/model/AbstractProject.java index aa039688f8..a15b83d804 100644 --- a/core/src/main/java/hudson/model/AbstractProject.java +++ b/core/src/main/java/hudson/model/AbstractProject.java @@ -46,7 +46,6 @@ import hudson.model.Descriptor.FormException; import hudson.model.Fingerprint.RangeSet; import hudson.model.Queue.Executable; import hudson.model.Queue.Task; -import hudson.model.listeners.SCMListener; import hudson.model.queue.QueueTaskFuture; import hudson.model.queue.SubTask; import hudson.model.Queue.WaitingItem; @@ -269,11 +268,19 @@ public abstract class AbstractProject

,R extends A public void onLoad(ItemGroup parent, String name) throws IOException { super.onLoad(parent, name); - this.builds = new RunMap(getBuildDir(),new Constructor() { + RunMap builds = new RunMap(getBuildDir(), new Constructor() { public R create(File dir) throws IOException { return loadBuild(dir); } }); + if (this.builds!=null) { + // if we are reloading, keep all those that are still building intact + for (R r : this.builds.getLoadedBuilds().values()) { + if (r.isBuilding()) + builds.put(r); + } + } + this.builds = builds; if(triggers==null) { // it didn't exist in < 1.28 diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index 83475402ff..4e8ef4ba5b 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -152,6 +152,13 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i return Collections.unmodifiableSet(all().entrySet()); } + /** + * Returns a read-only view of records that has already been loaded. + */ + public SortedMap getLoadedBuilds() { + return Collections.unmodifiableSortedMap(byNumber); + } + /** * @param fromKey * Biggest build number to be in the returned set. From 0500ec9c279f780ddb0e3491f084e6ed9c755a9e Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Wed, 19 Sep 2012 15:29:08 -0700 Subject: [PATCH 37/69] overridden a number of Job methods in AbstractProject to avoid the slow path (of loading everything) --- .../java/hudson/model/AbstractProject.java | 49 ++++++++++++++++++- core/src/main/java/hudson/model/Job.java | 17 +++++-- core/src/main/java/hudson/model/RunMap.java | 30 +++++++++++- .../model/lazy/AbstractLazyLoadRunMap.java | 12 ++++- 4 files changed, 100 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/hudson/model/AbstractProject.java b/core/src/main/java/hudson/model/AbstractProject.java index a15b83d804..400072669a 100644 --- a/core/src/main/java/hudson/model/AbstractProject.java +++ b/core/src/main/java/hudson/model/AbstractProject.java @@ -79,9 +79,11 @@ import hudson.util.AlternativeUiTextProvider.Message; import hudson.util.DescribableList; import hudson.util.EditDistance; import hudson.util.FormValidation; +import hudson.util.RunList; import hudson.widgets.BuildHistoryWidget; import hudson.widgets.HistoryWidget; import jenkins.model.Jenkins; +import jenkins.model.lazy.AbstractLazyLoadRunMap.Direction; import jenkins.scm.DefaultSCMCheckoutStrategyImpl; import jenkins.scm.SCMCheckoutStrategy; import jenkins.scm.SCMCheckoutStrategyDescriptor; @@ -961,6 +963,51 @@ public abstract class AbstractProject

,R extends A this.builds.remove(run); } + /** + * {@inheritDoc} + * + * More efficient implementation. + */ + @Override + public R getBuild(String id) { + return builds.getById(id); + } + + /** + * {@inheritDoc} + * + * More efficient implementation. + */ + @Override + public R getBuildByNumber(int n) { + return builds.getByNumber(n); + } + + /** + * {@inheritDoc} + * + * More efficient implementation. + */ + @Override + public R getFirstBuild() { + return builds.oldestBuild(); + } + + @Override + public R getLastBuild() { + return builds.newestBuild(); + } + + @Override + public R getNearestBuild(int n) { + return builds.search(n, Direction.ASC); + } + + @Override + public R getNearestOldBuild(int n) { + return builds.search(n, Direction.DESC); + } + /** * Determines Class<R>. */ @@ -1632,7 +1679,7 @@ public abstract class AbstractProject

,R extends A @Override protected HistoryWidget createHistoryWidget() { - return new BuildHistoryWidget(this,getBuilds(),HISTORY_ADAPTER); + return new BuildHistoryWidget(this,builds,HISTORY_ADAPTER); } public boolean isParameterized() { diff --git a/core/src/main/java/hudson/model/Job.java b/core/src/main/java/hudson/model/Job.java index 998139c30c..f6bfa669cf 100644 --- a/core/src/main/java/hudson/model/Job.java +++ b/core/src/main/java/hudson/model/Job.java @@ -518,12 +518,21 @@ public abstract class Job, RunT extends Run getBuilds() { return RunList.fromRuns(_getRuns().values()); } + /** + * Gets the read-only view of the recent builds. + * + */ + @Exported(name="builds") + public RunList getNewBuilds() { + return getBuilds().newBuilds(); + } + /** * Obtains all the {@link Run}s whose build numbers matches the given {@link RangeSet}. */ @@ -600,7 +609,7 @@ public abstract class Job, RunT extends Run m = _getRuns().headMap(n - 1); // the map should // include n, so n-1 if (m.isEmpty()) @@ -614,7 +623,7 @@ public abstract class Job, RunT extends Run m = _getRuns().tailMap(n); if (m.isEmpty()) return null; @@ -626,7 +635,7 @@ public abstract class Job, RunT extends Run> extends AbstractLazyLoadRunMap { +public final class RunMap> extends AbstractLazyLoadRunMap implements Iterable { /** * Read-only view of this map. */ @@ -83,6 +84,33 @@ public final class RunMap> extends AbstractLazyLoadRunMap return removeValue(run); } + /** + * Walks through builds, newer ones first. + */ + public Iterator iterator() { + return new Iterator() { + R last = null; + R next = newestBuild(); + + public boolean hasNext() { + return next!=null; + } + + public R next() { + R last = next; + if (last!=null) + next = last.getPreviousBuild(); + return last; + } + + public void remove() { + if (last==null) + throw new UnsupportedOperationException(); + removeValue(last); + } + }; + } + @Override public boolean removeValue(R run) { if(run.nextBuild!=null) diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index 4e8ef4ba5b..8368ad975f 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -189,17 +189,25 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i } public Integer firstKey() { - R r = search(Integer.MAX_VALUE, DESC); + R r = newestBuild(); if (r==null) throw new NoSuchElementException(); return getNumberOf(r); } public Integer lastKey() { - R r = search(Integer.MIN_VALUE, ASC); + R r = oldestBuild(); if (r==null) throw new NoSuchElementException(); return getNumberOf(r); } + public R newestBuild() { + return search(Integer.MAX_VALUE, DESC); + } + + public R oldestBuild() { + return search(Integer.MIN_VALUE, ASC); + } + @Override public R get(Object key) { if (key instanceof Integer) { From e357135e9f60f1192587d31e6fc9953e1eea1e24 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Wed, 19 Sep 2012 16:43:02 -0700 Subject: [PATCH 38/69] Rewrote RunList to derive from List, not ArrayList. This is a signature breaking change, but it's critical in avoiding unnecessary record load operations. To my limited repository search, I couldn't find any use in a plugin where RunList is casted to ArrayList, so hopefully this would be safe enough. --- .../java/hudson/model/AbstractProject.java | 4 +- core/src/main/java/hudson/model/Job.java | 2 +- core/src/main/java/hudson/util/Iterators.java | 48 ++++ core/src/main/java/hudson/util/RunList.java | 247 ++++++++++++------ .../test/java/hudson/util/IteratorsTest.java | 19 +- 5 files changed, 238 insertions(+), 82 deletions(-) diff --git a/core/src/main/java/hudson/model/AbstractProject.java b/core/src/main/java/hudson/model/AbstractProject.java index 400072669a..07139d444d 100644 --- a/core/src/main/java/hudson/model/AbstractProject.java +++ b/core/src/main/java/hudson/model/AbstractProject.java @@ -954,8 +954,8 @@ public abstract class AbstractProject

,R extends A } @Override - public SortedMap _getRuns() { - return builds.getView(); + public RunMap _getRuns() { + return builds; } @Override diff --git a/core/src/main/java/hudson/model/Job.java b/core/src/main/java/hudson/model/Job.java index f6bfa669cf..f810b38974 100644 --- a/core/src/main/java/hudson/model/Job.java +++ b/core/src/main/java/hudson/model/Job.java @@ -669,7 +669,7 @@ public abstract class Job, RunT extends Run _getRuns(); diff --git a/core/src/main/java/hudson/util/Iterators.java b/core/src/main/java/hudson/util/Iterators.java index f01b2635ca..485598532d 100644 --- a/core/src/main/java/hudson/util/Iterators.java +++ b/core/src/main/java/hudson/util/Iterators.java @@ -23,8 +23,11 @@ */ package hudson.util; +import com.google.common.base.Objects; +import com.google.common.base.Predicate; import com.google.common.base.Predicates; +import javax.annotation.Nullable; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -326,4 +329,49 @@ public class Iterators { public static Iterator sequence(Iterator... iterators) { return com.google.common.collect.Iterators.concat(iterators); } + + /** + * Returns the elements in the base iterator until it hits any element that doesn't satisfy the filter. + * Then the rest of the elements in the base iterator gets ignored. + */ + public static Iterator limit(final Iterator base, final CountingPredicate filter) { + return new Iterator() { + private T next; + private boolean end; + private int index=0; + public boolean hasNext() { + fetch(); + return next!=null; + } + + public T next() { + fetch(); + T r = next; + next = null; + return r; + } + + private void fetch() { + if (next==null && !end) { + if (base.hasNext()) { + next = base.next(); + if (!filter.apply(index++,next)) { + next = null; + end = true; + } + } else { + end = true; + } + } + } + + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + + public interface CountingPredicate { + boolean apply(int index, T input); + } } diff --git a/core/src/main/java/hudson/util/RunList.java b/core/src/main/java/hudson/util/RunList.java index 67e9d50d89..98b2439863 100644 --- a/core/src/main/java/hudson/util/RunList.java +++ b/core/src/main/java/hudson/util/RunList.java @@ -23,6 +23,9 @@ */ package hudson.util; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterables; +import com.google.common.collect.Iterators; import hudson.model.AbstractBuild; import hudson.model.Item; import hudson.model.Job; @@ -30,7 +33,9 @@ import hudson.model.Node; import hudson.model.Result; import hudson.model.Run; import hudson.model.View; +import hudson.util.Iterators.CountingPredicate; +import javax.annotation.Nullable; import java.util.AbstractList; import java.util.ArrayList; import java.util.Calendar; @@ -48,79 +53,186 @@ import java.util.List; * * @author Kohsuke Kawaguchi */ -public class RunList extends ArrayList { +public class RunList extends AbstractList { + + private Iterable base; + + private R first; + private Integer size; + public RunList() { + base = Collections.emptyList(); } public RunList(Job j) { - addAll(j.getBuilds()); - } - - public R getFirstBuild() { - return isEmpty() ? null : get(size()-1); - } - - public R getLastBuild() { - return isEmpty() ? null : get(0); + base = j.getBuilds(); } public RunList(View view) {// this is a type unsafe operation + List> jobs = new ArrayList>(); for (Item item : view.getItems()) for (Job j : item.getAllJobs()) - addAll((Collection)j.getBuilds()); - Collections.sort(this,Run.ORDER_BY_DATE); + jobs.add(((Job)j).getBuilds()); + + this.base = combine(jobs); } public RunList(Collection jobs) { + List> src = new ArrayList>(); for (Job j : jobs) - addAll(j.getBuilds()); - Collections.sort(this,Run.ORDER_BY_DATE); + src.add(j.getBuilds()); + this.base = combine(src); } - private RunList(Collection c, boolean hack) { - super(c); + private Iterable combine(Iterable> jobs) { + return Iterables.mergeSorted(jobs, new Comparator() { + public int compare(R o1, R o2) { + long lhs = o1.getTimeInMillis(); + long rhs = o2.getTimeInMillis(); + if (lhs > rhs) return -1; + if (lhs < rhs) return 1; + return 0; + } + }); + } + + private RunList(Iterable c) { + base = c; + } + + /** + * @deprecated + * {@link RunList}, despite its name, should be really used as {@link Iterable}, not as {@link List}. + */ + @Override + public int size() { + if (size==null) { + int sz=0; + for (R r : this) { + first = r; + sz++; + } + size = sz; + } + return size; + } + + /** + * @deprecated + * {@link RunList}, despite its name, should be really used as {@link Iterable}, not as {@link List}. + */ + @Override + public R get(int index) { + return Iterators.get(iterator(),index); + } + + @Override + public int indexOf(Object o) { + int index=0; + for (R r : this) { + if (r.equals(o)) + return index; + index++; + } + return -1; + } + + @Override + public int lastIndexOf(Object o) { + int a = -1; + int index=0; + for (R r : this) { + if (r.equals(o)) + a = index; + index++; + } + return a; + } + + @Override + public boolean isEmpty() { + return !iterator().hasNext(); + } + + public R getFirstBuild() { + size(); + return first; + } + + public R getLastBuild() { + Iterator itr = iterator(); + return itr.hasNext() ? itr.next() : null; } public static RunList fromRuns(Collection runs) { - return new RunList(runs,false); + return new RunList((Iterable)runs); + } + + /** + * Returns elements that satisfy the given predicate. + */ + // for compatibility reasons, this method doesn't create a new list but updates the current one + private RunList filter(Predicate predicate) { + size = null; + first = null; + base = Iterables.filter(base,predicate); + return this; + } + + /** + * Returns the first streak of the elements that satisfy the given predicate. + * + * For example, {@code filter([1,2,3,4],odd)==[1,3]} but {@code limit([1,2,3,4],odd)==[1]}. + */ + private RunList limit(final CountingPredicate predicate) { + size = null; + first = null; + final Iterable nested = base; + base = new Iterable() { + public Iterator iterator() { + return hudson.util.Iterators.limit(nested.iterator(),predicate); + } + + @Override + public String toString() { + return Iterables.toString(this); + } + }; + return this; } /** * Filter the list to non-successful builds only. */ public RunList failureOnly() { - for (Iterator itr = iterator(); itr.hasNext();) { - Run r = itr.next(); - if(r.getResult()==Result.SUCCESS) - itr.remove(); - } - return this; + return filter(new Predicate() { + public boolean apply(R r) { + return r.getResult()!=Result.SUCCESS; + } + }); } /** * Filter the list to builds on a single node only */ - public RunList node(Node node) { - for (Iterator itr = iterator(); itr.hasNext();) { - Run r = itr.next(); - if (!(r instanceof AbstractBuild) || ((AbstractBuild)r).getBuiltOn()!=node) { - itr.remove(); + public RunList node(final Node node) { + return filter(new Predicate() { + public boolean apply(R r) { + return (r instanceof AbstractBuild) && ((AbstractBuild)r).getBuiltOn()==node; } - } - return this; + }); } /** * Filter the list to regression builds only. */ public RunList regressionOnly() { - for (Iterator itr = iterator(); itr.hasNext();) { - Run r = itr.next(); - if(!r.getBuildStatusSummary().isWorse) - itr.remove(); - } - return this; + return filter(new Predicate() { + public boolean apply(R r) { + return r.getBuildStatusSummary().isWorse; + } + }); } /** @@ -128,30 +240,17 @@ public class RunList extends ArrayList { * * {@code s<=;e}. */ - public RunList byTimestamp(long start, long end) { - AbstractList TIMESTAMP_ADAPTER = new AbstractList() { - public Long get(int index) { - return RunList.this.get(index).getTimeInMillis(); + public RunList byTimestamp(final long start, final long end) { + return + limit(new CountingPredicate() { + public boolean apply(int index,R r) { + return r.getTimeInMillis()() { + public boolean apply(R r) { + return start<=r.getTimeInMillis(); } - }; - Comparator DESCENDING_ORDER = new Comparator() { - public int compare(Long o1, Long o2) { - if (o1 > o2) return -1; - if (o1 < o2) return +1; - return 0; - } - }; - - int s = Collections.binarySearch(TIMESTAMP_ADAPTER, start, DESCENDING_ORDER); - if (s<0) s=-(s+1); // min is inclusive - int e = Collections.binarySearch(TIMESTAMP_ADAPTER, end, DESCENDING_ORDER); - if (e<0) e=-(e+1); else e++; // max is exclusive, so the exact match should be excluded - - return fromRuns(subList(e,s)); + }); } /** @@ -160,27 +259,21 @@ public class RunList extends ArrayList { * if it changes. */ public RunList newBuilds() { - GregorianCalendar threshold = new GregorianCalendar(); - threshold.add(Calendar.DAY_OF_YEAR,-7); + GregorianCalendar cal = new GregorianCalendar(); + cal.add(Calendar.DAY_OF_YEAR, -7); + final long t = cal.getTimeInMillis(); - int count=0; - - for (Iterator itr = iterator(); itr.hasNext();) { - R r = itr.next(); - if(r.isBuilding()) { - // can't publish on-going builds - itr.remove(); - continue; + // can't publish on-going builds + return filter(new Predicate() { + public boolean apply(R r) { + return !r.isBuilding(); } - // at least put 10 items - if(count<10) { - count++; - continue; + }) + // put at least 10 builds, but otherwise ignore old builds + .limit(new CountingPredicate() { + public boolean apply(int index, R r) { + return index < 10 || r.getTimeInMillis() >= t; } - // anything older than 7 days will be ignored - if(r.getTimestamp().before(threshold)) - itr.remove(); - } - return this; + }); } } diff --git a/core/src/test/java/hudson/util/IteratorsTest.java b/core/src/test/java/hudson/util/IteratorsTest.java index dc78d1c235..ec0a518f38 100644 --- a/core/src/test/java/hudson/util/IteratorsTest.java +++ b/core/src/test/java/hudson/util/IteratorsTest.java @@ -23,16 +23,20 @@ */ package hudson.util; +import hudson.util.Iterators.CountingPredicate; import junit.framework.TestCase; import java.util.List; +import static java.util.Arrays.*; + /** * @author Kohsuke Kawaguchi */ public class IteratorsTest extends TestCase { + public void testReverseSequence() { - List lst = Iterators.reverseSequence(1,4); + List lst = Iterators.reverseSequence(1, 4); assertEquals(3,(int)lst.get(0)); assertEquals(2,(int)lst.get(1)); assertEquals(1,(int)lst.get(2)); @@ -44,6 +48,17 @@ public class IteratorsTest extends TestCase { assertEquals(1,(int)lst.get(0)); assertEquals(2,(int)lst.get(1)); assertEquals(3,(int)lst.get(2)); - assertEquals(3,lst.size()); + assertEquals(3, lst.size()); } + + public void testLimit() { + assertEquals("[0]",com.google.common.collect.Iterators.toString(Iterators.limit(asList(0,1,2,3,4).iterator(), EVEN))); + assertEquals("[]", com.google.common.collect.Iterators.toString(Iterators.limit(asList(1,2,4,6).iterator(), EVEN))); + } + + public static final CountingPredicate EVEN = new CountingPredicate() { + public boolean apply(int index, Integer input) { + return input % 2 == 0; + } + }; } From b4888f2dd1a9cd1376f8b7b56b58426b51f0ccc9 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Thu, 20 Sep 2012 10:12:58 -0700 Subject: [PATCH 39/69] bug fix --- core/src/main/java/hudson/util/RunList.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/java/hudson/util/RunList.java b/core/src/main/java/hudson/util/RunList.java index 98b2439863..6748875476 100644 --- a/core/src/main/java/hudson/util/RunList.java +++ b/core/src/main/java/hudson/util/RunList.java @@ -100,6 +100,11 @@ public class RunList extends AbstractList { base = c; } + @Override + public Iterator iterator() { + return base.iterator(); + } + /** * @deprecated * {@link RunList}, despite its name, should be really used as {@link Iterable}, not as {@link List}. From 5b00127eab7043234474fc978df06d460bd59930 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Thu, 20 Sep 2012 11:31:05 -0700 Subject: [PATCH 40/69] Needs to copy the collection before it mutates them. --- .../main/java/hudson/tasks/LogRotator.java | 22 ++++++++++++------- .../java/hudson/tasks/LogRotatorTest.java | 1 - 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/hudson/tasks/LogRotator.java b/core/src/main/java/hudson/tasks/LogRotator.java index 8e1a154403..389f0d4565 100644 --- a/core/src/main/java/hudson/tasks/LogRotator.java +++ b/core/src/main/java/hudson/tasks/LogRotator.java @@ -23,22 +23,22 @@ */ package hudson.tasks; +import com.google.common.collect.Lists; import hudson.model.Describable; import hudson.model.Descriptor; import hudson.model.Job; import hudson.model.Run; - import org.kohsuke.stapler.DataBoundConstructor; import java.io.IOException; import java.util.Calendar; +import java.util.Collection; import java.util.GregorianCalendar; -import static java.util.logging.Level.FINE; -import static java.util.logging.Level.FINER; - import java.util.List; import java.util.logging.Logger; +import static java.util.logging.Level.*; + /** * Deletes old builds. * @@ -115,7 +115,7 @@ public class LogRotator implements Describable { if(numToKeep!=-1) { List> builds = job.getBuilds(); - for (Run r : builds.subList(Math.min(builds.size(),numToKeep),builds.size())) { + for (Run r : copy(builds.subList(Math.min(builds.size(), numToKeep), builds.size()))) { if (r.isKeepLog()) { LOGGER.log(FINER,r.getFullDisplayName()+" is not GC-ed because it's marked as a keeper"); continue; @@ -136,7 +136,7 @@ public class LogRotator implements Describable { if(daysToKeep!=-1) { Calendar cal = new GregorianCalendar(); cal.add(Calendar.DAY_OF_YEAR,-daysToKeep); - for( Run r : job.getBuilds() ) { + for( Run r : copy(job.getBuilds()) ) { if (r.isKeepLog()) { LOGGER.log(FINER,r.getFullDisplayName()+" is not GC-ed because it's marked as a keeper"); continue; @@ -160,7 +160,7 @@ public class LogRotator implements Describable { if(artifactNumToKeep!=null && artifactNumToKeep!=-1) { List> builds = job.getBuilds(); - for (Run r : builds.subList(Math.min(builds.size(),artifactNumToKeep),builds.size())) { + for (Run r : copy(builds.subList(Math.min(builds.size(), artifactNumToKeep), builds.size()))) { if (r.isKeepLog()) { LOGGER.log(FINER,r.getFullDisplayName()+" is not purged of artifacts because it's marked as a keeper"); continue; @@ -180,7 +180,7 @@ public class LogRotator implements Describable { if(artifactDaysToKeep!=null && artifactDaysToKeep!=-1) { Calendar cal = new GregorianCalendar(); cal.add(Calendar.DAY_OF_YEAR,-artifactDaysToKeep); - for( Run r : job.getBuilds() ) { + for( Run r : copy(job.getBuilds())) { if (r.isKeepLog()) { LOGGER.log(FINER,r.getFullDisplayName()+" is not purged of artifacts because it's marked as a keeper"); continue; @@ -200,7 +200,13 @@ public class LogRotator implements Describable { r.deleteArtifacts(); } } + } + /** + * Creates a copy since we'll be deleting some entries from them. + */ + private Collection copy(Iterable src) { + return Lists.newArrayList(src); } public int getDaysToKeep() { diff --git a/test/src/test/java/hudson/tasks/LogRotatorTest.java b/test/src/test/java/hudson/tasks/LogRotatorTest.java index 18a736a774..6dda4ba23a 100644 --- a/test/src/test/java/hudson/tasks/LogRotatorTest.java +++ b/test/src/test/java/hudson/tasks/LogRotatorTest.java @@ -27,7 +27,6 @@ package hudson.tasks; import hudson.Launcher; import hudson.model.AbstractBuild; import hudson.model.BuildListener; -import hudson.model.Cause.LegacyCodeCause; import hudson.model.Descriptor; import hudson.model.FreeStyleProject; import hudson.model.Result; From 516c4a7430f80d5a5afff3a3c55b9fe78790cf06 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Thu, 20 Sep 2012 13:15:40 -0700 Subject: [PATCH 41/69] leaving notes for future TODO --- core/src/main/java/hudson/util/RunList.java | 1 - .../main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/hudson/util/RunList.java b/core/src/main/java/hudson/util/RunList.java index 6748875476..a3c50846bd 100644 --- a/core/src/main/java/hudson/util/RunList.java +++ b/core/src/main/java/hudson/util/RunList.java @@ -35,7 +35,6 @@ import hudson.model.Run; import hudson.model.View; import hudson.util.Iterators.CountingPredicate; -import javax.annotation.Nullable; import java.util.AbstractList; import java.util.ArrayList; import java.util.Calendar; diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index 8368ad975f..fb0d9c42c0 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -166,6 +166,11 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i * Smallest build number-1 to be in the returned set (-1 because this is exclusive) */ public SortedMap subMap(Integer fromKey, Integer toKey) { + // TODO: if this method can produce a lazy map, that'd be wonderful + // because due to the lack of floor/ceil/higher/lower kind of methods + // to look up keys in SortedMap, various places of Jenkins rely on + // subMap+firstKey/lastKey combo. + R start = search(fromKey, DESC); if (start==null) return EMPTY_SORTED_MAP; From f8014d974c6eb9f30098851bd7eeeec1edd1928f Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Thu, 20 Sep 2012 14:05:53 -0700 Subject: [PATCH 42/69] Path might be via symlink that uses build number. --- core/src/main/java/hudson/model/Run.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/hudson/model/Run.java b/core/src/main/java/hudson/model/Run.java index 9c17f98518..ff35c740f9 100644 --- a/core/src/main/java/hudson/model/Run.java +++ b/core/src/main/java/hudson/model/Run.java @@ -324,7 +324,9 @@ public abstract class Run ,RunT extends Run Date: Thu, 20 Sep 2012 14:13:38 -0700 Subject: [PATCH 43/69] we can't let these fields persisted! --- core/src/main/java/hudson/model/AbstractBuild.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/hudson/model/AbstractBuild.java b/core/src/main/java/hudson/model/AbstractBuild.java index 02f9152cbe..92d6461d2f 100644 --- a/core/src/main/java/hudson/model/AbstractBuild.java +++ b/core/src/main/java/hudson/model/AbstractBuild.java @@ -171,7 +171,7 @@ public abstract class AbstractBuild

,R extends Abs return getParent(); } - private boolean previousBuildComputed, nextBuildComputed; + private transient boolean previousBuildComputed, nextBuildComputed; @Override public R getPreviousBuild() { From 8add09588179212832d08187854fc1e984efb61f Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Thu, 20 Sep 2012 14:13:49 -0700 Subject: [PATCH 44/69] added logging --- core/src/main/java/hudson/model/RunMap.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/src/main/java/hudson/model/RunMap.java b/core/src/main/java/hudson/model/RunMap.java index 7f0ac661c0..5a18028a23 100644 --- a/core/src/main/java/hudson/model/RunMap.java +++ b/core/src/main/java/hudson/model/RunMap.java @@ -39,6 +39,7 @@ import java.util.SortedMap; import java.util.logging.Level; import java.util.logging.Logger; +import static java.util.logging.Level.*; import static jenkins.model.lazy.AbstractLazyLoadRunMap.Direction.*; /** @@ -204,6 +205,8 @@ public final class RunMap> extends AbstractLazyLoadRunMap try { R b = cons.create(d); b.onLoad(); + if (LOGGER.isLoggable(FINE) || LOG_RETRIEVAL) + LOGGER.log(LOG_RETRIEVAL?INFO:FINE,"Loaded " + b.getFullDisplayName(),new Exception()); return b; } catch (IOException e) { LOGGER.log(Level.WARNING, "could not load " + d, e); @@ -233,4 +236,6 @@ public final class RunMap> extends AbstractLazyLoadRunMap } private static final Logger LOGGER = Logger.getLogger(RunMap.class.getName()); + + public static boolean LOG_RETRIEVAL = true; } From d11284b1522afbadfaa062c1f64a9176a4d168f0 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Thu, 20 Sep 2012 14:41:57 -0700 Subject: [PATCH 45/69] the registerService call can be slow. run these asynchronously --- core/src/main/java/hudson/DNSMultiCast.java | 84 +++++++++++---------- 1 file changed, 46 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/hudson/DNSMultiCast.java b/core/src/main/java/hudson/DNSMultiCast.java index d238a86daa..5740862751 100644 --- a/core/src/main/java/hudson/DNSMultiCast.java +++ b/core/src/main/java/hudson/DNSMultiCast.java @@ -1,6 +1,7 @@ package hudson; import jenkins.model.Jenkins; +import jenkins.model.Jenkins.MasterComputer; import javax.jmdns.JmDNS; import javax.jmdns.ServiceInfo; @@ -9,6 +10,7 @@ import java.io.IOException; import java.net.URL; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.Callable; import java.util.logging.Level; import java.util.logging.Logger; @@ -20,49 +22,55 @@ import java.util.logging.Logger; public class DNSMultiCast implements Closeable { private JmDNS jmdns; - public DNSMultiCast(Jenkins hudson) { + public DNSMultiCast(final Jenkins jenkins) { if (disabled) return; // escape hatch - try { - this.jmdns = JmDNS.create(); + // the registerService call can be slow. run these asynchronously + MasterComputer.threadPoolForRemoting.submit(new Callable() { + public Object call() { + try { + jmdns = JmDNS.create(); - Map props = new HashMap(); - String rootURL = hudson.getRootUrl(); - if (rootURL==null) return; + Map props = new HashMap(); + String rootURL = jenkins.getRootUrl(); + if (rootURL==null) return null; - props.put("url", rootURL); - try { - props.put("version",String.valueOf(Jenkins.getVersion())); - } catch (IllegalArgumentException e) { - // failed to parse the version number + props.put("url", rootURL); + try { + props.put("version",String.valueOf(Jenkins.getVersion())); + } catch (IllegalArgumentException e) { + // failed to parse the version number + } + + TcpSlaveAgentListener tal = jenkins.getTcpSlaveAgentListener(); + if (tal!=null) + props.put("slave-port",String.valueOf(tal.getPort())); + + props.put("server-id", Util.getDigestOf(jenkins.getSecretKey())); + + URL jenkins_url = new URL(rootURL); + int jenkins_port = jenkins_url.getPort(); + if (jenkins_port == -1) { + jenkins_port = 80; + } + if (jenkins_url.getPath().length() > 0) { + props.put("path", jenkins_url.getPath()); + } + + jmdns.registerService(ServiceInfo.create("_hudson._tcp.local.","jenkins", + jenkins_port,0,0,props)); // for backward compatibility + jmdns.registerService(ServiceInfo.create("_jenkins._tcp.local.","jenkins", + jenkins_port,0,0,props)); + + // Make Jenkins appear in Safari's Bonjour bookmarks + jmdns.registerService(ServiceInfo.create("_http._tcp.local.","Jenkins", + jenkins_port,0,0,props)); + } catch (IOException e) { + LOGGER.log(Level.WARNING,"Failed to advertise the service to DNS multi-cast",e); + } + return null; } - - TcpSlaveAgentListener tal = hudson.getTcpSlaveAgentListener(); - if (tal!=null) - props.put("slave-port",String.valueOf(tal.getPort())); - - props.put("server-id", Util.getDigestOf(hudson.getSecretKey())); - - URL jenkins_url = new URL(rootURL); - int jenkins_port = jenkins_url.getPort(); - if (jenkins_port == -1) { - jenkins_port = 80; - } - if (jenkins_url.getPath().length() > 0) { - props.put("path", jenkins_url.getPath()); - } - - jmdns.registerService(ServiceInfo.create("_hudson._tcp.local.","hudson", - jenkins_port,0,0,props)); // for backward compatibility - jmdns.registerService(ServiceInfo.create("_jenkins._tcp.local.","jenkins", - jenkins_port,0,0,props)); - - // Make Jenkins appear in Safari's Bonjour bookmarks - jmdns.registerService(ServiceInfo.create("_http._tcp.local.","Jenkins", - jenkins_port,0,0,props)); - } catch (IOException e) { - LOGGER.log(Level.WARNING,"Failed to advertise the service to DNS multi-cast",e); - } + }); } public void close() { From 43a31d8faee89c7320f422757fb3d13ccd7dafac Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Thu, 20 Sep 2012 14:43:55 -0700 Subject: [PATCH 46/69] Recording my notes --- note.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/note.txt b/note.txt index 5fd75f669d..3f887094ea 100644 --- a/note.txt +++ b/note.txt @@ -1,3 +1,5 @@ TODO: - remove Java6 dependency (or do we really?) - - ship it! \ No newline at end of file + +NOTE: + - permalinks, such as "last unstable build" can end up loading all the records if there's no build that has the said status. From 6043cdc97ec2ee8be0f22a71b88a7dc61aef3014 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Thu, 20 Sep 2012 15:06:53 -0700 Subject: [PATCH 47/69] doc improvements --- .../model/lazy/AbstractLazyLoadRunMap.java | 43 ++++++++++++++++--- .../java/jenkins/model/lazy/Boundary.java | 15 ++++++- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index fb0d9c42c0..6a9decbca8 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -23,7 +23,6 @@ */ package jenkins.model.lazy; -import hudson.model.Run; import org.kohsuke.accmod.Restricted; import org.kohsuke.accmod.restrictions.NoExternalUse; @@ -50,12 +49,42 @@ import static jenkins.model.lazy.Boundary.*; * {@link SortedMap} that keeps build records by their build numbers, in the descending order * (newer ones first.) * + *

* The main thing about this class is that it encapsulates the lazy loading logic. - * That is, - * think of subtype of {@link Run} + * That is, while this class looks and feels like a normal {@link SortedMap} from outside, + * it actually doesn't have every item in the map instantiated yet. As items in the map get + * requested, this class {@link #retrieve(File) retrieves them} on demand, one by one. * - * This implementation is in 2 states. An instance can be {@linkplain #fullyLoaded fully loaded} state, - * where everything that can be loaded gets loaded. Otherwise it's in a partially loaded state. + *

+ * The lookup is primarily done by using the build number as the key (hence the key type is {@link Integer}), + * but this class also provides look up based on {@linkplain #getIdOf(Object) the build ID}. + * + *

+ * This class makes the following assumption about the on-disk layout of the data: + * + *

    + *
  • Every build is stored in a directory, named after its ID. + *
  • ID and build number are in the consistent order. That is, + * if there are two builds #M and #N, {@code M>N <=> M.id > N.id}. + *
+ * + *

+ * On certain platforms, there are symbolic links named after build numbers that link to the build ID. + * If these are available, they are used as a hint to speed up the lookup. Otherwise + * we rely on the assumption above and perform a binary search to locate the build. + * (notice that we'll have to do linear search if we don't have the consistent ordering assumption, + * which robs the whole point of doing lazy loading.) + * + *

+ * Some of the {@link SortedMap} operations are weakly implemented. For example, + * {@link #size()} may be inaccurate because we only count the number of directories that look like + * build records, without checking if they are loadable. But these weaknesses aren't distinguishable + * from concurrent modifications, where another thread deletes a build while one thread iterates them. + * + *

+ * Some of the {@link SortedMap} operations are inefficiently implemented, by + * {@linkplain #all() loading all the build records eagerly}. We hope to replace + * these implementations by more efficient lazy-loading ones as we go. * *

* Object lock of {@code this} is used to make sure mutation occurs sequentially. @@ -65,7 +94,9 @@ import static jenkins.model.lazy.Boundary.*; * @author Kohsuke Kawaguchi */ public abstract class AbstractLazyLoadRunMap extends AbstractMap implements SortedMap { - + /** + * Used in {@link #all()} to quickly determine if we've already loaded everything. + */ private boolean fullyLoaded; /** diff --git a/core/src/main/java/jenkins/model/lazy/Boundary.java b/core/src/main/java/jenkins/model/lazy/Boundary.java index e4625b4cbd..00459523d2 100644 --- a/core/src/main/java/jenkins/model/lazy/Boundary.java +++ b/core/src/main/java/jenkins/model/lazy/Boundary.java @@ -25,7 +25,15 @@ package jenkins.model.lazy; /** * ceil/floor/lower/higher implementations - * based on the result from a binary search. + * that takes the return value of a binary search as an input. + * + *

+ * Consider a sorted array of int X={xi} and a binary search of p on it. + * this class provides likes of {@code ceil(X,p)} which is the smallest xi + * that still satisfies xi >= p. + * + * Simiarly, {@link #HIGHER} is the smallest xi + * that still satisfies xi > p. * * @author Kohsuke Kawaguchi */ @@ -42,9 +50,12 @@ enum Boundary { this.offsetOfInsertionPoint = offsetOfInsertionPoint; } + /** + * Computes the boundary value. + */ public int apply(int binarySearchOutput) { int r = binarySearchOutput; - if (r>=0) return r+offsetOfExactMatch; + if (r>=0) return r+offsetOfExactMatch; // if we had some x_i==p int ip = -(r+1); return ip+offsetOfInsertionPoint; From c15791c0926fbb522e6d12a1ac04acb0aaaf9b29 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Thu, 20 Sep 2012 15:10:04 -0700 Subject: [PATCH 48/69] doc improvements --- core/src/main/java/hudson/model/Job.java | 1 + core/src/main/java/hudson/model/RunMap.java | 6 +++--- core/src/main/java/hudson/util/Iterators.java | 2 ++ core/src/main/java/hudson/util/RunList.java | 6 ++---- .../java/jenkins/model/lazy/AbstractLazyLoadRunMap.java | 1 + 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/hudson/model/Job.java b/core/src/main/java/hudson/model/Job.java index f810b38974..182cb319c9 100644 --- a/core/src/main/java/hudson/model/Job.java +++ b/core/src/main/java/hudson/model/Job.java @@ -527,6 +527,7 @@ public abstract class Job, RunT extends Run getNewBuilds() { diff --git a/core/src/main/java/hudson/model/RunMap.java b/core/src/main/java/hudson/model/RunMap.java index 5a18028a23..36a9962aeb 100644 --- a/core/src/main/java/hudson/model/RunMap.java +++ b/core/src/main/java/hudson/model/RunMap.java @@ -65,7 +65,7 @@ public final class RunMap> extends AbstractLazyLoadRunMap /** - * @deprecated + * @deprecated as of 1.LAZYLOAD * Use {@link #RunMap(File, Constructor)}. */ public RunMap() { @@ -144,7 +144,7 @@ public final class RunMap> extends AbstractLazyLoadRunMap } /** - * @deprecated + * @deprecated as of 1.LAZYLOAD * Use {@link ReverseComparator} */ public static final Comparator COMPARATOR = new Comparator() { @@ -227,7 +227,7 @@ public final class RunMap> extends AbstractLazyLoadRunMap * Job that owns this map. * @param cons * Used to create new instance of {@link Run}. - * @deprecated + * @deprecated as of 1.LAZYLOAD * Use {@link #RunMap(File, Constructor)} */ public void load(Job job, Constructor cons) { diff --git a/core/src/main/java/hudson/util/Iterators.java b/core/src/main/java/hudson/util/Iterators.java index 485598532d..b00540e9c5 100644 --- a/core/src/main/java/hudson/util/Iterators.java +++ b/core/src/main/java/hudson/util/Iterators.java @@ -333,6 +333,8 @@ public class Iterators { /** * Returns the elements in the base iterator until it hits any element that doesn't satisfy the filter. * Then the rest of the elements in the base iterator gets ignored. + * + * @since 1.LAZYLOAD */ public static Iterator limit(final Iterator base, final CountingPredicate filter) { return new Iterator() { diff --git a/core/src/main/java/hudson/util/RunList.java b/core/src/main/java/hudson/util/RunList.java index a3c50846bd..60166d9e71 100644 --- a/core/src/main/java/hudson/util/RunList.java +++ b/core/src/main/java/hudson/util/RunList.java @@ -48,8 +48,6 @@ import java.util.List; /** * {@link List} of {@link Run}s, sorted in the descending date order. * - * TODO: this should be immutable - * * @author Kohsuke Kawaguchi */ public class RunList extends AbstractList { @@ -105,7 +103,7 @@ public class RunList extends AbstractList { } /** - * @deprecated + * @deprecated as of 1.LAZYLOAD * {@link RunList}, despite its name, should be really used as {@link Iterable}, not as {@link List}. */ @Override @@ -122,7 +120,7 @@ public class RunList extends AbstractList { } /** - * @deprecated + * @deprecated as of 1.LAZYLOAD * {@link RunList}, despite its name, should be really used as {@link Iterable}, not as {@link List}. */ @Override diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index 6a9decbca8..bf831de2ef 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -92,6 +92,7 @@ import static jenkins.model.lazy.Boundary.*; * updating {@link #byNumber} and {@link #byId}. * * @author Kohsuke Kawaguchi + * @since 1.LAZYLOAD */ public abstract class AbstractLazyLoadRunMap extends AbstractMap implements SortedMap { /** From eb8818641ee1b7717c43378322164f90fd6e0a32 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Thu, 20 Sep 2012 16:17:20 -0700 Subject: [PATCH 49/69] avoid recursinv the history forever --- .../src/main/java/hudson/tasks/Fingerprinter.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/hudson/tasks/Fingerprinter.java b/core/src/main/java/hudson/tasks/Fingerprinter.java index d94bc0f0cc..752d10d3f0 100644 --- a/core/src/main/java/hudson/tasks/Fingerprinter.java +++ b/core/src/main/java/hudson/tasks/Fingerprinter.java @@ -340,11 +340,16 @@ public class Fingerprinter extends Recorder implements Serializable, DependecyDe } public void onLoad() { - Run pb = build.getPreviousBuild(); - if (pb!=null) { - FingerprintAction a = pb.getAction(FingerprintAction.class); - if (a!=null) - compact(a); + // share data structure with nearby builds, but to keep lazy loading efficient, + // don't go back the history forever. By %4!=0, 4 neighboring builds will share + // the data structure, so we'll get good enough saving. + if (build.getNumber()%4!=0) { + Run pb = build.getPreviousBuild(); + if (pb!=null) { + FingerprintAction a = pb.getAction(FingerprintAction.class); + if (a!=null) + compact(a); + } } } From 1bc181a9ebf2d43bd0132afa28e70b629599e5f3 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Thu, 20 Sep 2012 16:18:05 -0700 Subject: [PATCH 50/69] Exception looks bad to admins --- core/src/main/java/hudson/model/RunMap.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/hudson/model/RunMap.java b/core/src/main/java/hudson/model/RunMap.java index 36a9962aeb..e046e3e6c4 100644 --- a/core/src/main/java/hudson/model/RunMap.java +++ b/core/src/main/java/hudson/model/RunMap.java @@ -206,7 +206,7 @@ public final class RunMap> extends AbstractLazyLoadRunMap R b = cons.create(d); b.onLoad(); if (LOGGER.isLoggable(FINE) || LOG_RETRIEVAL) - LOGGER.log(LOG_RETRIEVAL?INFO:FINE,"Loaded " + b.getFullDisplayName(),new Exception()); + LOGGER.log(LOG_RETRIEVAL?INFO:FINE,"Loaded " + b.getFullDisplayName(),new ThisIsHowItsLoaded()); return b; } catch (IOException e) { LOGGER.log(Level.WARNING, "could not load " + d, e); @@ -238,4 +238,6 @@ public final class RunMap> extends AbstractLazyLoadRunMap private static final Logger LOGGER = Logger.getLogger(RunMap.class.getName()); public static boolean LOG_RETRIEVAL = true; + + private static class ThisIsHowItsLoaded extends Exception {} } From 9e6d8f71ce3f5844b31ed5055ac1354d150edf1f Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Fri, 21 Sep 2012 09:12:50 -0700 Subject: [PATCH 51/69] build numbers might exhibit some uniformity that makes it problematic with %4==0, so switching to random. The average memory saving should be about the same. --- core/src/main/java/hudson/tasks/Fingerprinter.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/hudson/tasks/Fingerprinter.java b/core/src/main/java/hudson/tasks/Fingerprinter.java index 752d10d3f0..fd81e69999 100644 --- a/core/src/main/java/hudson/tasks/Fingerprinter.java +++ b/core/src/main/java/hudson/tasks/Fingerprinter.java @@ -70,6 +70,7 @@ import java.util.List; import java.util.ListIterator; import java.util.Map; import java.util.Map.Entry; +import java.util.Random; import java.util.Set; import java.util.TreeMap; import java.util.logging.Level; @@ -295,6 +296,8 @@ public class Fingerprinter extends Recorder implements Serializable, DependecyDe private final AbstractBuild build; + private static final Random rand = new Random(); + /** * From file name to the digest. */ @@ -341,9 +344,9 @@ public class Fingerprinter extends Recorder implements Serializable, DependecyDe public void onLoad() { // share data structure with nearby builds, but to keep lazy loading efficient, - // don't go back the history forever. By %4!=0, 4 neighboring builds will share + // don't go back the history forever. By RAND!=0, 4 neighboring builds will share // the data structure, so we'll get good enough saving. - if (build.getNumber()%4!=0) { + if (rand.nextInt(4)!=0) { Run pb = build.getPreviousBuild(); if (pb!=null) { FingerprintAction a = pb.getAction(FingerprintAction.class); From c32f006fe40d424309c09f6d4291b210ec5fc1c1 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Fri, 21 Sep 2012 12:28:26 -0700 Subject: [PATCH 52/69] Allow AbstractBuilds to be garbage collected (and reloaded lazy later) --- .../main/java/hudson/model/AbstractBuild.java | 95 ++++- core/src/main/java/hudson/model/Run.java | 18 +- core/src/main/java/hudson/model/RunMap.java | 21 +- .../model/lazy/AbstractLazyLoadRunMap.java | 91 +++-- .../jenkins/model/lazy/BuildReference.java | 41 ++ .../model/lazy/BuildReferenceMapAdapter.java | 355 ++++++++++++++++++ 6 files changed, 567 insertions(+), 54 deletions(-) create mode 100644 core/src/main/java/jenkins/model/lazy/BuildReference.java create mode 100644 core/src/main/java/jenkins/model/lazy/BuildReferenceMapAdapter.java diff --git a/core/src/main/java/hudson/model/AbstractBuild.java b/core/src/main/java/hudson/model/AbstractBuild.java index 92d6461d2f..6b7d7e2e1e 100644 --- a/core/src/main/java/hudson/model/AbstractBuild.java +++ b/core/src/main/java/hudson/model/AbstractBuild.java @@ -63,6 +63,7 @@ import hudson.util.LogTaskListener; import hudson.util.VariableResolver; import jenkins.model.Jenkins; import jenkins.model.lazy.AbstractLazyLoadRunMap.Direction; +import jenkins.model.lazy.BuildReference; import org.kohsuke.stapler.Stapler; import org.kohsuke.stapler.StaplerRequest; import org.kohsuke.stapler.StaplerResponse; @@ -155,6 +156,20 @@ public abstract class AbstractBuild

,R extends Abs */ protected transient List buildEnvironments; + /** + * Pointers to form bi-directional link between adjacent {@link AbstractBuild}s. + * + *

+ * Unlike {@link Run}, {@link AbstractBuild}s do lazy-loading, so we don't use + * {@link Run#previousBuild} and {@link Run#nextBuild}, and instead use these + * fields and point to {@link #selfReference} of adjacent builds. + */ + private volatile transient BuildReference previousBuild, nextBuild; + + /*package*/ final transient BuildReference selfReference = new BuildReference(getId(),_this()); + + + protected AbstractBuild(P job) throws IOException { super(job); } @@ -171,32 +186,78 @@ public abstract class AbstractBuild

,R extends Abs return getParent(); } - private transient boolean previousBuildComputed, nextBuildComputed; + @Override + void dropLinks() { + super.dropLinks(); + + if(nextBuild!=null) { + AbstractBuild nb = nextBuild.get(); + if (nb!=null) nb.previousBuild = previousBuild; + } + if(previousBuild!=null) { + AbstractBuild pb = previousBuild.get(); + if (pb!=null) pb.nextBuild = nextBuild; + } + } @Override public R getPreviousBuild() { - if (previousBuild==null && !previousBuildComputed) { - // having two neighbors pointing to each other is important to make RunMap.removeValue work - R previousBuild = getParent().builds.search(number-1, Direction.DESC); - if (previousBuild!=null) - previousBuild.nextBuild = (R)this; - this.previousBuild = previousBuild; - previousBuildComputed = true; + while (true) { + BuildReference r = previousBuild; // capture the value once + + if (r==null) { + // having two neighbors pointing to each other is important to make RunMap.removeValue work + R pb = getParent().builds.search(number-1, Direction.DESC); + if (pb!=null) { + ((AbstractBuild)pb).nextBuild = selfReference; // establish bi-di link + this.previousBuild = pb.selfReference; + return pb; + } else { + // this indicates that we know there's no previous build + // (as opposed to we don't know if/what our previous build is. + this.previousBuild = selfReference; + return null; + } + } + if (r==selfReference) + return null; + + R referent = r.get(); + if (referent!=null) return referent; + + // the reference points to a GC-ed object, drop the reference and do it again + this.previousBuild = null; } - return previousBuild; } @Override public R getNextBuild() { - if (nextBuild==null && !nextBuildComputed) { - // having two neighbors pointing to each other is important to make RunMap.removeValue work - R nextBuild = getParent().builds.search(number+1, Direction.ASC); - if (nextBuild!=null) - nextBuild.previousBuild = (R)this; - this.nextBuild = nextBuild; - nextBuildComputed = true; + while (true) { + BuildReference r = nextBuild; // capture the value once + + if (r==null) { + // having two neighbors pointing to each other is important to make RunMap.removeValue work + R nb = getParent().builds.search(number+1, Direction.ASC); + if (nb!=null) { + ((AbstractBuild)nb).previousBuild = selfReference; // establish bi-di link + this.nextBuild = nb.selfReference; + return nb; + } else { + // this indicates that we know there's no next build + // (as opposed to we don't know if/what our next build is. + this.nextBuild = selfReference; + return null; + } + } + if (r==selfReference) + return null; + + R referent = r.get(); + if (referent!=null) return referent; + + // the reference points to a GC-ed object, drop the reference and do it again + this.nextBuild = null; } - return nextBuild; } /** diff --git a/core/src/main/java/hudson/model/Run.java b/core/src/main/java/hudson/model/Run.java index ff35c740f9..e5043378ae 100644 --- a/core/src/main/java/hudson/model/Run.java +++ b/core/src/main/java/hudson/model/Run.java @@ -1,9 +1,9 @@ /* * The MIT License * - * Copyright (c) 2004-2010, Sun Microsystems, Inc., Kohsuke Kawaguchi, + * Copyright (c) 2004-2012, Sun Microsystems, Inc., Kohsuke Kawaguchi, * Daniel Dyer, Red Hat, Inc., Tom Huybrechts, Romain Seguy, Yahoo! Inc., - * Darek Ostolski + * Darek Ostolski, CloudBees, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -96,6 +96,7 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletResponse; import jenkins.model.Jenkins; +import jenkins.model.lazy.BuildReference; import jenkins.util.io.OnMaster; import net.sf.json.JSONObject; import org.apache.commons.io.input.NullInputStream; @@ -338,7 +339,7 @@ public abstract class Run ,RunT extends Run,RunT extends Run> extends AbstractLazyLoadRunMap implements Iterable { /** * Read-only view of this map. @@ -114,11 +117,7 @@ public final class RunMap> extends AbstractLazyLoadRunMap @Override public boolean removeValue(R run) { - if(run.nextBuild!=null) - run.nextBuild.previousBuild = run.previousBuild; - if(run.previousBuild!=null) - run.previousBuild.nextBuild = run.nextBuild; - + run.dropLinks(); return super.removeValue(run); } @@ -170,6 +169,18 @@ public final class RunMap> extends AbstractLazyLoadRunMap return r.getId(); } + /** + * Reuses the same reference as much as we can. + *

+ * If concurrency ends up creating a few extra, that's OK, because + * we are really just trying to reduce the # of references we create. + */ + @Override + protected BuildReference createReference(R r) { + if (r instanceof AbstractBuild) return ((AbstractBuild)r).selfReference; + else return super.createReference(r); + } + @Override protected FilenameFilter createDirectoryFilter() { final SimpleDateFormat formatter = Run.ID_FORMATTER.get(); diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index bf831de2ef..57c788daff 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -23,12 +23,14 @@ */ package jenkins.model.lazy; +import hudson.model.Run; import org.kohsuke.accmod.Restricted; import org.kohsuke.accmod.restrictions.NoExternalUse; import java.io.File; import java.io.FilenameFilter; import java.io.IOException; +import java.lang.ref.Reference; import java.util.AbstractMap; import java.util.ArrayList; import java.util.Arrays; @@ -104,13 +106,13 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i * Stores the mapping from build number to build, for builds that are already loaded. */ // copy on write - private volatile TreeMap byNumber = new TreeMap(COMPARATOR); + private volatile TreeMap> byNumber = new TreeMap>(COMPARATOR); /** * Stores the build ID to build number for builds that we already know */ // copy on write - private volatile TreeMap byId = new TreeMap(); + private volatile TreeMap> byId = new TreeMap>(); /** * Build IDs found as directories, in the ascending order. @@ -181,14 +183,14 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i @Override public Set> entrySet() { - return Collections.unmodifiableSet(all().entrySet()); + return Collections.unmodifiableSet(new BuildReferenceMapAdapter(this,all()).entrySet()); } /** * Returns a read-only view of records that has already been loaded. */ public SortedMap getLoadedBuilds() { - return Collections.unmodifiableSortedMap(byNumber); + return Collections.unmodifiableSortedMap(new BuildReferenceMapAdapter(this,byNumber)); } /** @@ -214,7 +216,7 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i assert i!=null; } - return Collections.unmodifiableSortedMap(byNumber.subMap(fromKey, toKey)); + return Collections.unmodifiableSortedMap(new BuildReferenceMapAdapter(this,byNumber.subMap(fromKey, toKey))); } public SortedMap headMap(Integer toKey) { @@ -273,8 +275,12 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i * If DESC, finds the closest #M that satisfies M<=N. */ public R search(final int n, final Direction d) { - Entry c = byNumber.ceilingEntry(n); - if (c!=null && c.getKey()== n) return c.getValue(); // found the exact #n + Entry> c = byNumber.ceilingEntry(n); + if (c!=null && c.getKey()== n) { + R r = c.getValue().get(); + if (r!=null) + return r; // found the exact #n + } // at this point we know that we don't have #n loaded yet @@ -332,11 +338,11 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i // first, narrow down the candidate IDs to try by using two known number-to-ID mapping if (idOnDisk.isEmpty()) return null; - Entry f = byNumber.floorEntry(n); + Entry> f = byNumber.floorEntry(n); // if bound is null, use a sentinel value - String cid = c==null ? "\u0000" : getIdOf(c.getValue()); - String fid = f==null ? "\uFFFF" : getIdOf(f.getValue()); + String cid = c==null ? "\u0000" : c.getValue().id; + String fid = f==null ? "\uFFFF" : f.getValue().id; // We know that the build we are looking for exists in this range // we will narrow this down via binary search @@ -390,8 +396,9 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i } public R getById(String id) { - if (byId.containsKey(id)) - return byId.get(id); + if (byId.containsKey(id)) { + return unwrap(byId.get(id)); + } return load(id,true); } @@ -409,11 +416,12 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i int n = getNumberOf(r); copy(); - R old = byId.put(id,r); - byNumber.put(n,r); + BuildReference ref = createReference(r); + BuildReference old = byId.put(id,ref); + byNumber.put(n,ref); /* - search relies on the fact that every objet added via + search relies on the fact that every object added via put() method be available in the xyzOnDisk index, so I'm adding them here however, this is awfully inefficient. I wonder if there's any better way to do this? */ @@ -431,15 +439,21 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i numberOnDisk = a; } - return old; + return unwrap(old); + } + + private R unwrap(Reference ref) { + return ref!=null ? ref.get() : null; } @Override public synchronized void putAll(Map rhs) { copy(); for (R r : rhs.values()) { - byId.put(getIdOf(r),r); - byNumber.put(getNumberOf(r),r); + String id = getIdOf(r); + BuildReference ref = createReference(r); + byId.put(id,ref); + byNumber.put(getNumberOf(r),ref); } } @@ -452,7 +466,7 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i * @return * fully populated map. */ - private TreeMap all() { + private TreeMap> all() { if (!fullyLoaded) { synchronized (this) { if (!fullyLoaded) { @@ -472,8 +486,8 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i * Creates a duplicate for the COW data structure in preparation for mutation. */ private void copy() { - byId = new TreeMap(byId); - byNumber = new TreeMap(byNumber); + byId = new TreeMap>(byId); + byNumber = new TreeMap>(byNumber); } /** @@ -515,8 +529,11 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i if (r==null) return null; if (copy) copy(); - byId.put(getIdOf(r),r); - byNumber.put(getNumberOf(r),r); + + String id = getIdOf(r); + BuildReference ref = createReference(r); + byId.put(id,ref); + byNumber.put(getNumberOf(r),ref); return r; } catch (IOException e) { LOGGER.log(Level.WARNING, "Failed to load "+dataDir,e); @@ -524,9 +541,23 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i return null; } + /** + * Subtype to provide {@link Run#getNumber()} so that this class doesn't have to depend on it. + */ protected abstract int getNumberOf(R r); + /** + * Subtype to provide {@link Run#getId()} so that this class doesn't have to depend on it. + */ protected abstract String getIdOf(R r); + /** + * Allow subtype to capture a reference. + */ + protected BuildReference createReference(R r) { + return new BuildReference(getIdOf(r),r); + } + + /** * Parses {@code R} instance from data in the specified directory. * @@ -541,19 +572,21 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i public synchronized boolean removeValue(R run) { copy(); byNumber.remove(getNumberOf(run)); - R old = byId.remove(getIdOf(run)); - return old!=null; + BuildReference old = byId.remove(getIdOf(run)); + return unwrap(old)!=null; } /** * Replaces all the current loaded Rs with the given ones. */ public synchronized void reset(TreeMap builds) { - TreeMap byNumber = new TreeMap(COMPARATOR); - TreeMap byId = new TreeMap(COMPARATOR); + TreeMap> byNumber = new TreeMap>(COMPARATOR); + TreeMap> byId = new TreeMap>(COMPARATOR); for (R r : builds.values()) { - byId.put(getIdOf(r),r); - byNumber.put(getNumberOf(r),r); + String id = getIdOf(r); + BuildReference ref = createReference(r); + byId.put(id,ref); + byNumber.put(getNumberOf(r),ref); } this.byNumber = byNumber; diff --git a/core/src/main/java/jenkins/model/lazy/BuildReference.java b/core/src/main/java/jenkins/model/lazy/BuildReference.java new file mode 100644 index 0000000000..eb437164b4 --- /dev/null +++ b/core/src/main/java/jenkins/model/lazy/BuildReference.java @@ -0,0 +1,41 @@ +package jenkins.model.lazy; + +import java.lang.ref.SoftReference; + +/** + * {@link SoftReference} to a build object. + * + *

+ * To be able to re-retrieve the referent in case it is lost, this class + * remembers its ID (the job name is provided by the context because a {@link BuildReference} + * belongs to one and only {@link AbstractLazyLoadRunMap}.) + * + *

+ * We use this ID for equality/hashCode so that we can have a collection of {@link BuildReference} + * and find things in it. + * + * @author Kohsuke Kawaguchi + */ +public final class BuildReference extends SoftReference { + final String id; + + public BuildReference(String id, R referent) { + super(referent); + this.id = id; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + BuildReference that = (BuildReference) o; + return id.equals(that.id); + + } + + @Override + public int hashCode() { + return id.hashCode(); + } +} diff --git a/core/src/main/java/jenkins/model/lazy/BuildReferenceMapAdapter.java b/core/src/main/java/jenkins/model/lazy/BuildReferenceMapAdapter.java new file mode 100644 index 0000000000..50ddc5df4b --- /dev/null +++ b/core/src/main/java/jenkins/model/lazy/BuildReferenceMapAdapter.java @@ -0,0 +1,355 @@ +package jenkins.model.lazy; + +import groovy.util.MapEntry; +import hudson.util.AdaptedIterator; + +import javax.annotation.Nullable; +import java.lang.reflect.Array; +import java.util.Collection; +import java.util.Comparator; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; + +/** + * Take {@code SortedMap>} and make it look like {@code SortedMap}. + * + * When {@link BuildReference} lost the build object, we'll use {@link AbstractLazyLoadRunMap#getById(String)} + * to obtain one. + * + * @author Kohsuke Kawaguchi + */ +class BuildReferenceMapAdapter implements SortedMap { + private final AbstractLazyLoadRunMap loader; + + private final SortedMap> core; + + BuildReferenceMapAdapter(AbstractLazyLoadRunMap loader, SortedMap> core) { + this.loader = loader; + this.core = core; + } + + private R unwrap(@Nullable BuildReference ref) { + if (ref==null) return null; + + R v = ref.get(); + if (v==null) + v = loader.getById(ref.id); + return v; + } + + private BuildReference wrap(@Nullable R value) { + if (value==null) return null; + return loader.createReference(value); + } + + + + + public Comparator comparator() { + return core.comparator(); + } + + public SortedMap subMap(Integer fromKey, Integer toKey) { + return new BuildReferenceMapAdapter(loader,core.subMap(fromKey, toKey)); + } + + public SortedMap headMap(Integer toKey) { + return new BuildReferenceMapAdapter(loader,core.headMap(toKey)); + } + + public SortedMap tailMap(Integer fromKey) { + return new BuildReferenceMapAdapter(loader,core.tailMap(fromKey)); + } + + public Integer firstKey() { + return core.firstKey(); + } + + public Integer lastKey() { + return core.lastKey(); + } + + public Set keySet() { + return core.keySet(); + } + + public Collection values() { + return new CollectionAdapter(core.values()); + } + + public Set> entrySet() { + return new SetAdapter(core.entrySet()); + } + + public int size() { + return core.size(); + } + + public boolean isEmpty() { + return core.isEmpty(); + } + + public boolean containsKey(Object key) { + return core.containsKey(key); + } + + public boolean containsValue(Object value) { + return core.containsValue(value); + } + + public R get(Object key) { + return unwrap(core.get(key)); + } + + public R put(Integer key, R value) { + return unwrap(core.put(key, wrap(value))); + } + + public R remove(Object key) { + return unwrap(core.remove(key)); + } + + public void putAll(Map m) { + for (Entry e : m.entrySet()) + put(e.getKey(), e.getValue()); + } + + public void clear() { + core.clear(); + } + + @Override + public boolean equals(Object o) { + return core.equals(o); + } + + @Override + public int hashCode() { + return core.hashCode(); + } + + private class CollectionAdapter implements Collection { + private final Collection> core; + + private CollectionAdapter(Collection> core) { + this.core = core; + } + + public int size() { + return core.size(); + } + + public boolean isEmpty() { + return core.isEmpty(); + } + + public boolean contains(Object o) { + // TODO: to properly pass this onto core, we need to wrap o into BuildReference but also needs to figure out ID. + throw new UnsupportedOperationException(); + } + + public Iterator iterator() { + return new AdaptedIterator,R>(core.iterator()) { + protected R adapt(BuildReference ref) { + return unwrap(ref); + } + }; + } + + public Object[] toArray() { + return _unwrap(core.toArray()); + } + + public T[] toArray(T[] a) { + int size = size(); + T[] r = a; + if (r.length>size) + r = (T[]) Array.newInstance(a.getClass().getComponentType(), size); + + Iterator itr = iterator(); + int i=0; + + while (itr.hasNext()) { + r[i++] = (T)itr.next(); + } + + return r; + } + + public boolean add(R value) { + return core.add(wrap(value)); + } + + public boolean remove(Object o) { +// return core.remove(o); + // TODO: to properly pass this onto core, we need to wrap o into BuildReference but also needs to figure out ID. + throw new UnsupportedOperationException(); + } + + public boolean containsAll(Collection c) { + for (Object o : c) { + if (!contains(o)) + return false; + } + return true; + } + + public boolean addAll(Collection c) { + boolean b=false; + for (R r : c) { + b |= add(r); + } + return b; + } + + public boolean removeAll(Collection c) { + boolean b=false; + for (Object o : c) { + b|=remove(o); + } + return b; + } + + public boolean retainAll(Collection c) { + // TODO: to properly pass this onto core, we need to wrap o into BuildReference but also needs to figure out ID. + throw new UnsupportedOperationException(); + } + + public void clear() { + core.clear(); + } + + @Override + public boolean equals(Object o) { + return core.equals(o); + } + + @Override + public int hashCode() { + return core.hashCode(); + } + + private Object[] _unwrap(Object[] r) { + for (int i=0; i) r[i]); + return r; + } + } + + private class SetAdapter implements Set> { + private final Set>> core; + + private SetAdapter(Set>> core) { + this.core = core; + } + + public int size() { + return core.size(); + } + + public boolean isEmpty() { + return core.isEmpty(); + } + + public boolean contains(Object o) { + // TODO: to properly pass this onto core, we need to wrap o into BuildReference but also needs to figure out ID. + throw new UnsupportedOperationException(); + } + + public Iterator> iterator() { + return new AdaptedIterator>,Entry>(core.iterator()) { + protected Entry adapt(Entry> e) { + return _unwrap(e); + } + }; + } + + public Object[] toArray() { + return _unwrap(core.toArray()); + } + + public T[] toArray(T[] a) { + int size = size(); + T[] r = a; + if (r.length>size) + r = (T[]) Array.newInstance(a.getClass().getComponentType(), size); + + Iterator> itr = iterator(); + int i=0; + + while (itr.hasNext()) { + r[i++] = (T)itr.next(); + } + + return r; + } + + public boolean add(Entry value) { + return core.add(_wrap(value)); + } + + public boolean remove(Object o) { +// return core.remove(o); + // TODO: to properly pass this onto core, we need to wrap o into BuildReference but also needs to figure out ID. + throw new UnsupportedOperationException(); + } + + public boolean containsAll(Collection c) { + for (Object o : c) { + if (!contains(o)) + return false; + } + return true; + } + + public boolean addAll(Collection> c) { + boolean b=false; + for (Entry r : c) { + b |= add(r); + } + return b; + } + + public boolean removeAll(Collection c) { + boolean b=false; + for (Object o : c) { + b|=remove(o); + } + return b; + } + + public boolean retainAll(Collection c) { + // TODO: to properly pass this onto core, we need to wrap o into BuildReference but also needs to figure out ID. + throw new UnsupportedOperationException(); + } + + public void clear() { + core.clear(); + } + + @Override + public boolean equals(Object o) { + return core.equals(o); + } + + @Override + public int hashCode() { + return core.hashCode(); + } + + private Entry> _wrap(Entry e) { + return new MapEntry(e.getKey(),wrap(e.getValue())); + } + private Entry _unwrap(Entry> e) { + return new MapEntry(e.getKey(),unwrap(e.getValue())); + } + + private Object[] _unwrap(Object[] r) { + for (int i=0; i Date: Fri, 21 Sep 2012 13:35:11 -0700 Subject: [PATCH 53/69] don't do this by default --- core/src/main/java/hudson/model/RunMap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/hudson/model/RunMap.java b/core/src/main/java/hudson/model/RunMap.java index 77c918a821..5ca1004465 100644 --- a/core/src/main/java/hudson/model/RunMap.java +++ b/core/src/main/java/hudson/model/RunMap.java @@ -248,7 +248,7 @@ public final class RunMap> extends AbstractLazyLoadRunMap private static final Logger LOGGER = Logger.getLogger(RunMap.class.getName()); - public static boolean LOG_RETRIEVAL = true; + public static boolean LOG_RETRIEVAL = false; private static class ThisIsHowItsLoaded extends Exception {} } From de15893cc4a4aa90be27567f7093f8e672dac8ab Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Fri, 21 Sep 2012 14:57:07 -0700 Subject: [PATCH 54/69] tweaking the co-efficient --- core/src/main/java/hudson/tasks/Fingerprinter.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/hudson/tasks/Fingerprinter.java b/core/src/main/java/hudson/tasks/Fingerprinter.java index fd81e69999..6600bd4aa0 100644 --- a/core/src/main/java/hudson/tasks/Fingerprinter.java +++ b/core/src/main/java/hudson/tasks/Fingerprinter.java @@ -344,9 +344,8 @@ public class Fingerprinter extends Recorder implements Serializable, DependecyDe public void onLoad() { // share data structure with nearby builds, but to keep lazy loading efficient, - // don't go back the history forever. By RAND!=0, 4 neighboring builds will share - // the data structure, so we'll get good enough saving. - if (rand.nextInt(4)!=0) { + // don't go back the history forever. + if (rand.nextInt(2)!=0) { Run pb = build.getPreviousBuild(); if (pb!=null) { FingerprintAction a = pb.getAction(FingerprintAction.class); From 1f38518f530ec9e8f0e4e5b9e9d9aa7e3afc8468 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Fri, 21 Sep 2012 15:38:14 -0700 Subject: [PATCH 55/69] Fixing concurrency problems. The following bug report from olamy made me look hard at the code, but I'm not spotting an obvious problem. The binary search algorithm here seems to always satisfy the invariant of 0<=lo<=pivot<=hi<=size, so I can't understand how lo==size+1 at the end, for this to happen. So the only thing I can think of is concurrency related, which is indeed buggy --- we are editing indices after they are set to instance fields, which violates the copy-on-write semantics. This commit fixes that. Sep 21, 2012 9:38:35 PM jenkins.InitReactorRunner$1 onTaskFailed SEVERE: Failed Loading job MahoutQM java.lang.IndexOutOfBoundsException: Index: 1, Size: 1 at java.util.ArrayList.rangeCheck(ArrayList.java:604) at java.util.ArrayList.get(ArrayList.java:382) at jenkins.model.lazy.SortedList.get(SortedList.java:60) at jenkins.model.lazy.AbstractLazyLoadRunMap.search(AbstractLazyLoadRunMap.java:381) at jenkins.model.lazy.AbstractLazyLoadRunMap.newestBuild(AbstractLazyLoadRunMap.java:243) at hudson.model.AbstractProject.getLastBuild(AbstractProject.java:998) at hudson.maven.AbstractMavenProject.createTransientActions(AbstractMavenProject.java:184) at hudson.model.AbstractProject.updateTransientActions(AbstractProject.java:665) at hudson.maven.MavenModule.updateTransientActions(MavenModule.java:411) at hudson.model.AbstractProject.onLoad(AbstractProject.java:299) at hudson.maven.MavenModule.onLoad(MavenModule.java:236) at hudson.model.Items.load(Items.java:221) at hudson.model.ItemGroupMixIn.loadChildren(ItemGroupMixIn.java:99) --- .../model/lazy/AbstractLazyLoadRunMap.java | 142 ++++++++++++------ .../java/jenkins/model/lazy/SortedList.java | 12 ++ 2 files changed, 105 insertions(+), 49 deletions(-) diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index 57c788daff..e313544488 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -91,7 +91,7 @@ import static jenkins.model.lazy.Boundary.*; *

* Object lock of {@code this} is used to make sure mutation occurs sequentially. * That is, ensure that only one thread is actually calling {@link #retrieve(File)} and - * updating {@link #byNumber} and {@link #byId}. + * updating {@link Index#byNumber} and {@link Index#byId}. * * @author Kohsuke Kawaguchi * @since 1.LAZYLOAD @@ -103,16 +103,39 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i private boolean fullyLoaded; /** - * Stores the mapping from build number to build, for builds that are already loaded. + * Currently visible index. + * Updated atomically. Once set to this field, the index object may not be modified. */ - // copy on write - private volatile TreeMap> byNumber = new TreeMap>(COMPARATOR); + private volatile Index index = new Index(); /** - * Stores the build ID to build number for builds that we already know + * Pair of two maps into a single class, so that the changes can be made visible atomically, + * and updates can happen concurrently to read. + * + * The idiom is that you put yourself in a synchronized block, {@linkplain #copy() make a copy of this}, + * update the copy, then set it to {@link #index}. */ - // copy on write - private volatile TreeMap> byId = new TreeMap>(); + private class Index { + /** + * Stores the mapping from build number to build, for builds that are already loaded. + */ + private final TreeMap> byNumber; + + /** + * Stores the build ID to build number for builds that we already know + */ + private final TreeMap> byId; + + private Index() { + byId = new TreeMap>(); + byNumber = new TreeMap>(COMPARATOR); + } + + private Index(Index rhs) { + byId = new TreeMap>(rhs.byId); + byNumber = new TreeMap>(rhs.byNumber); + } + } /** * Build IDs found as directories, in the ascending order. @@ -178,7 +201,7 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i */ @Override public boolean isEmpty() { - return byId.isEmpty() && search(Integer.MAX_VALUE, DESC)==null; + return index.byId.isEmpty() && search(Integer.MAX_VALUE, DESC)==null; } @Override @@ -190,7 +213,7 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i * Returns a read-only view of records that has already been loaded. */ public SortedMap getLoadedBuilds() { - return Collections.unmodifiableSortedMap(new BuildReferenceMapAdapter(this,byNumber)); + return Collections.unmodifiableSortedMap(new BuildReferenceMapAdapter(this, index.byNumber)); } /** @@ -216,7 +239,7 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i assert i!=null; } - return Collections.unmodifiableSortedMap(new BuildReferenceMapAdapter(this,byNumber.subMap(fromKey, toKey))); + return Collections.unmodifiableSortedMap(new BuildReferenceMapAdapter(this, index.byNumber.subMap(fromKey, toKey))); } public SortedMap headMap(Integer toKey) { @@ -275,7 +298,7 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i * If DESC, finds the closest #M that satisfies M<=N. */ public R search(final int n, final Direction d) { - Entry> c = byNumber.ceilingEntry(n); + Entry> c = index.byNumber.ceilingEntry(n); if (c!=null && c.getKey()== n) { R r = c.getValue().get(); if (r!=null) @@ -287,7 +310,7 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i {// check numberOnDisk as a cache to see if we can find it there int npos = numberOnDisk.find(n); if (npos>=0) {// found exact match - R r = load(numberOnDisk.get(npos), true); + R r = load(numberOnDisk.get(npos), null); if (r!=null) return r; } @@ -331,14 +354,15 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i // so fall back to the slow search } - // capture the snapshot and work off with it since it can be changed by other threads + // capture the snapshot and work off with it since it can be overwritten by other threads SortedList idOnDisk = this.idOnDisk; + boolean clonedIdOnDisk = false; // slow path: we have to find the build from idOnDisk. // first, narrow down the candidate IDs to try by using two known number-to-ID mapping if (idOnDisk.isEmpty()) return null; - Entry> f = byNumber.floorEntry(n); + Entry> f = index.byNumber.floorEntry(n); // if bound is null, use a sentinel value String cid = c==null ? "\u0000" : c.getValue().id; @@ -349,15 +373,21 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i int lo = idOnDisk.higher(cid); int hi = idOnDisk.lower(fid)+1; + // invariant: 0<=lo<=pivot<=hi<=idOnDisk.size() + int pivot; while (true) { pivot = (lo+hi)/2; if (hi<=lo) break; // end of search - R r = load(idOnDisk.get(pivot), true); + R r = load(idOnDisk.get(pivot), null); if (r==null) { // this ID isn't valid. get rid of that and retry pivot hi--; + if (!clonedIdOnDisk) {// if we are making an edit, we need to own a copy + idOnDisk = new SortedList(idOnDisk); + clonedIdOnDisk = true; + } idOnDisk.remove(pivot); continue; } @@ -370,6 +400,9 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i else hi = pivot; // the pivot was too big. look in the lower half } + if (clonedIdOnDisk) + this.idOnDisk = idOnDisk; // feedback the modified result atomically + // didn't find the exact match // 'pivot' points to the insertion point on idOnDisk switch (d) { @@ -396,10 +429,11 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i } public R getById(String id) { - if (byId.containsKey(id)) { - return unwrap(byId.get(id)); + Index snapshot = index; + if (snapshot.byId.containsKey(id)) { + return unwrap(snapshot.byId.get(id)); } - return load(id,true); + return load(id,null); } public R getByNumber(int n) { @@ -415,10 +449,11 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i String id = getIdOf(r); int n = getNumberOf(r); - copy(); + Index copy = copy(); BuildReference ref = createReference(r); - BuildReference old = byId.put(id,ref); - byNumber.put(n,ref); + BuildReference old = copy.byId.put(id,ref); + copy.byNumber.put(n,ref); + index = copy; /* search relies on the fact that every object added via @@ -448,13 +483,14 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i @Override public synchronized void putAll(Map rhs) { - copy(); + Index copy = copy(); for (R r : rhs.values()) { String id = getIdOf(r); BuildReference ref = createReference(r); - byId.put(id,ref); - byNumber.put(getNumberOf(r),ref); + copy.byId.put(id,ref); + copy.byNumber.put(getNumberOf(r),ref); } + index = copy; } /** @@ -470,37 +506,37 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i if (!fullyLoaded) { synchronized (this) { if (!fullyLoaded) { - copy(); + Index copy = copy(); for (String id : idOnDisk) { - if (!byId.containsKey(id)) - load(id,false); // copy() called above, so no need to copy inside + if (!copy.byId.containsKey(id)) + load(id,copy); } + index = copy; fullyLoaded = true; } } } - return byNumber; + return index.byNumber; } /** * Creates a duplicate for the COW data structure in preparation for mutation. */ - private void copy() { - byId = new TreeMap>(byId); - byNumber = new TreeMap>(byNumber); - } + private Index copy() { + return new Index(index); + } /** * Tries to load the record #N by using the shortcut. * * @return null if the data failed to load. */ - protected R load(int n, boolean copy) { + protected R load(int n, Index editInPlace) { R r = null; File shortcut = new File(dir,String.valueOf(n)); if (shortcut.isDirectory()) { synchronized (this) { - r = load(shortcut,copy); + r = load(shortcut,editInPlace); // make sure what we actually loaded is #n, // because the shortcuts can lie. @@ -519,21 +555,29 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i } - protected R load(String id, boolean copy) { - return load(new File(dir,id),copy); + protected R load(String id, Index editInPlace) { + return load(new File(dir,id),editInPlace); } - protected synchronized R load(File dataDir, boolean copy) { + /** + * @param editInPlace + * If non-null, update this data structure. + * Otherwise do a copy-on-write of {@link #index} + */ + protected synchronized R load(File dataDir, Index editInPlace) { try { R r = retrieve(dataDir); if (r==null) return null; - if (copy) copy(); + Index copy = editInPlace!=null ? editInPlace : new Index(index); String id = getIdOf(r); BuildReference ref = createReference(r); - byId.put(id,ref); - byNumber.put(getNumberOf(r),ref); + copy.byId.put(id,ref); + copy.byNumber.put(getNumberOf(r),ref); + + if (editInPlace==null) index = copy; + return r; } catch (IOException e) { LOGGER.log(Level.WARNING, "Failed to load "+dataDir,e); @@ -570,9 +614,11 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i protected abstract R retrieve(File dir) throws IOException; public synchronized boolean removeValue(R run) { - copy(); - byNumber.remove(getNumberOf(run)); - BuildReference old = byId.remove(getIdOf(run)); + Index copy = copy(); + copy.byNumber.remove(getNumberOf(run)); + BuildReference old = copy.byId.remove(getIdOf(run)); + this.index = copy; + return unwrap(old)!=null; } @@ -580,17 +626,15 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i * Replaces all the current loaded Rs with the given ones. */ public synchronized void reset(TreeMap builds) { - TreeMap> byNumber = new TreeMap>(COMPARATOR); - TreeMap> byId = new TreeMap>(COMPARATOR); + Index index = new Index(); for (R r : builds.values()) { String id = getIdOf(r); BuildReference ref = createReference(r); - byId.put(id,ref); - byNumber.put(getNumberOf(r),ref); + index.byId.put(id,ref); + index.byNumber.put(getNumberOf(r),ref); } - this.byNumber = byNumber; - this.byId = byId; + this.index = index; } @Override diff --git a/core/src/main/java/jenkins/model/lazy/SortedList.java b/core/src/main/java/jenkins/model/lazy/SortedList.java index 37139cafe1..22241e0f63 100644 --- a/core/src/main/java/jenkins/model/lazy/SortedList.java +++ b/core/src/main/java/jenkins/model/lazy/SortedList.java @@ -77,6 +77,9 @@ class SortedList> extends AbstractList { /** * Finds the index of the entry lower than v. + * + * @return + * return value will be in the [-1,size) range */ public int lower(T v) { return Boundary.LOWER.apply(find(v)); @@ -84,6 +87,9 @@ class SortedList> extends AbstractList { /** * Finds the index of the entry greater than v. + * + * @return + * return value will be in the [0,size] range */ public int higher(T v) { return Boundary.HIGHER.apply(find(v)); @@ -91,6 +97,9 @@ class SortedList> extends AbstractList { /** * Finds the index of the entry lower or equal to v. + * + * @return + * return value will be in the [-1,size) range */ public int floor(T v) { return Boundary.FLOOR.apply(find(v)); @@ -98,6 +107,9 @@ class SortedList> extends AbstractList { /** * Finds the index of the entry greater or equal to v. + * + * @return + * return value will be in the [0,size] range */ public int ceil(T v) { return Boundary.CEIL.apply(find(v)); From 7917ccb7238e98781f5416559db23984c53dd08d Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Fri, 21 Sep 2012 15:39:26 -0700 Subject: [PATCH 56/69] reducing variable scope --- .../main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index e313544488..4ad671e7d6 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -375,11 +375,11 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i // invariant: 0<=lo<=pivot<=hi<=idOnDisk.size() - int pivot; while (true) { - pivot = (lo+hi)/2; if (hi<=lo) break; // end of search + final int pivot = (lo+hi)/2; + R r = load(idOnDisk.get(pivot), null); if (r==null) { // this ID isn't valid. get rid of that and retry pivot From 550afcc9d7f0e1df0cf656ff3150c28cbabe0b1a Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Fri, 21 Sep 2012 15:39:53 -0700 Subject: [PATCH 57/69] simplification --- .../main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java index 4ad671e7d6..9507d62d79 100644 --- a/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java +++ b/core/src/main/java/jenkins/model/lazy/AbstractLazyLoadRunMap.java @@ -375,9 +375,7 @@ public abstract class AbstractLazyLoadRunMap extends AbstractMap i // invariant: 0<=lo<=pivot<=hi<=idOnDisk.size() - while (true) { - if (hi<=lo) break; // end of search - + while (lo Date: Fri, 21 Sep 2012 15:59:28 -0700 Subject: [PATCH 58/69] Missing @since --- core/src/main/java/jenkins/model/lazy/BuildReference.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/main/java/jenkins/model/lazy/BuildReference.java b/core/src/main/java/jenkins/model/lazy/BuildReference.java index eb437164b4..328e4c665d 100644 --- a/core/src/main/java/jenkins/model/lazy/BuildReference.java +++ b/core/src/main/java/jenkins/model/lazy/BuildReference.java @@ -15,6 +15,7 @@ import java.lang.ref.SoftReference; * and find things in it. * * @author Kohsuke Kawaguchi + * @since 1.LAZYLOAD */ public final class BuildReference extends SoftReference { final String id; From 5c1374ab5143862b93324e9e23b9a8b196d9f9c2 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 23 Sep 2012 19:57:17 -0700 Subject: [PATCH 59/69] [maven-release-plugin] prepare release jenkins-1.483 --- cli/pom.xml | 2 +- core/pom.xml | 2 +- maven-plugin/pom.xml | 2 +- pom.xml | 2 +- test/pom.xml | 2 +- ui-samples-plugin/pom.xml | 2 +- war/pom.xml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cli/pom.xml b/cli/pom.xml index 282ba45645..b77712e0c6 100644 --- a/cli/pom.xml +++ b/cli/pom.xml @@ -5,7 +5,7 @@ pom org.jenkins-ci.main - 1.483-SNAPSHOT + 1.483 cli diff --git a/core/pom.xml b/core/pom.xml index f0a16f175f..45993faf4c 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -29,7 +29,7 @@ THE SOFTWARE. org.jenkins-ci.main pom - 1.483-SNAPSHOT + 1.483 ../pom.xml diff --git a/maven-plugin/pom.xml b/maven-plugin/pom.xml index 9f5ac0136d..ab19b54f6d 100644 --- a/maven-plugin/pom.xml +++ b/maven-plugin/pom.xml @@ -29,7 +29,7 @@ THE SOFTWARE. org.jenkins-ci.main pom - 1.483-SNAPSHOT + 1.483 maven-plugin diff --git a/pom.xml b/pom.xml index 94785ae28d..5d487d47ab 100644 --- a/pom.xml +++ b/pom.xml @@ -33,7 +33,7 @@ THE SOFTWARE. org.jenkins-ci.main pom - 1.483-SNAPSHOT + 1.483 pom Jenkins main module diff --git a/test/pom.xml b/test/pom.xml index e847155063..60173acf37 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -29,7 +29,7 @@ THE SOFTWARE. pom org.jenkins-ci.main - 1.483-SNAPSHOT + 1.483 org.jenkins-ci.main jenkins-test-harness diff --git a/ui-samples-plugin/pom.xml b/ui-samples-plugin/pom.xml index bbc75b62a4..9aeb961116 100644 --- a/ui-samples-plugin/pom.xml +++ b/ui-samples-plugin/pom.xml @@ -29,7 +29,7 @@ THE SOFTWARE. org.jenkins-ci.main pom - 1.483-SNAPSHOT + 1.483 ui-samples-plugin diff --git a/war/pom.xml b/war/pom.xml index 74c4d19c04..9d422e85e7 100644 --- a/war/pom.xml +++ b/war/pom.xml @@ -28,7 +28,7 @@ THE SOFTWARE. org.jenkins-ci.main pom - 1.483-SNAPSHOT + 1.483 ../pom.xml From dcc63c261ce7d9c095c7ee614e39b8a10dc1c626 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 23 Sep 2012 19:57:17 -0700 Subject: [PATCH 60/69] [maven-release-plugin] prepare for next development iteration --- cli/pom.xml | 2 +- core/pom.xml | 2 +- maven-plugin/pom.xml | 2 +- pom.xml | 2 +- test/pom.xml | 2 +- ui-samples-plugin/pom.xml | 2 +- war/pom.xml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cli/pom.xml b/cli/pom.xml index b77712e0c6..4040a9b669 100644 --- a/cli/pom.xml +++ b/cli/pom.xml @@ -5,7 +5,7 @@ pom org.jenkins-ci.main - 1.483 + 1.484-SNAPSHOT cli diff --git a/core/pom.xml b/core/pom.xml index 45993faf4c..c93400ce11 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -29,7 +29,7 @@ THE SOFTWARE. org.jenkins-ci.main pom - 1.483 + 1.484-SNAPSHOT ../pom.xml diff --git a/maven-plugin/pom.xml b/maven-plugin/pom.xml index ab19b54f6d..5ec7af42f7 100644 --- a/maven-plugin/pom.xml +++ b/maven-plugin/pom.xml @@ -29,7 +29,7 @@ THE SOFTWARE. org.jenkins-ci.main pom - 1.483 + 1.484-SNAPSHOT maven-plugin diff --git a/pom.xml b/pom.xml index 5d487d47ab..f2e79e0fb7 100644 --- a/pom.xml +++ b/pom.xml @@ -33,7 +33,7 @@ THE SOFTWARE. org.jenkins-ci.main pom - 1.483 + 1.484-SNAPSHOT pom Jenkins main module diff --git a/test/pom.xml b/test/pom.xml index 60173acf37..f9019d3dfa 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -29,7 +29,7 @@ THE SOFTWARE. pom org.jenkins-ci.main - 1.483 + 1.484-SNAPSHOT org.jenkins-ci.main jenkins-test-harness diff --git a/ui-samples-plugin/pom.xml b/ui-samples-plugin/pom.xml index 9aeb961116..58169bad8e 100644 --- a/ui-samples-plugin/pom.xml +++ b/ui-samples-plugin/pom.xml @@ -29,7 +29,7 @@ THE SOFTWARE. org.jenkins-ci.main pom - 1.483 + 1.484-SNAPSHOT ui-samples-plugin diff --git a/war/pom.xml b/war/pom.xml index 9d422e85e7..c0f35286fc 100644 --- a/war/pom.xml +++ b/war/pom.xml @@ -28,7 +28,7 @@ THE SOFTWARE. org.jenkins-ci.main pom - 1.483 + 1.484-SNAPSHOT ../pom.xml From 788a663ddf83b7fbf7cdb12d07b9a2d10508d2f7 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Sun, 23 Sep 2012 20:12:38 -0700 Subject: [PATCH 61/69] updated changelog for release --- changelog.html | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/changelog.html b/changelog.html index bfbab93822..65b17d22d2 100644 --- a/changelog.html +++ b/changelog.html @@ -61,7 +61,10 @@ Upcoming changes

+

What's new in 1.483 (2012/09/23)

  • JDK installation failed on some slaves with InvalidClassException: hudson.tools.JDKInstaller$Platform$1; local class incompatible: … @@ -90,7 +93,6 @@ Upcoming changes Track and verify plugins used in configuration XML (issue 15003)
-

What's new in 1.482 (2012/09/16)