From 2d7afedd6c9711a8993a3615ddc0054c462a811c Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Wed, 30 Mar 2011 21:28:52 -0700 Subject: [PATCH 1/4] I'm seeing an occasional SocketTimeoutException in waiting for a response from the server. I suspect there's some concurrency issue in the ThreadPool implementation, as the problem disappears when I attach a debugger (and let it run for 8 hours.) Swapping to JDK ExecutorService to see if that makes a difference --- .../org/jvnet/hudson/test/HudsonTestCase.java | 5 ++ .../org/jvnet/hudson/test/ThreadPoolImpl.java | 53 +++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 test/src/main/java/org/jvnet/hudson/test/ThreadPoolImpl.java diff --git a/test/src/main/java/org/jvnet/hudson/test/HudsonTestCase.java b/test/src/main/java/org/jvnet/hudson/test/HudsonTestCase.java index 91eb662917..3778e90180 100644 --- a/test/src/main/java/org/jvnet/hudson/test/HudsonTestCase.java +++ b/test/src/main/java/org/jvnet/hudson/test/HudsonTestCase.java @@ -128,7 +128,11 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.jar.Manifest; import java.util.logging.Filter; import java.util.logging.Level; @@ -421,6 +425,7 @@ public abstract class HudsonTestCase extends TestCase implements RootAction { context.setMimeTypes(MIME_TYPES); SocketConnector connector = new SocketConnector(); + server.setThreadPool(new ThreadPoolImpl(new ThreadPoolExecutor(1, 10, 10L, TimeUnit.SECONDS, new SynchronousQueue()))); server.addConnector(connector); server.addUserRealm(configureUserRealm()); server.start(); diff --git a/test/src/main/java/org/jvnet/hudson/test/ThreadPoolImpl.java b/test/src/main/java/org/jvnet/hudson/test/ThreadPoolImpl.java new file mode 100644 index 0000000000..4807f66363 --- /dev/null +++ b/test/src/main/java/org/jvnet/hudson/test/ThreadPoolImpl.java @@ -0,0 +1,53 @@ +package org.jvnet.hudson.test; + +import org.mortbay.component.AbstractLifeCycle; +import org.mortbay.thread.ThreadPool; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * @author Kohsuke Kawaguchi + */ +public class ThreadPoolImpl extends AbstractLifeCycle implements ThreadPool { + private final ExecutorService es; + + public ThreadPoolImpl(ExecutorService es) { + this.es = es; + } + + public boolean dispatch(Runnable job) { + if (!isRunning() || job==null) + return false; + + es.submit(job); + return true; + } + + public void join() throws InterruptedException { + while(!es.awaitTermination(999, TimeUnit.DAYS)) + ; + } + + public int getThreads() { + return 999; + } + + public int getIdleThreads() { + return 999; + } + + public boolean isLowOnThreads() { + return false; + } + + @Override + protected void doStart() throws Exception { + // noop + } + + @Override + protected void doStop() throws Exception { + es.shutdown(); + } +} From 11f04a7fda4af6286cce333315b36b9de694963a Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Wed, 30 Mar 2011 21:30:19 -0700 Subject: [PATCH 2/4] set the thread name --- .../main/java/org/jvnet/hudson/test/HudsonTestCase.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/src/main/java/org/jvnet/hudson/test/HudsonTestCase.java b/test/src/main/java/org/jvnet/hudson/test/HudsonTestCase.java index 3778e90180..b8cae4b9aa 100644 --- a/test/src/main/java/org/jvnet/hudson/test/HudsonTestCase.java +++ b/test/src/main/java/org/jvnet/hudson/test/HudsonTestCase.java @@ -131,6 +131,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.jar.Manifest; @@ -425,7 +426,13 @@ public abstract class HudsonTestCase extends TestCase implements RootAction { context.setMimeTypes(MIME_TYPES); SocketConnector connector = new SocketConnector(); - server.setThreadPool(new ThreadPoolImpl(new ThreadPoolExecutor(1, 10, 10L, TimeUnit.SECONDS, new SynchronousQueue()))); + server.setThreadPool(new ThreadPoolImpl(new ThreadPoolExecutor(1, 10, 10L, TimeUnit.SECONDS, new SynchronousQueue(),new ThreadFactory() { + public Thread newThread(Runnable r) { + Thread t = new Thread(r); + t.setName("Jetty Thread Pool"); + return t; + } + }))); server.addConnector(connector); server.addUserRealm(configureUserRealm()); server.start(); From 393be058a5d8dc94f4f4b8bd80565f30d3eb5cea Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Wed, 30 Mar 2011 21:31:45 -0700 Subject: [PATCH 3/4] added a hook for trouble-shooting rare test failures --- .../org/jvnet/hudson/test/HudsonTestCase.java | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/test/src/main/java/org/jvnet/hudson/test/HudsonTestCase.java b/test/src/main/java/org/jvnet/hudson/test/HudsonTestCase.java index b8cae4b9aa..25afcbd82c 100644 --- a/test/src/main/java/org/jvnet/hudson/test/HudsonTestCase.java +++ b/test/src/main/java/org/jvnet/hudson/test/HudsonTestCase.java @@ -32,6 +32,7 @@ import hudson.Extension; import hudson.ExtensionList; import hudson.FilePath; import hudson.Functions; +import hudson.Functions.ThreadGroupMap; import hudson.Launcher; import hudson.Launcher.LocalLauncher; import hudson.Main; @@ -107,6 +108,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.lang.annotation.Annotation; +import java.lang.management.ThreadInfo; import java.lang.ref.WeakReference; import java.lang.reflect.Array; import java.lang.reflect.Constructor; @@ -386,9 +388,28 @@ public abstract class HudsonTestCase extends TestCase implements RootAction { System.out.println("=== Starting "+ getClass().getSimpleName() + "." + getName()); // so that test code has all the access to the system SecurityContextHolder.getContext().setAuthentication(ACL.SYSTEM); - super.runTest(); + + try { + super.runTest(); + } catch (Throwable t) { + // allow the late attachment of a debugger in case of a failure. Useful + // for diagnosing a rare failure + try { + throw new BreakException(); + } catch (BreakException e) {} + + // dump threads + ThreadInfo[] threadInfos = Functions.getThreadInfos(); + ThreadGroupMap m = Functions.sortThreadsAndGetGroupMap(threadInfos); + for (ThreadInfo ti : threadInfos) { + System.err.println(Functions.dumpThreadInfo(ti, m)); + } + throw t; + } } + public static class BreakException extends Exception {} + public String getIconFileName() { return null; } From ff05abe46211613237191330b3afa04325bb5c12 Mon Sep 17 00:00:00 2001 From: Kohsuke Kawaguchi Date: Wed, 30 Mar 2011 21:38:43 -0700 Subject: [PATCH 4/4] version with NPE fix --- test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pom.xml b/test/pom.xml index 336f6dbd2c..604cb0d56d 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -178,7 +178,7 @@ THE SOFTWARE. org.jenkins-ci htmlunit - 2.6-jenkins-3 + 2.6-jenkins-4