Files
ECC/tests/test_react_responsiveness.py
T

225 lines
8.2 KiB
Python

from __future__ import annotations
import asyncio
import contextvars
import threading
from dataclasses import dataclass, field
from typing import Any
import pytest
from llm.core.interface import LLMError
from llm.core.types import LLMInput, LLMOutput, Message, Role
from llm.tools import ReActAgent, ToolExecutor
pytestmark = pytest.mark.unit
def _input() -> LLMInput:
return LLMInput(messages=[Message(role=Role.USER, content="hello")])
def test_agent_keeps_event_loop_live_during_provider_request() -> None:
request_id = contextvars.ContextVar("request_id", default="missing")
release = threading.Event()
observed: list[str] = []
async def scenario() -> LLMOutput:
entered = asyncio.Event()
loop = asyncio.get_running_loop()
class WaitingProvider:
def generate(self, agent_input: LLMInput) -> LLMOutput:
observed.append(request_id.get())
loop.call_soon_threadsafe(entered.set)
if not release.wait(timeout=2):
return LLMOutput(content="event loop did not release the request")
return LLMOutput(content="done")
request_id.set("request-17")
agent = ReActAgent(WaitingProvider(), ToolExecutor())
pending = asyncio.create_task(agent.run(_input()))
try:
await asyncio.wait_for(entered.wait(), timeout=5)
release.set()
return await asyncio.wait_for(pending, timeout=5)
finally:
release.set()
output = asyncio.run(scenario())
assert output.content == "done"
assert observed == ["request-17"]
def test_agent_can_be_cancelled_while_provider_is_waiting() -> None:
release = threading.Event()
async def scenario() -> None:
entered = asyncio.Event()
loop = asyncio.get_running_loop()
class WaitingProvider:
def generate(self, agent_input: LLMInput) -> LLMOutput:
loop.call_soon_threadsafe(entered.set)
release.wait(timeout=2)
return LLMOutput(content="provider finished")
pending = asyncio.create_task(
ReActAgent(WaitingProvider(), ToolExecutor()).run(_input())
)
try:
await asyncio.wait_for(entered.wait(), timeout=5)
pending.cancel()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(pending, timeout=5)
finally:
# Cancelling the await does not abort synchronous provider I/O.
release.set()
asyncio.run(scenario())
def test_provider_error_keeps_the_agent_error_contract() -> None:
class FailingProvider:
def generate(self, agent_input: LLMInput) -> LLMOutput:
raise LLMError("upstream unavailable", code="connection_error")
output = asyncio.run(ReActAgent(FailingProvider(), ToolExecutor()).run(_input()))
assert output.stop_reason == "provider_error"
assert output.content == "Provider error: connection_error"
async def _run_started(agent: ReActAgent, started: asyncio.Event) -> LLMOutput:
started.set()
return await agent.run(_input())
@dataclass
class SharedProvider:
"""An unhashable custom provider with mutable, non-reentrant request state."""
loop: asyncio.AbstractEventLoop
fail_first: bool = False
release: threading.Event = field(default_factory=threading.Event)
entered: list[asyncio.Event] = field(
default_factory=lambda: [asyncio.Event(), asyncio.Event()]
)
guard: Any = field(default_factory=threading.Lock)
calls: int = 0
active: int = 0
max_active: int = 0
def generate(self, agent_input: LLMInput) -> LLMOutput:
with self.guard:
index = self.calls
self.calls += 1
self.active += 1
self.max_active = max(self.max_active, self.active)
self.loop.call_soon_threadsafe(self.entered[index].set)
try:
if index == 0:
if not self.release.wait(timeout=5):
raise RuntimeError("Provider release timed out")
if self.fail_first:
raise LLMError("upstream unavailable", code="connection_error")
return LLMOutput(content=f"response-{index + 1}")
finally:
with self.guard:
self.active -= 1
@pytest.mark.parametrize("first_result", ["success", "cancelled", "error"])
def test_shared_provider_remains_serial_until_the_worker_finishes(
first_result: str,
) -> None:
async def scenario() -> SharedProvider:
loop = asyncio.get_running_loop()
provider = SharedProvider(loop, fail_first=first_result == "error")
first = asyncio.create_task(ReActAgent(provider, ToolExecutor()).run(_input()))
pending = [first]
try:
await asyncio.wait_for(provider.entered[0].wait(), timeout=5)
if first_result == "cancelled":
first.cancel()
with pytest.raises(asyncio.CancelledError):
await first
second_waiting = asyncio.Event()
second = asyncio.create_task(
_run_started(ReActAgent(provider, ToolExecutor()), second_waiting)
)
pending.append(second)
await asyncio.wait_for(second_waiting.wait(), timeout=5)
# The second run has reached its provider await. It must stay queued
# while the first generate is blocked, even after the first is cancelled.
with pytest.raises(TimeoutError):
await asyncio.wait_for(provider.entered[1].wait(), timeout=0.1)
provider.release.set()
assert (await asyncio.wait_for(second, timeout=5)).content == "response-2"
if first_result != "cancelled":
output = await first
if first_result == "error":
assert output.stop_reason == "provider_error"
else:
assert output.content == "response-1"
return provider
finally:
provider.release.set()
await asyncio.gather(*pending, return_exceptions=True)
provider = asyncio.run(scenario())
assert provider.calls == 2
assert provider.max_active == 1
def test_cancelling_a_queued_run_does_not_start_another_provider_request() -> None:
async def scenario() -> SharedProvider:
loop = asyncio.get_running_loop()
provider = SharedProvider(loop)
first = asyncio.create_task(ReActAgent(provider, ToolExecutor()).run(_input()))
pending = [first]
try:
await asyncio.wait_for(provider.entered[0].wait(), timeout=5)
second_waiting = asyncio.Event()
second = asyncio.create_task(
_run_started(ReActAgent(provider, ToolExecutor()), second_waiting)
)
pending.append(second)
await asyncio.wait_for(second_waiting.wait(), timeout=5)
second.cancel()
with pytest.raises(asyncio.CancelledError):
await second
provider.release.set()
assert (await asyncio.wait_for(first, timeout=5)).content == "response-1"
return provider
finally:
provider.release.set()
await asyncio.gather(*pending, return_exceptions=True)
# asyncio.run also joins the executor, so a cancelled but still-running
# worker cannot call generate after the assertion has already passed.
provider = asyncio.run(scenario())
assert provider.calls == 1
def test_distinct_providers_can_make_progress_independently() -> None:
async def scenario() -> None:
loop = asyncio.get_running_loop()
first_provider = SharedProvider(loop)
second_provider = SharedProvider(loop)
second_provider.release.set()
first = asyncio.create_task(
ReActAgent(first_provider, ToolExecutor()).run(_input())
)
try:
await asyncio.wait_for(first_provider.entered[0].wait(), timeout=5)
second = ReActAgent(second_provider, ToolExecutor()).run(_input())
assert (await asyncio.wait_for(second, timeout=2)).content == "response-1"
assert not first.done()
finally:
first_provider.release.set()
await first
asyncio.run(scenario())