internal/jsonrpc2_v2: rework Connection concurrency
This change fixes the semantics of Close to actually wait for in-flight requests before closing the ReadWriteCloser. (Previously, the Close method closed the ReadWriteCloser immediately, which I suspect is what led to many of the failures observed in golang/go#49387 and golang/go#46520.) It achieves this by explicitly tracking the number of in-flight requests, including requests with pending async responses, and explicitly rejecting new Call requests (while keeping the read loop open!) once Close has begun. To make it easier for me to reason about the request lifetimes, I reduced the number of long-lived goroutines from three to just one (the Read loop), with an additional Handler goroutine that runs only while the Handler queue is non-empty. Now, it is clearer (I hope!) that the number of in-flight async requests strictly decreases after Close has begun, even though the Read goroutine continues to read requests (and, importantly, responses) and to forward Notifications to the preempter. For golang/go#49387 For golang/go#46520 Change-Id: Idf5960f848108a7ced78c5382099c8692e9b181e Reviewed-on: https://go-review.googlesource.com/c/tools/+/388134 gopls-CI: kokoro <noreply+kokoro@google.com> Run-TryBot: Bryan Mills <bcmills@google.com> TryBot-Result: Gopher Robot <gobot@golang.org> Reviewed-by: Alan Donovan <adonovan@google.com>
This commit is contained in:
parent
5d6e2930bf
commit
1afb0eb5ac
1 changed files with 621 additions and 349 deletions
|
|
@ -230,7 +230,7 @@ func TestIdleListenerAcceptCloseRace(t *testing.T) {
|
|||
watchdog := time.Duration(n) * 1000 * time.Millisecond
|
||||
timer := time.AfterFunc(watchdog, func() {
|
||||
debug.SetTraceback("all")
|
||||
panic(fmt.Sprintf("TestAcceptCloseRace deadlocked after %v", watchdog))
|
||||
panic(fmt.Sprintf("%s deadlocked after %v", t.Name(), watchdog))
|
||||
})
|
||||
defer timer.Stop()
|
||||
|
||||
|
|
@ -261,3 +261,84 @@ func TestIdleListenerAcceptCloseRace(t *testing.T) {
|
|||
<-done
|
||||
}
|
||||
}
|
||||
|
||||
// TestCloseCallRace checks for a race resulting in a deadlock when a Call on
|
||||
// one side of the connection races with a Close (or otherwise broken
|
||||
// connection) initiated from the other side.
|
||||
//
|
||||
// (The Call method was waiting for a result from the Read goroutine to
|
||||
// determine which error value to return, but the Read goroutine was waiting for
|
||||
// in-flight calls to complete before reporting that result.)
|
||||
func TestCloseCallRace(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
n := 10
|
||||
|
||||
watchdog := time.Duration(n) * 1000 * time.Millisecond
|
||||
timer := time.AfterFunc(watchdog, func() {
|
||||
debug.SetTraceback("all")
|
||||
panic(fmt.Sprintf("%s deadlocked after %v", t.Name(), watchdog))
|
||||
})
|
||||
defer timer.Stop()
|
||||
|
||||
for ; n > 0; n-- {
|
||||
listener, err := jsonrpc2.NetPipeListener(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
pokec := make(chan *jsonrpc2.AsyncCall, 1)
|
||||
|
||||
s, err := jsonrpc2.Serve(ctx, listener, jsonrpc2.BinderFunc(func(_ context.Context, srvConn *jsonrpc2.Connection) (jsonrpc2.ConnectionOptions, error) {
|
||||
h := jsonrpc2.HandlerFunc(func(ctx context.Context, _ *jsonrpc2.Request) (interface{}, error) {
|
||||
// Start a concurrent call from the server to the client.
|
||||
// The point of this test is to ensure this doesn't deadlock
|
||||
// if the client shuts down the connection concurrently.
|
||||
//
|
||||
// The racing Call may or may not receive a response: it should get a
|
||||
// response if it is sent before the client closes the connection, and
|
||||
// it should fail with some kind of "connection closed" error otherwise.
|
||||
go func() {
|
||||
pokec <- srvConn.Call(ctx, "poke", nil)
|
||||
}()
|
||||
|
||||
return &msg{"pong"}, nil
|
||||
})
|
||||
return jsonrpc2.ConnectionOptions{Handler: h}, nil
|
||||
}))
|
||||
if err != nil {
|
||||
listener.Close()
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dialConn, err := jsonrpc2.Dial(ctx, listener.Dialer(), jsonrpc2.ConnectionOptions{})
|
||||
if err != nil {
|
||||
listener.Close()
|
||||
s.Wait()
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Calling any method on the server should provoke it to asynchronously call
|
||||
// us back. While it is starting that call, we will close the connection.
|
||||
if err := dialConn.Call(ctx, "ping", nil).Await(ctx, nil); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if err := dialConn.Close(); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
// Ensure that the Call on the server side did not block forever when the
|
||||
// connection closed.
|
||||
pokeCall := <-pokec
|
||||
if err := pokeCall.Await(ctx, nil); err == nil {
|
||||
t.Errorf("unexpected nil error from server-initited call")
|
||||
} else if errors.Is(err, jsonrpc2.ErrMethodNotFound) {
|
||||
// The call completed before the Close reached the handler.
|
||||
} else {
|
||||
// The error was something else.
|
||||
t.Logf("server-initiated call completed with expected error: %v", err)
|
||||
}
|
||||
|
||||
listener.Close()
|
||||
s.Wait()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue