internal/jsonrpc2_v2: rework Connection concurrency

This change fixes the semantics of Close to actually wait for
in-flight requests before closing the ReadWriteCloser. (Previously,
the Close method closed the ReadWriteCloser immediately, which I
suspect is what led to many of the failures observed in
golang/go#49387 and golang/go#46520.)

It achieves this by explicitly tracking the number of in-flight
requests, including requests with pending async responses, and
explicitly rejecting new Call requests (while keeping the read loop
open!) once Close has begun.

To make it easier for me to reason about the request lifetimes, I
reduced the number of long-lived goroutines from three to just one
(the Read loop), with an additional Handler goroutine that runs only
while the Handler queue is non-empty. Now, it is clearer (I hope!)
that the number of in-flight async requests strictly decreases
after Close has begun, even though the Read goroutine continues
to read requests (and, importantly, responses) and to forward
Notifications to the preempter.

For golang/go#49387
For golang/go#46520

Change-Id: Idf5960f848108a7ced78c5382099c8692e9b181e
Reviewed-on: https://go-review.googlesource.com/c/tools/+/388134
gopls-CI: kokoro <noreply+kokoro@google.com>
Run-TryBot: Bryan Mills <bcmills@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Alan Donovan <adonovan@google.com>
This commit is contained in:
Bryan C. Mills 2022-04-01 10:59:34 -04:00 • committed by Bryan Mills
commit 1afb0eb5ac

View file

@ -230,7 +230,7 @@ func TestIdleListenerAcceptCloseRace(t *testing.T) {
watchdog := time.Duration(n) * 1000 * time.Millisecond
timer := time.AfterFunc(watchdog, func() {
debug.SetTraceback("all")
panic(fmt.Sprintf("TestAcceptCloseRace deadlocked after %v", watchdog))
panic(fmt.Sprintf("%s deadlocked after %v", t.Name(), watchdog))
})
defer timer.Stop()
@ -261,3 +261,84 @@ func TestIdleListenerAcceptCloseRace(t *testing.T) {
<-done
}
}
// TestCloseCallRace checks for a race resulting in a deadlock when a Call on
// one side of the connection races with a Close (or otherwise broken
// connection) initiated from the other side.
//
// (The Call method was waiting for a result from the Read goroutine to
// determine which error value to return, but the Read goroutine was waiting for
// in-flight calls to complete before reporting that result.)
func TestCloseCallRace(t *testing.T) {
ctx := context.Background()
n := 10
watchdog := time.Duration(n) * 1000 * time.Millisecond
timer := time.AfterFunc(watchdog, func() {
debug.SetTraceback("all")
panic(fmt.Sprintf("%s deadlocked after %v", t.Name(), watchdog))
})
defer timer.Stop()
for ; n > 0; n-- {
listener, err := jsonrpc2.NetPipeListener(ctx)
if err != nil {
t.Fatal(err)
}
pokec := make(chan *jsonrpc2.AsyncCall, 1)
s, err := jsonrpc2.Serve(ctx, listener, jsonrpc2.BinderFunc(func(_ context.Context, srvConn *jsonrpc2.Connection) (jsonrpc2.ConnectionOptions, error) {
h := jsonrpc2.HandlerFunc(func(ctx context.Context, _ *jsonrpc2.Request) (interface{}, error) {
// Start a concurrent call from the server to the client.
// The point of this test is to ensure this doesn't deadlock
// if the client shuts down the connection concurrently.
//
// The racing Call may or may not receive a response: it should get a
// response if it is sent before the client closes the connection, and
// it should fail with some kind of "connection closed" error otherwise.
go func() {
pokec <- srvConn.Call(ctx, "poke", nil)
}()
return &msg{"pong"}, nil
})
return jsonrpc2.ConnectionOptions{Handler: h}, nil
}))
if err != nil {
listener.Close()
t.Fatal(err)
}
dialConn, err := jsonrpc2.Dial(ctx, listener.Dialer(), jsonrpc2.ConnectionOptions{})
if err != nil {
listener.Close()
s.Wait()
t.Fatal(err)
}
// Calling any method on the server should provoke it to asynchronously call
// us back. While it is starting that call, we will close the connection.
if err := dialConn.Call(ctx, "ping", nil).Await(ctx, nil); err != nil {
t.Error(err)
}
if err := dialConn.Close(); err != nil {
t.Error(err)
}
// Ensure that the Call on the server side did not block forever when the
// connection closed.
pokeCall := <-pokec
if err := pokeCall.Await(ctx, nil); err == nil {
t.Errorf("unexpected nil error from server-initited call")
} else if errors.Is(err, jsonrpc2.ErrMethodNotFound) {
// The call completed before the Close reached the handler.
} else {
// The error was something else.
t.Logf("server-initiated call completed with expected error: %v", err)
}
listener.Close()
s.Wait()
}
}