fix: data race issues with api.Server (#11724)

## Description

Closes: #11692

This PR adds a `sync.Mutex` to `api.Server` in order to prevent data race issues. In practice, these data races are unlikely to occur as you'd typically wait for Start to return. However, this does cause an issue with testing using the `-race` flag when using the `testutil/network` package. Currently races occur by default when running tests on projects generated using `ignite`. We should fix this for the sake of passing race detecting tests.

---

### Author Checklist

*All items are required. Please add a note to the item if the item is not applicable and
please add links to any relevant follow up issues.*

I have...

- [X] included the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title
- [X] added `!` to the type prefix if API or client breaking change
- [X] targeted the correct branch (see [PR Targeting](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#pr-targeting))
- [X] provided a link to the relevant issue or specification
- [X] followed the guidelines for [building modules](https://github.com/cosmos/cosmos-sdk/blob/master/docs/building-modules)
- [X] included the necessary unit and integration [tests](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#testing)
- [x] added a changelog entry to `CHANGELOG.md`
- [X] included comments for [documenting Go code](https://blog.golang.org/godoc)
- [X] updated the relevant documentation or specification
- [x] reviewed "Files changed" and left comments if necessary
- [ ] confirmed all CI checks have passed

### Reviewers Checklist

*All items are required. Please add a note if the item is not applicable and please add
your handle next to the items reviewed if you only reviewed selected items.*

I have...

- [ ] confirmed the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title
- [ ] confirmed `!` in the type prefix if API or client breaking change
- [ ] confirmed all author checklist items have been addressed 
- [ ] reviewed state machine logic
- [ ] reviewed API design and naming
- [ ] reviewed documentation is accurate
- [ ] reviewed tests and test coverage
- [ ] manually tested (if applicable)

CHANGELOG.md

@@ -211,6 +211,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 ### Bug Fixes
 
+* [\#11724](https://github.com/cosmos/cosmos-sdk/pull/11724) Fix data race issues with api.Server
 * [\#11693](https://github.com/cosmos/cosmos-sdk/pull/11693) Add validation for gentx cmd.
 * [\#11645](https://github.com/cosmos/cosmos-sdk/pull/11645) Fix `--home` flag ignored when running help.
 * [\#11558](https://github.com/cosmos/cosmos-sdk/pull/11558) Fix `--dry-run` not working when using tx command.

server/api/server.go

@@ -5,6 +5,7 @@ import (
 	"net"
 	"net/http"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/gogo/gateway"
@@ -30,8 +31,13 @@ type Server struct {
 	GRPCGatewayRouter *runtime.ServeMux
 	ClientCtx         client.Context
 
-	logger   log.Logger
-	metrics  *telemetry.Metrics
+	logger  log.Logger
+	metrics *telemetry.Metrics
+	// Start() is blocking and generally called from a separate goroutine.
+	// Close() can be called asynchronously and access shared memory
+	// via the listener. Therefore, we sync access to Start and Close with
+	// this mutex to avoid data races.
+	mtx      sync.Mutex
 	listener net.Listener
 }
 
@@ -83,9 +89,11 @@ func New(clientCtx client.Context, logger log.Logger) *Server {
 // and are delegated to the Tendermint JSON RPC server. The process is
 // non-blocking, so an external signal handler must be used.
 func (s *Server) Start(cfg config.Config) error {
+	s.mtx.Lock()
 	if cfg.Telemetry.Enabled {
 		m, err := telemetry.New(cfg.Telemetry)
 		if err != nil {
+			s.mtx.Unlock()
 			return err
 		}
 
@@ -101,6 +109,7 @@ func (s *Server) Start(cfg config.Config) error {
 
 	listener, err := tmrpcserver.Listen(cfg.API.Address, tmCfg.MaxOpenConnections)
 	if err != nil {
+		s.mtx.Unlock()
 		return err
 	}
 
@@ -111,15 +120,19 @@ func (s *Server) Start(cfg config.Config) error {
 
 	if cfg.API.EnableUnsafeCORS {
 		allowAllCORS := handlers.CORS(handlers.AllowedHeaders([]string{"Content-Type"}))
+		s.mtx.Unlock()
 		return tmrpcserver.Serve(s.listener, allowAllCORS(h), s.logger, tmCfg)
 	}
 
 	s.logger.Info("starting API server...")
+	s.mtx.Unlock()
 	return tmrpcserver.Serve(s.listener, s.Router, s.logger, tmCfg)
 }
 
 // Close closes the API server.
 func (s *Server) Close() error {
+	s.mtx.Lock()
+	defer s.mtx.Unlock()
 	return s.listener.Close()
 }