BackgroundService 无法正常终止

无法弄清楚为什么我的BackgroundService（.NET Core 3.1）无法正常关闭。

编写了一个服务，它应该监听一个 tcp 端口并处理某种格式的消息。写了以下内容BackgroundService：

internal sealed class TcpListenerBackgroundService : BackgroundService
{
    private readonly ITcpPortListener _tcpPortListener;
    private readonly ILogger<TcpListenerBackgroundService> _logger;

    public TcpListenerBackgroundService(ITcpPortListener tcpListener, ILogger<TcpListenerBackgroundService> logger)
    {
        _tcpPortListener = tcpListener;
        _logger = logger;
    }

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        try
        {
            await _tcpPortListener.ListenAsync(stoppingToken);
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, "Unexpected error was occured");
        }
    }
}

ITcpPortListener此处注入以下内容TcpPortListener：

public sealed class TcpPortListener : ITcpPortListener
{
    private readonly ILogger<TcpPortListener> _logger;
    private readonly IPacketProcessor _packetProcessor;
    private readonly TcpListener _listener;
    private TaskCompletionSource<object> _appTerminationSource;

    public TcpPortListener(
        ITcpPortListenerConfiguration config,
        IPacketProcessor packetProcessor,
        ILogger<TcpPortListener> logger)
    {
        _listener = new TcpListener(IPAddress.Any, config.PortNumber);
        _packetProcessor = packetProcessor;
        _logger = logger;
    }

    public async Task ListenAsync(CancellationToken stoppingToken)
    {
        _appTerminationSource = new TaskCompletionSource<object>();
        await using (stoppingToken.Register(_appTerminationSource.SetCanceled))
        {
            var taskList = new List<Task>();
            _listener.Start();

            while (!stoppingToken.IsCancellationRequested)
            {
                var acceptClientTask = _listener.AcceptTcpClientAsync();
                await Task.WhenAny(acceptClientTask, _appTerminationSource.Task);

                if (acceptClientTask.IsCompletedSuccessfully)
                {
                    var client = acceptClientTask.Result;
                    var processTask = ProcessClientAsync(client, stoppingToken);

                    taskList.Add(processTask);
                }

                taskList.RemoveAll(p => p.IsCompleted);
            }

            _logger.LogInformation("Waiting for all clients termination...");
            await Task.WhenAll(taskList.ToArray());

            _logger.LogInformation("Terminate listening...");
            _listener.Stop();
            _logger.LogInformation("Listening was terminated successfully!");
        }
    }

    private async Task ProcessClientAsync(TcpClient client, CancellationToken stoppingToken)
    {
        var clientIp = GetClientIp(client);
        _logger.LogInformation($"Client[{clientIp}]. Connected");

        try
        {
            var stream = client.GetStream();
            var socket = client.Client;

            _logger.LogInformation($"Client[{clientIp}]. Start processing");

            var buffer = new byte[1024];
            var packetContainer = new SuntechPacketContainer();
            var isSocketConnected = CheckSocketConnection(socket);

            while (isSocketConnected && !stoppingToken.IsCancellationRequested)
            {
                var readTask = stream.ReadAsync(buffer, 0, buffer.Length, stoppingToken);
                await Task.WhenAny(readTask, _appTerminationSource.Task);

                if (readTask.IsCompletedSuccessfully)
                {
                    var bytesRead = readTask.Result;
                    if (bytesRead == 0)
                    {
                        isSocketConnected = false;
                        break;
                    }

                    packetContainer.Append(buffer.Take(bytesRead));
                    var packets = packetContainer.FetchFullPackets();
                    await ProcessPacketsAsync(packets);

                    isSocketConnected = CheckSocketConnection(socket);
                }
            }

            if (!isSocketConnected)
            {
                _logger.LogInformation($"Client[{clientIp}]. Socket disconnected. The message processing for the client has been stopped");
            }
        }
        catch (Exception ex)
        {
            _logger.LogError(ex, $"Client[{clientIp}]. Unexpected error was occured during client processing");
        }
        finally
        {
            _logger.LogInformation($"Client[{clientIp}]. Closing the client connection");
            client.Close();
        }
    }

    private static string GetClientIp(TcpClient client)
    {
        return ((IPEndPoint)client.Client.RemoteEndPoint).Address.ToString();
    }

    private async Task ProcessPacketsAsync(IEnumerable<byte[]> packets)
    {
        foreach (var packet in packets)
        {
            await _packetProcessor.ProcessPacketAsync(packet);
        }
    }

    private static bool CheckSocketConnection(Socket socket)
    {
        const int connectionTimeoutInMicroseconds = 1_000;

        var poll = socket.Poll(connectionTimeoutInMicroseconds, SelectMode.SelectRead);
        return !((poll && (socket.Available == 0)) || !socket.Connected);
    }
}

在 CentOS 7 上使用systemd.

问题是有时候（我不知道具体是在什么情况下），如果至少有一个客户端连接到服务，而那一刻我想停止服务（调用service stop），那么服务将无限期结束并systemd挂起，因此我不得不将其杀死，之后，在了解服务的状态后，我会被告知该进程由于等待超时到期而被杀死。我完全不明白为什么会发生这种情况，因为该方法ListenAsync是根据日志得出的。这绝对不是因为该方法需要ListenAsync很长时间才能完成。我附上一个示例日志：

//...
2020-11-03 04:01:22.299 -05:00 [INF] Application is shutting down...
2020-11-03 04:01:22.333 -05:00 [INF] Client[127.0.0.1]. Closing the client connection
2020-11-03 04:01:22.334 -05:00 [INF] Waiting for all clients termination...
2020-11-03 04:01:22.335 -05:00 [INF] Terminate listening...
2020-11-03 04:01:22.336 -05:00 [INF] Listening was terminated successfully!

有一种感觉，在某些情况下并非所有资源都被释放。请检查我的代码是否存在资源管理可能不准确的地方。我的眼睛已经起泡了，什么也看不见。