Fix some race conditions in listener_test (#13385) #4324

Job	Run time
build-linux-cmake-mingw	11m 50s
check-format-and-targets	27s
build-linux	16m 47s
build-linux-cmake-with-benchmark	19m 5s
build-linux-cmake-with-folly	27m 26s
build-linux-cmake-with-folly-coroutines	31m 52s
build-linux-cmake-with-folly-lite-no-test	7m 33s
build-linux-encrypted_env-no_compression	15m 36s
build-linux-make-with-folly	28m 7s
build-linux-make-with-folly-lite-no-test	9m 33s
build-linux-release-rtti	12m 5s
build-examples	10m 36s
build-linux-release	14m 0s
build-fuzzers	9m 9s
build-linux-clang10-asan	16m 53s
build-linux-clang10-clang-analyze	18m 1s
build-linux-clang10-ubsan	17m 1s
build-linux-clang13-mini-tsan	22m 8s
build-linux-clang-13-no_test_run	7m 35s
build-linux-clang-no_test_run	12m 50s
build-linux-gcc-10-cxx20-no_test_run	9m 37s
build-linux-gcc-11-no_test_run	10m 15s
build-linux-gcc-8-no_test_run	8m 36s
build-linux-mini-crashtest	28m 22s
build-linux-unity-and-headers	12m 40s
build-linux-arm	13m 22s
build-linux-java	14m 28s
build-linux-java-pmd	11m 32s
build-linux-java-static	10m 58s
build-linux-static_lib-alt_namespace-status_checked	18m 46s
build-macos	21m 15s
build-macos-cmake (true)	45m 33s
build-macos-java	17m 20s
build-macos-cmake (false)	55m 32s
build-macos-java-static	17m 32s
build-macos-java-static-universal	18m 54s
build-windows-vs2019	53m 28s
	11h 16m 44s

Provide feedback