UT occurrence of: fatal error: concurrent map read and map write

[peterbroadhurst]

https://github.com/hyperledger/firefly/runs/5997719078?check_suite_focus=true

2022-04-12T21:19:07.828Z]  INFO : Shutting down due to interrupt
fatal error: concurrent map read and map write

goroutine 14 [running]:
runtime.throw(0x10b4d78, 0x21)
	/opt/hostedtoolcache/go/1.16.15/x64/src/runtime/panic.go:1117 +0x72 fp=0xc000013ab0 sp=0xc000013a80 pc=0x441492
runtime.mapaccess2_faststr(0xf68900, 0xc000652ab0, 0xc0000b8cd5, 0x3, 0xc0002a1128, 0x0)
	/opt/hostedtoolcache/go/1.16.15/x64/src/runtime/map_faststr.go:116 +0x4a5 fp=0xc000013b20 sp=0xc000013ab0 pc=0x41d2c5
github.com/spf13/viper.(*Viper).searchMap(0xc0000ca1a0, 0xc000652ab0, 0xc00022e730, 0x2, 0x2, 0x1, 0x1)
	/home/runner/go/pkg/mod/github.com/spf13/[email protected]/viper.go:613 +0x5f fp=0xc000013b68 sp=0xc000013b20 pc=0x96c99f
github.com/spf13/viper.(*Viper).searchMap(0xc0000ca1a0, 0xc000367830, 0xc00022e720, 0x3, 0x3, 0x0, 0x0)
	/home/runner/go/pkg/mod/github.com/spf13/[email protected]/viper.go:627 +0xee fp=0xc000013bb0 sp=0xc000013b68 pc=0x96ca2e
github.com/spf13/viper.(*Viper).find(0xc0000ca1a0, 0xc0000b8cd0, 0x10, 0x1, 0x10, 0x10)
	/home/runner/go/pkg/mod/github.com/spf13/[email protected]/viper.go:1263 +0x40e fp=0xc000013cc0 sp=0xc000013bb0 pc=0x96ea8e
github.com/spf13/viper.(*Viper).Get(0xc0000ca1a0, 0xc0000b8cd0, 0x10, 0xc0000b8cd0, 0xc0004f0ab0)
	/home/runner/go/pkg/mod/github.com/spf13/[email protected]/viper.go:843 +0x85 fp=0xc000013d58 sp=0xc000013cc0 pc=0x96d765
github.com/spf13/viper.(*Viper).GetBool(0xc0000ca1a0, 0xc0000b8cd0, 0x10, 0x10)
	/home/runner/go/pkg/mod/github.com/spf13/[email protected]/viper.go:917 +0x3f fp=0xc000013d90 sp=0xc000013d58 pc=0x96dfdf
github.com/spf13/viper.GetBool(...)
	/home/runner/go/pkg/mod/github.com/spf13/[email protected]/viper.go:914
github.com/hyperledger/firefly/pkg/config.(*configPrefix).GetBool(0xc00004b630, 0x1098f81, 0xb, 0xc00029e300)
	/home/runner/work/firefly/firefly/pkg/config/config.go:358 +0x118 fp=0xc000013e10 sp=0xc000013d90 pc=0x97b8d8
github.com/hyperledger/firefly/pkg/httpserver.NewHTTPServer(0x11f8170, 0xc00007d9c0, 0x108e886, 0x3, 0xc00029e3c0, 0xc0002531a0, 0x1204ff8, 0xc00004b630, 0x0, 0x1, ...)
	/home/runner/work/firefly/firefly/pkg/httpserver/httpserver.go:102 +0x50 fp=0xc000013e90 sp=0xc000013e10 pc=0xd0bd30
github.com/hyperledger/firefly/internal/apiserver.(*apiServer).Serve(0xc00007da00, 0x11f8170, 0xc00007d9c0, 0x120a3a8, 0xc0002a2820, 0x0, 0x1)
	/home/runner/work/firefly/firefly/internal/apiserver/server.go:110 +0x3c8 fp=0xc000013f18 sp=0xc000013e90 pc=0xd11168
github.com/hyperledger/firefly/cmd.startFirefly(0x11f8170, 0xc00007d9c0, 0xc00033fd70, 0x120a3a8, 0xc0002a2820, 0x11e3200, 0xc00007da00, 0xc000252f60)
	/home/runner/work/firefly/firefly/cmd/firefly.go:173 +0x176 fp=0xc000013fa0 sp=0xc000013f18 pc=0xdc1bf6
runtime.goexit()
	/opt/hostedtoolcache/go/1.16.15/x64/src/runtime/asm_amd64.s:1371 +0x1 fp=0xc000013fa8 sp=0xc000013fa0 pc=0x47af21
created by github.com/hyperledger/firefly/cmd.run
	/home/runner/work/firefly/firefly/cmd/firefly.go:122 +0x5c9

goroutine 1 [chan receive]:
testing.(*T).Run(0xc00057e600, 0x10a9668, 0x19, 0x10fc5d0, 0x49cd01)
	/opt/hostedtoolcache/go/1.16.15/x64/src/testing/testing.go:1249 +0x2da
testing.runTests.func1(0xc000001b00)
	/opt/hostedtoolcache/go/1.16.15/x64/src/testing/testing.go:1521 +0x78
testing.tRunner(0xc000001b00, 0xc000497da8)
	/opt/hostedtoolcache/go/1.16.15/x64/src/testing/testing.go:1203 +0xe5
testing.runTests(0xc00000ed08, 0x1739d20, 0x9, 0x9, 0xc08d991a7082eb97, 0x6fcd38908, 0x17a6e40, 0x109b35a)
	/opt/hostedtoolcache/go/1.16.15/x64/src/testing/testing.go:1519 +0x2fe
testing.(*M).Run(0xc000314c00, 0x0)
	/opt/hostedtoolcache/go/1.16.15/x64/src/testing/testing.go:1427 +0x1eb
main.main()
	_testmain.go:107 +0x1c5

goroutine 37 [runnable]:
sync.(*Mutex).lockSlow(0x17d63a8)
	/opt/hostedtoolcache/go/1.16.15/x64/src/sync/mutex.go:84 +0x2cc
sync.(*Mutex).Lock(...)
	/opt/hostedtoolcache/go/1.16.15/x64/src/sync/mutex.go:81
github.com/hyperledger/firefly/pkg/config.(*configPrefix).AddKnownKey(0xc00004b630, 0x1098da8, 0xb, 0xc000644860, 0x1, 0x1)
	/home/runner/work/firefly/firefly/pkg/config/config.go:309 +0x131
github.com/hyperledger/firefly/pkg/httpserver.InitHTTPConfPrefix(0x1204ff8, 0xc00004b630, 0x1388)
	/home/runner/work/firefly/firefly/pkg/httpserver/httpserver.go:88 +0x1bf
github.com/hyperledger/firefly/internal/apiserver.InitConfig()
	/home/runner/work/firefly/firefly/internal/apiserver/server.go:81 +0x46
github.com/hyperledger/firefly/cmd.Execute(0x10a1556, 0x13)
	/home/runner/work/firefly/firefly/cmd/firefly.go:89 +0x32
github.com/hyperledger/firefly/cmd.TestExecOkRestartThenExit(0xc00057e600)
	/home/runner/work/firefly/firefly/cmd/firefly_test.go:119 +0x3b0
testing.tRunner(0xc00057e600, 0x10fc5d0)
	/opt/hostedtoolcache/go/1.16.15/x64/src/testing/testing.go:1203 +0xe5
created by testing.(*T).Run
	/opt/hostedtoolcache/go/1.16.15/x64/src/testing/testing.go:1248 +0x2b3

goroutine 29 [syscall]:
os/signal.signal_recv(0x0)
	/opt/hostedtoolcache/go/1.16.15/x64/src/runtime/sigqueue.go:168 +0xa5
os/signal.loop()
	/opt/hostedtoolcache/go/1.16.15/x64/src/os/signal/signal_unix.go:23 +0x25
created by os/signal.Notify.func1.1
	/opt/hostedtoolcache/go/1.16.15/x64/src/os/signal/signal.go:151 +0x45

goroutine 6 [runnable]:
github.com/hyperledger/firefly/cmd.startFirefly.func1(0x0, 0xc00029e000)
	/home/runner/work/firefly/firefly/cmd/firefly.go:156
created by github.com/hyperledger/firefly/cmd.startFirefly
	/home/runner/work/firefly/firefly/cmd/firefly.go:156 +0x3d2

goroutine 10 [IO wait]:
internal/poll.runtime_pollWait(0x7f86a90de298, 0x72, 0x0)
	/opt/hostedtoolcache/go/1.16.15/x64/src/runtime/netpoll.go:227 +0x55
internal/poll.(*pollDesc).wait(0xc000122b18, 0x72, 0x0, 0x0, 0x[1093](https://github.com/hyperledger/firefly/runs/5997719078?check_suite_focus=true#step:4:1093)31f)
	/opt/hostedtoolcache/go/1.16.15/x64/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.16.15/x64/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc000122b00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/opt/hostedtoolcache/go/1.16.15/x64/src/internal/poll/fd_unix.go:401 +0x212
net.(*netFD).accept(0xc000122b00, 0x30, 0x30, 0x7f86d4ef3108)
	/opt/hostedtoolcache/go/1.16.15/x64/src/net/fd_unix.go:172 +0x45
net.(*TCPListener).accept(0xc000451da0, 0xc00006ddb0, 0x417158, 0x30)
	/opt/hostedtoolcache/go/1.16.15/x64/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc000451da0, 0x100c6a0, 0xc000551ad0, 0xf57060, 0x1727ae0)
	/opt/hostedtoolcache/go/1.16.15/x64/src/net/tcpsock.go:261 +0x65
net/http.(*Server).Serve(0xc00057c0e0, 0x11f46f8, 0xc000451da0, 0x0, 0x0)
	/opt/hostedtoolcache/go/1.16.15/x64/src/net/http/server.go:2962 +0x285
net/http.(*Server).ListenAndServe(0xc00057c0e0, 0xc00057c0e0, 0xc00038dfb0)
	/opt/hostedtoolcache/go/1.16.15/x64/src/net/http/server.go:2891 +0xba
net/http.ListenAndServe(...)
	/opt/hostedtoolcache/go/1.16.15/x64/src/net/http/server.go:3145
github.com/hyperledger/firefly/cmd.startFirefly.func1(0x0, 0xc00029e180)
	/home/runner/work/firefly/firefly/cmd/firefly.go:157 +0xfb
created by github.com/hyperledger/firefly/cmd.startFirefly
	/home/runner/work/firefly/firefly/cmd/firefly.go:156 +0x3d2

goroutine 15 [IO wait]:
internal/poll.runtime_pollWait(0x7f86a90de380, 0x72, 0x0)
	/opt/hostedtoolcache/go/1.16.15/x64/src/runtime/netpoll.go:227 +0x55
internal/poll.(*pollDesc).wait(0xc000122a98, 0x72, 0x0, 0x0, 0x109331f)
	/opt/hostedtoolcache/go/1.16.15/x64/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/opt/hostedtoolcache/go/1.16.15/x64/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc000122a80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/opt/hostedtoolcache/go/1.16.15/x64/src/internal/poll/fd_unix.go:401 +0x212
net.(*netFD).accept(0xc000122a80, 0x30, 0x30, 0x7f86d4ef3108)
	/opt/hostedtoolcache/go/1.16.15/x64/src/net/fd_unix.go:172 +0x45
net.(*TCPListener).accept(0xc000451d70, 0xc000339db0, 0x417158, 0x30)
	/opt/hostedtoolcache/go/1.16.15/x64/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc000451d70, 0x100c6a0, 0xc0005518c0, 0xf57060, 0x1727ae0)
	/opt/hostedtoolcache/go/1.16.15/x64/src/net/tcpsock.go:261 +0x65
net/http.(*Server).Serve(0xc00057c000, 0x11f46f8, 0xc000451d70, 0x0, 0x0)
	/opt/hostedtoolcache/go/1.16.15/x64/src/net/http/server.go:2962 +0x285
net/http.(*Server).ListenAndServe(0xc00057c000, 0xc00057c000, 0xc00038f7b0)
	/opt/hostedtoolcache/go/1.16.15/x64/src/net/http/server.go:2891 +0xba
net/http.ListenAndServe(...)
	/opt/hostedtoolcache/go/1.16.15/x64/src/net/http/server.go:3145
github.com/hyperledger/firefly/cmd.startFirefly.func1(0x0, 0xc00029e300)
	/home/runner/work/firefly/firefly/cmd/firefly.go:157 +0xfb
created by github.com/hyperledger/firefly/cmd.startFirefly
	/home/runner/work/firefly/firefly/cmd/firefly.go:156 +0x3d2
FAIL	github.com/hyperledger/firefly/cmd	0.039s
ok  	github.com/hyperledger/firefly/docs	0.017s	coverage: [no statements]
FAIL
make: *** [Makefile:16: test] Error 1
Error: Process completed with exit code 2.