Skip to content
This repository has been archived by the owner on May 3, 2023. It is now read-only.

fatal error: concurrent map iteration and map write #67

Open
invidian opened this issue Dec 10, 2021 · 0 comments
Open

fatal error: concurrent map iteration and map write #67

invidian opened this issue Dec 10, 2021 · 0 comments

Comments

@invidian
Copy link
Member

On image: docker.io/kinvolk/gadget:v0.3.1

{"level":"info","msg":"BPF Tracer ready","time":"2021-12-10T14:04:22Z"}
{"level":"info","msg":"Starting proc informer","time":"2021-12-10T14:04:22Z"}
{"level":"info","msg":"Starting annotation publisher","time":"2021-12-10T14:04:22Z"}
{"level":"info","msg":"Starting updater loop","time":"2021-12-10T14:04:22Z"}
fatal error: concurrent map iteration and map write

goroutine 280 [running]:
runtime.throw(0x1608852, 0x26)
        /opt/hostedtoolcache/go/1.13.15/x64/src/runtime/panic.go:774 +0x72 fp=0xc012a6cf60 sp=0xc012a6cf30 pc=0x431282
runtime.mapiternext(0xc012a6d078)
        /opt/hostedtoolcache/go/1.13.15/x64/src/runtime/map.go:858 +0x579 fp=0xc012a6cfe8 sp=0xc012a6cf60 pc=0x4126f9
github.com/kinvolk/traceloop/pkg/podinformer.(*PodInformer).GetPodFromContainerID(0xc0000cf540, 0xc0129f8d80, 0x40, 0x1, 0x1, 0x0)
        /home/runner/work/traceloop/traceloop/pkg/podinformer/podinformer.go:148 +0xae fp=0xc012a6d0e8 sp=0xc012a6cfe8 pc=0x12e15ee
github.com/kinvolk/traceloop/pkg/straceback.(*StraceBack).updater(0xc0001165a0, 0x0, 0x0)
        /home/runner/work/traceloop/traceloop/pkg/straceback/straceback.go:402 +0x158d fp=0xc012a6dfc8 sp=0xc012a6d0e8 pc=0x130ee8d
runtime.goexit()
        /opt/hostedtoolcache/go/1.13.15/x64/src/runtime/asm_amd64.s:1357 +0x1 fp=0xc012a6dfd0 sp=0xc012a6dfc8 pc=0x45e631
created by github.com/kinvolk/traceloop/pkg/straceback.NewTracer
        /home/runner/work/traceloop/traceloop/pkg/straceback/straceback.go:246 +0x114b

goroutine 1 [IO wait]:
internal/poll.runtime_pollWait(0x7f184c498e38, 0x72, 0x0)
        /opt/hostedtoolcache/go/1.13.15/x64/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc01298da98, 0x72, 0x0, 0x0, 0x15e6a28)
        /opt/hostedtoolcache/go/1.13.15/x64/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
        /opt/hostedtoolcache/go/1.13.15/x64/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc01298da80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /opt/hostedtoolcache/go/1.13.15/x64/src/internal/poll/fd_unix.go:384 +0x1f8
net.(*netFD).accept(0xc01298da80, 0xc00064e380, 0x7f1850709d98, 0x0)
        /opt/hostedtoolcache/go/1.13.15/x64/src/net/fd_unix.go:238 +0x42
net.(*UnixListener).accept(0xc01297d890, 0xc000029b18, 0x40fc68, 0x30)
        /opt/hostedtoolcache/go/1.13.15/x64/src/net/unixsock_posix.go:162 +0x32
net.(*UnixListener).Accept(0xc01297d890, 0x1505c40, 0xc01297d980, 0x142d9c0, 0x2212180)
        /opt/hostedtoolcache/go/1.13.15/x64/src/net/unixsock.go:260 +0x47
net/http.(*Server).Serve(0xc00036c0e0, 0x1821f60, 0xc01297d890, 0x0, 0x0)
        /opt/hostedtoolcache/go/1.13.15/x64/src/net/http/server.go:2925 +0x280
main.main()
        /home/runner/work/traceloop/traceloop/traceloop.go:273 +0x1389

goroutine 6 [syscall]:
os/signal.signal_recv(0x0)
        /opt/hostedtoolcache/go/1.13.15/x64/src/runtime/sigqueue.go:147 +0x9c
os/signal.loop()
        /opt/hostedtoolcache/go/1.13.15/x64/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.init.0
        /opt/hostedtoolcache/go/1.13.15/x64/src/os/signal/signal_unix.go:29 +0x41

goroutine 7 [chan receive]:
k8s.io/klog/v2.(*loggingT).flushDaemon(0x22926e0)
        /home/runner/go/pkg/mod/k8s.io/klog/[email protected]/klog.go:1169 +0x8b
created by k8s.io/klog/v2.init.0
        /home/runner/go/pkg/mod/k8s.io/klog/[email protected]/klog.go:417 +0xdd

goroutine 8 [chan receive]:
k8s.io/klog.(*loggingT).flushDaemon(0x2292600)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/klog.go:1010 +0x8b
created by k8s.io/klog.init.0
        /home/runner/go/pkg/mod/k8s.io/[email protected]/klog.go:411 +0xd6

goroutine 11 [chan receive]:
k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc000248060)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:198 +0xe0
created by k8s.io/client-go/util/workqueue.newQueue
        /home/runner/go/pkg/mod/k8s.io/[email protected]/util/workqueue/queue.go:58 +0x132

goroutine 13 [select]:
k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0xc000248180)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:231 +0x405
created by k8s.io/client-go/util/workqueue.newDelayingQueue
        /home/runner/go/pkg/mod/k8s.io/[email protected]/util/workqueue/delaying_queue.go:68 +0x184

goroutine 14 [chan receive]:
github.com/kinvolk/traceloop/pkg/podinformer.(*PodInformer).Run(0xc0000cf540, 0x1, 0xc00004a480)
        /home/runner/work/traceloop/traceloop/pkg/podinformer/podinformer.go:296 +0x2b6
created by github.com/kinvolk/traceloop/pkg/podinformer.NewPodInformer
        /home/runner/work/traceloop/traceloop/pkg/podinformer/podinformer.go:128 +0x6a7

goroutine 23 [sync.Cond.Wait]:
runtime.goparkunlock(...)
        /opt/hostedtoolcache/go/1.13.15/x64/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0002900c8, 0x3)
        /opt/hostedtoolcache/go/1.13.15/x64/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0002900b8)
        /opt/hostedtoolcache/go/1.13.15/x64/src/sync/cond.go:56 +0x9d
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0002900a0, 0xc000120f30, 0x0, 0x0, 0x0, 0x0)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/delta_fifo.go:493 +0xaa
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000116120)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:183 +0x40
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc00025bfb0)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x5e
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00025bfb0, 0x17ef3c0, 0xc00024c000, 0xc0001ac001, 0xc00004a480)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xa3
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00025bfb0, 0x3b9aca00, 0x0, 0xc000392c01, 0xc00004a480)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0xaa
k8s.io/apimachinery/pkg/util/wait.Until(...)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*controller).Run(0xc000116120, 0xc00004a480)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:154 +0x2f6
created by github.com/kinvolk/traceloop/pkg/podinformer.(*PodInformer).Run
        /home/runner/work/traceloop/traceloop/pkg/podinformer/podinformer.go:284 +0x157

goroutine 323 [syscall]:
github.com/iovisor/gobpf/elf._C2func_poll(0xc00c1cfe00, 0x8, 0x1f4, 0x0, 0x0, 0x0)
        _cgo_gotypes.go:352 +0x55
github.com/iovisor/gobpf/elf.perfEventPoll(0xc000044f60, 0x8, 0x8, 0xc0003380f0, 0xc00007fe40)
        /home/runner/go/pkg/mod/github.com/kinvolk/[email protected]/elf/perf.go:425 +0xe6
github.com/iovisor/gobpf/elf.(*PerfMap).PollStart.func1(0xc0003380f0, 0xc00003da80, 0xc0003948c0)
        /home/runner/go/pkg/mod/github.com/kinvolk/[email protected]/elf/perf.go:329 +0x792
created by github.com/iovisor/gobpf/elf.(*PerfMap).PollStart
        /home/runner/go/pkg/mod/github.com/kinvolk/[email protected]/elf/perf.go:312 +0xbf

goroutine 50 [chan receive]:
k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc00004a480, 0xc000116120)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:130 +0x34
created by k8s.io/client-go/tools/cache.(*controller).Run
        /home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/controller.go:129 +0x9f

goroutine 51 [select]:
k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc0001d0000, 0xc064f577dead9950, 0x32468e0, 0x22921a0, 0x17fab20, 0xc00095fec0, 0xc000c3bb98, 0xc00026fe00, 0xc00004a480, 0x0, ...)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:463 +0x1ab
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0001d0000, 0xc00004a480, 0x0, 0x0)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:427 +0x669
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
        /home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:221 +0x38
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc00033f6d0)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x5e
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc000c3bed0, 0x17ef3a0, 0xc0006a00a0, 0x1, 0xc00004a480)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xa3
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0001d0000, 0xc00004a480)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:220 +0x1b5
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
        /home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:56 +0x2e
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0001ac010, 0xc0002ae020)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:73 +0x59
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
        /home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:71 +0x62

goroutine 16 [select]:
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc0001d0000, 0xc00004a480, 0xc000976300, 0xc00026fe00)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:373 +0x17e
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
        /home/runner/go/pkg/mod/k8s.io/[email protected]/tools/cache/reflector.go:367 +0x2bf

goroutine 279 [chan send]:
github.com/kinvolk/traceloop/pkg/procinformer.(*ProcInformer).update(0xc0002af3c0, 0xc000343dd0, 0x2)
        /home/runner/work/traceloop/traceloop/pkg/procinformer/procinformer.go:131 +0x6dc
github.com/kinvolk/traceloop/pkg/procinformer.NewProcInformer.func1(0xc0002af3c0)
        /home/runner/work/traceloop/traceloop/pkg/procinformer/procinformer.go:50 +0x128
created by github.com/kinvolk/traceloop/pkg/procinformer.NewProcInformer
        /home/runner/work/traceloop/traceloop/pkg/procinformer/procinformer.go:41 +0xd7

goroutine 38 [IO wait]:
internal/poll.runtime_pollWait(0x7f184c498f08, 0x72, 0xffffffffffffffff)
        /opt/hostedtoolcache/go/1.13.15/x64/src/runtime/netpoll.go:184 +0x55
internal/poll.(*pollDesc).wait(0xc000254398, 0x72, 0xb600, 0xb63a, 0xffffffffffffffff)
        /opt/hostedtoolcache/go/1.13.15/x64/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
        /opt/hostedtoolcache/go/1.13.15/x64/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000254380, 0xc00017e000, 0xb63a, 0xb63a, 0x0, 0x0, 0x0)
        /opt/hostedtoolcache/go/1.13.15/x64/src/internal/poll/fd_unix.go:169 +0x1cf
net.(*netFD).Read(0xc000254380, 0xc00017e000, 0xb63a, 0xb63a, 0x203000, 0x7f184c441ba8, 0x0)
        /opt/hostedtoolcache/go/1.13.15/x64/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000010a90, 0xc00017e000, 0xb63a, 0xb63a, 0x0, 0x0, 0x0)
        /opt/hostedtoolcache/go/1.13.15/x64/src/net/net.go:184 +0x68
crypto/tls.(*atLeastReader).Read(0xc00091c000, 0xc00017e000, 0xb63a, 0xb63a, 0x1f, 0x17ee400, 0xc00010f920)
        /opt/hostedtoolcache/go/1.13.15/x64/src/crypto/tls/conn.go:780 +0x60
bytes.(*Buffer).ReadFrom(0xc000071758, 0x17ee260, 0xc00091c000, 0x40d3d5, 0x1459560, 0x1587420)
        /opt/hostedtoolcache/go/1.13.15/x64/src/bytes/buffer.go:204 +0xb4
crypto/tls.(*Conn).readFromUntil(0xc000071500, 0x17ef840, 0xc000010a90, 0x5, 0xc000010a90, 0x9)
        /opt/hostedtoolcache/go/1.13.15/x64/src/crypto/tls/conn.go:802 +0xec
crypto/tls.(*Conn).readRecordOrCCS(0xc000071500, 0x0, 0x0, 0x442a9f)
        /opt/hostedtoolcache/go/1.13.15/x64/src/crypto/tls/conn.go:609 +0x124
crypto/tls.(*Conn).readRecord(...)
        /opt/hostedtoolcache/go/1.13.15/x64/src/crypto/tls/conn.go:577
crypto/tls.(*Conn).Read(0xc000071500, 0xc000374000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
        /opt/hostedtoolcache/go/1.13.15/x64/src/crypto/tls/conn.go:1255 +0x161
bufio.(*Reader).Read(0xc00027c720, 0xc0001d0118, 0x9, 0x9, 0xc00010fcc0, 0x0, 0x90d7b5)
        /opt/hostedtoolcache/go/1.13.15/x64/src/bufio/bufio.go:226 +0x26a
io.ReadAtLeast(0x17ee0e0, 0xc00027c720, 0xc0001d0118, 0x9, 0x9, 0x9, 0xc000082060, 0x0, 0x17ee400)
        /opt/hostedtoolcache/go/1.13.15/x64/src/io/io.go:310 +0x87
io.ReadFull(...)
        /opt/hostedtoolcache/go/1.13.15/x64/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0001d0118, 0x9, 0x9, 0x17ee0e0, 0xc00027c720, 0x0, 0x0, 0xc0002c4918, 0x101)
        /home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0001d00e0, 0xc00c8e8030, 0x0, 0x0, 0x0)
        /home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/frame.go:492 +0xa1
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc00010ffb8, 0x0, 0x0)
        /home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1819 +0xbe
golang.org/x/net/http2.(*ClientConn).readLoop(0xc00066af00)
        /home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:1741 +0xa3
created by golang.org/x/net/http2.(*Transport).newClientConn
        /home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:705 +0x68b

goroutine 67 [sync.Cond.Wait]:
runtime.goparkunlock(...)
        /opt/hostedtoolcache/go/1.13.15/x64/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc0002510c0, 0x2)
        /opt/hostedtoolcache/go/1.13.15/x64/src/runtime/sema.go:510 +0xf8
sync.(*Cond).Wait(0xc0002510b0)
        /opt/hostedtoolcache/go/1.13.15/x64/src/sync/cond.go:56 +0x9d
golang.org/x/net/http2.(*pipe).Read(0xc0002510a8, 0xc003d84001, 0x1dff, 0x1dff, 0x0, 0x0, 0x0)
        /home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/pipe.go:65 +0xa6
golang.org/x/net/http2.transportResponseBody.Read(0xc000251080, 0xc003d84001, 0x1dff, 0x1dff, 0x0, 0x0, 0x0)
        /home/runner/go/pkg/mod/golang.org/x/[email protected]/http2/transport.go:2108 +0xac
encoding/json.(*Decoder).refill(0xc000251340, 0xc00027000a, 0x9)
        /opt/hostedtoolcache/go/1.13.15/x64/src/encoding/json/stream.go:161 +0xeb
encoding/json.(*Decoder).readValue(0xc000251340, 0x0, 0x0, 0x1430c40)
        /opt/hostedtoolcache/go/1.13.15/x64/src/encoding/json/stream.go:136 +0x1dc
encoding/json.(*Decoder).Decode(0xc000251340, 0x144d480, 0xc0002700e0, 0x0, 0x0)
        /opt/hostedtoolcache/go/1.13.15/x64/src/encoding/json/stream.go:63 +0x79
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc00098ac90, 0xc003d88000, 0x2000, 0x2600, 0x17f8320, 0x0, 0x38)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/framer/framer.go:152 +0x1a1
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0000cfe50, 0x0, 0x17fa760, 0xc0009bf600, 0x0, 0x0, 0x0, 0xc000976478, 0x4584d0)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/runtime/serializer/streaming/streaming.go:77 +0x89
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc000990ce0, 0xc00010df58, 0x8, 0x17f8320, 0xc012af8000, 0x0, 0x0)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/rest/watch/decoder.go:49 +0x7c
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc00095fec0)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:104 +0x17e
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
        /home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/watch/streamwatcher.go:71 +0xbe

goroutine 68 [chan send]:
github.com/kinvolk/traceloop/pkg/podinformer.(*PodInformer).syncToStdout(0xc0000cf540, 0xc012a88b20, 0x18, 0x13e2200, 0xc0001503d0)
        /home/runner/work/traceloop/traceloop/pkg/podinformer/podinformer.go:238 +0x633
github.com/kinvolk/traceloop/pkg/podinformer.(*PodInformer).processNextItem(0xc0000cf540, 0x0)
        /home/runner/work/traceloop/traceloop/pkg/podinformer/podinformer.go:185 +0xf4
github.com/kinvolk/traceloop/pkg/podinformer.(*PodInformer).runWorker(0xc0000cf540)
        /home/runner/work/traceloop/traceloop/pkg/podinformer/podinformer.go:301 +0x2b
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc000688130)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:155 +0x5e
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc000688130, 0x17ef3c0, 0xc0003ac000, 0x1, 0xc00004a480)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:156 +0xa3
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000688130, 0x3b9aca00, 0x0, 0x1, 0xc00004a480)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:133 +0xaa
k8s.io/apimachinery/pkg/util/wait.Until(0xc000688130, 0x3b9aca00, 0xc00004a480)
        /home/runner/go/pkg/mod/k8s.io/[email protected]/pkg/util/wait/wait.go:90 +0x4d
created by github.com/kinvolk/traceloop/pkg/podinformer.(*PodInformer).Run
        /home/runner/work/traceloop/traceloop/pkg/podinformer/podinformer.go:293 +0x22d
@invidian invidian transferred this issue from inspektor-gadget/inspektor-gadget Jan 5, 2022
Sign up for free to subscribe to this conversation on GitHub. Already have an account? Sign in.
Assignees
No one assigned
Labels
None yet
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
1 participant
@invidian