goroutine 404810 [running]: runtime/pprof.writeGoroutineStacks({0x2606300, 0xc0003e2460}) runtime/pprof/pprof.go:764 +0x6a runtime/pprof.writeGoroutine({0x2606300?, 0xc0003e2460?}, 0x67?) runtime/pprof/pprof.go:753 +0x25 runtime/pprof.(*Profile).WriteTo(0x386d730?, {0x2606300?, 0xc0003e2460?}, 0xc?) runtime/pprof/pprof.go:377 +0x14b net/http/pprof.handler.ServeHTTP({0x22f4803, 0x9}, {0x26200c8, 0xc0003e2460}, 0xc000227540) net/http/pprof/pprof.go:272 +0x52f github.com/gorilla/mux.(*Router).ServeHTTP(0xc000052240, {0x26200c8, 0xc0003e2460}, 0xc0002272c0) github.com/gorilla/mux@v1.8.1/mux.go:212 +0x1e2 net/http.serverHandler.ServeHTTP({0xc000740e40?}, {0x26200c8?, 0xc0003e2460?}, 0x6?) net/http/server.go:3301 +0x8e net/http.(*conn).serve(0xc0007d65a0, {0x26312f8, 0xc0002b4360}) net/http/server.go:2102 +0x625 created by net/http.(*Server).Serve in goroutine 38 net/http/server.go:3454 +0x485 goroutine 1 [chan receive, 24715 minutes]: github.com/NVIDIA/dcgm-exporter/pkg/cmd.StartDCGMExporterWithSignalSource(0xc0001b99c0, {0x0, 0x0}) github.com/NVIDIA/dcgm-exporter/pkg/cmd/app.go:542 +0xab4 github.com/NVIDIA/dcgm-exporter/pkg/cmd.startDCGMExporter(...) github.com/NVIDIA/dcgm-exporter/pkg/cmd/app.go:581 github.com/NVIDIA/dcgm-exporter/pkg/cmd.action.func1() github.com/NVIDIA/dcgm-exporter/pkg/cmd/app.go:397 +0x55 github.com/NVIDIA/dcgm-exporter/internal/pkg/stdout.Capture({0x2631090, 0x3948900}, 0xc00035dbb0) github.com/NVIDIA/dcgm-exporter/internal/pkg/stdout/capture.go:76 +0x1d6 github.com/NVIDIA/dcgm-exporter/pkg/cmd.action(0xc0001b99c0?) github.com/NVIDIA/dcgm-exporter/pkg/cmd/app.go:388 +0x47 github.com/urfave/cli/v2.(*Command).Run(0xc00038b080, 0xc0001b99c0, {0xc0001aa050, 0x1, 0x1}) github.com/urfave/cli/v2@v2.27.6/command.go:276 +0x7be github.com/urfave/cli/v2.(*App).RunContext(0xc00042c000, {0x2631090, 0x3948900}, {0xc0001aa050, 0x1, 0x1}) github.com/urfave/cli/v2@v2.27.6/app.go:333 +0x5a5 github.com/urfave/cli/v2.(*App).Run(0xc00035df30?, {0xc0001aa050?, 0x1?, 0x0?}) github.com/urfave/cli/v2@v2.27.6/app.go:307 +0x2f main.main() github.com/NVIDIA/dcgm-exporter/cmd/dcgm-exporter/main.go:30 +0x5f goroutine 66 [IO wait, 24715 minutes]: internal/poll.runtime_pollWait(0x7a1c898c5ef8, 0x72) runtime/netpoll.go:351 +0x85 internal/poll.(*pollDesc).wait(0xc000119320?, 0xc000322000?, 0x1) internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000119320, {0xc000322000, 0x1000, 0x1000}) internal/poll/fd_unix.go:165 +0x27a os.(*File).read(...) os/file_posix.go:29 os.(*File).Read(0xc0003d0ad8, {0xc000322000?, 0x0?, 0x0?}) os/file.go:124 +0x4f bufio.(*Scanner).Scan(0xc0003d4500) bufio/scan.go:219 +0x81e github.com/NVIDIA/dcgm-exporter/internal/pkg/stdout.Capture.func2() github.com/NVIDIA/dcgm-exporter/internal/pkg/stdout/capture.go:58 +0x47 created by github.com/NVIDIA/dcgm-exporter/internal/pkg/stdout.Capture in goroutine 1 github.com/NVIDIA/dcgm-exporter/internal/pkg/stdout/capture.go:57 +0x1c9 goroutine 82 [syscall, 24715 minutes]: os/signal.signal_recv() runtime/sigqueue.go:152 +0x29 os/signal.loop() os/signal/signal_unix.go:23 +0x13 created by os/signal.Notify.func1.1 in goroutine 1 os/signal/signal.go:152 +0x1f goroutine 22 [chan receive, 24715 minutes]: github.com/NVIDIA/dcgm-exporter/internal/pkg/server.(*MetricsServer).Run(0xc0003d4d00, {0x2631090, 0x3948900}, 0xc00041a4d0) github.com/NVIDIA/dcgm-exporter/internal/pkg/server/server.go:219 +0x17e github.com/NVIDIA/dcgm-exporter/pkg/cmd.StartDCGMExporterWithSignalSource.func2() github.com/NVIDIA/dcgm-exporter/pkg/cmd/app.go:513 +0x53 created by github.com/NVIDIA/dcgm-exporter/pkg/cmd.StartDCGMExporterWithSignalSource in goroutine 1 github.com/NVIDIA/dcgm-exporter/pkg/cmd/app.go:511 +0x751 goroutine 23 [select, 24715 minutes]: github.com/NVIDIA/dcgm-exporter/internal/pkg/watcher.(*FileWatcher).Watch(0xc0000344a0, {0x2631330, 0xc0002f8870}, 0xc000024ba0) github.com/NVIDIA/dcgm-exporter/internal/pkg/watcher/file.go:110 +0x6a6 github.com/NVIDIA/dcgm-exporter/pkg/cmd.runWatcher.func1() github.com/NVIDIA/dcgm-exporter/pkg/cmd/app.go:1131 +0x73 created by github.com/NVIDIA/dcgm-exporter/pkg/cmd.runWatcher in goroutine 1 github.com/NVIDIA/dcgm-exporter/pkg/cmd/app.go:1129 +0xc5 goroutine 54 [IO wait, 24715 minutes]: internal/poll.runtime_pollWait(0x7a1c898c5de0, 0x72) runtime/netpoll.go:351 +0x85 internal/poll.(*pollDesc).wait(0xc0001ba120?, 0xc000623e3b?, 0x1) internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0001ba120, {0xc000623e3b, 0x10000, 0x10000}) internal/poll/fd_unix.go:165 +0x27a os.(*File).read(...) os/file_posix.go:29 os.(*File).Read(0xc0001a2158, {0xc000623e3b?, 0x0?, 0x0?}) os/file.go:124 +0x4f github.com/fsnotify/fsnotify.(*Watcher).readEvents(0xc000026040) github.com/fsnotify/fsnotify@v1.7.0/backend_inotify.go:483 +0xc5 created by github.com/fsnotify/fsnotify.NewBufferedWatcher in goroutine 23 github.com/fsnotify/fsnotify@v1.7.0/backend_inotify.go:270 +0x1c5 goroutine 36 [sync.WaitGroup.Wait, 24715 minutes]: sync.runtime_SemacquireWaitGroup(0xc0001dc2a0?) runtime/sema.go:110 +0x25 sync.(*WaitGroup).Wait(0x2132520?) sync/waitgroup.go:118 +0x48 golang.org/x/sync/errgroup.(*Group).Wait(0xc0004428c0) golang.org/x/sync@v0.18.0/errgroup/errgroup.go:56 +0x1e github.com/prometheus/exporter-toolkit/web.ServeMultiple({0xc000000040, 0x1, 0x22ec114?}, 0xc0004b6300, 0xc000010378, 0xc00059e9b0) github.com/prometheus/exporter-toolkit@v0.14.0/web/tls_config.go:277 +0x117 github.com/prometheus/exporter-toolkit/web.ListenAndServe(0xc0004b6300, 0xc000010378, 0xc00059e9b0) github.com/prometheus/exporter-toolkit@v0.14.0/web/tls_config.go:325 +0x412 github.com/NVIDIA/dcgm-exporter/internal/pkg/server.(*MetricsServer).Run.func1() github.com/NVIDIA/dcgm-exporter/internal/pkg/server/server.go:190 +0x345 created by github.com/NVIDIA/dcgm-exporter/internal/pkg/server.(*MetricsServer).Run in goroutine 22 github.com/NVIDIA/dcgm-exporter/internal/pkg/server/server.go:175 +0xc5 goroutine 37 [select, 55 minutes]: github.com/NVIDIA/dcgm-exporter/internal/pkg/server.(*MetricsServer).Run.func2() github.com/NVIDIA/dcgm-exporter/internal/pkg/server/server.go:204 +0x145 created by github.com/NVIDIA/dcgm-exporter/internal/pkg/server.(*MetricsServer).Run in goroutine 22 github.com/NVIDIA/dcgm-exporter/internal/pkg/server/server.go:197 +0x16f goroutine 38 [IO wait]: internal/poll.runtime_pollWait(0x7a1c898c5a98, 0x72) runtime/netpoll.go:351 +0x85 internal/poll.(*pollDesc).wait(0xc0003d6080?, 0x900000036?, 0x0) internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Accept(0xc0003d6080) internal/poll/fd_unix.go:620 +0x295 net.(*netFD).accept(0xc0003d6080) net/fd_unix.go:172 +0x29 net.(*TCPListener).accept(0xc000442880) net/tcpsock_posix.go:159 +0x1b net.(*TCPListener).Accept(0xc000442880) net/tcpsock.go:380 +0x30 net/http.(*Server).Serve(0xc0004b6300, {0x261fee8, 0xc000442880}) net/http/server.go:3424 +0x30c github.com/prometheus/exporter-toolkit/web.Serve({0x261fee8, 0xc000442880}, 0xc0004b6300, 0xc000010378, 0xc00059e9b0) github.com/prometheus/exporter-toolkit@v0.14.0/web/tls_config.go:351 +0x756 github.com/prometheus/exporter-toolkit/web.ServeMultiple.func1() github.com/prometheus/exporter-toolkit@v0.14.0/web/tls_config.go:274 +0x27 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync@v0.18.0/errgroup/errgroup.go:93 +0x50 created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 36 golang.org/x/sync@v0.18.0/errgroup/errgroup.go:78 +0x93 goroutine 404811 [IO wait]: internal/poll.runtime_pollWait(0x7a1c898c5bb0, 0x72) runtime/netpoll.go:351 +0x85 internal/poll.(*pollDesc).wait(0xc000176780?, 0xc000740e51?, 0x0) internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000176780, {0xc000740e51, 0x1, 0x1}) internal/poll/fd_unix.go:165 +0x27a net.(*netFD).Read(0xc000176780, {0xc000740e51?, 0xc0001b8b98?, 0xc0004fdf70?}) net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc0003d00a8, {0xc000740e51?, 0xc00082f1c0?, 0x19a74e0?}) net/net.go:194 +0x45 net/http.(*connReader).backgroundRead(0xc000740e40) net/http/server.go:690 +0x37 created by net/http.(*connReader).startBackgroundRead in goroutine 404810 net/http/server.go:686 +0xb6