goroutine 2850 [running]: runtime/pprof.writeGoroutineStacks({0x2606300, 0xc000000000}) runtime/pprof/pprof.go:764 +0x6a runtime/pprof.writeGoroutine({0x2606300?, 0xc000000000?}, 0x7b?) runtime/pprof/pprof.go:753 +0x25 runtime/pprof.(*Profile).WriteTo(0x386d730?, {0x2606300?, 0xc000000000?}, 0xc?) runtime/pprof/pprof.go:377 +0x14b net/http/pprof.handler.ServeHTTP({0x22f4803, 0x9}, {0x26200c8, 0xc000000000}, 0xc000168640) net/http/pprof/pprof.go:272 +0x52f github.com/gorilla/mux.(*Router).ServeHTTP(0xc000324000, {0x26200c8, 0xc000000000}, 0xc000168280) github.com/gorilla/mux@v1.8.1/mux.go:212 +0x1e2 net/http.serverHandler.ServeHTTP({0xc0003d20f0?}, {0x26200c8?, 0xc000000000?}, 0x6?) net/http/server.go:3301 +0x8e net/http.(*conn).serve(0xc000110120, {0x26312f8, 0xc00054a000}) net/http/server.go:2102 +0x625 created by net/http.(*Server).Serve in goroutine 39 net/http/server.go:3454 +0x485 goroutine 1 [chan receive, 169 minutes]: github.com/NVIDIA/dcgm-exporter/pkg/cmd.StartDCGMExporterWithSignalSource(0xc000344e80, {0x0, 0x0}) github.com/NVIDIA/dcgm-exporter/pkg/cmd/app.go:542 +0xab4 github.com/NVIDIA/dcgm-exporter/pkg/cmd.startDCGMExporter(...) github.com/NVIDIA/dcgm-exporter/pkg/cmd/app.go:581 github.com/NVIDIA/dcgm-exporter/pkg/cmd.action.func1() github.com/NVIDIA/dcgm-exporter/pkg/cmd/app.go:397 +0x55 github.com/NVIDIA/dcgm-exporter/internal/pkg/stdout.Capture({0x2631090, 0x3948900}, 0xc000521bb0) github.com/NVIDIA/dcgm-exporter/internal/pkg/stdout/capture.go:76 +0x1d6 github.com/NVIDIA/dcgm-exporter/pkg/cmd.action(0xc000344e80?) github.com/NVIDIA/dcgm-exporter/pkg/cmd/app.go:388 +0x47 github.com/urfave/cli/v2.(*Command).Run(0xc000022b00, 0xc000344e80, {0xc0000500a0, 0x1, 0x1}) github.com/urfave/cli/v2@v2.27.6/command.go:276 +0x7be github.com/urfave/cli/v2.(*App).RunContext(0xc00034a400, {0x2631090, 0x3948900}, {0xc0000500a0, 0x1, 0x1}) github.com/urfave/cli/v2@v2.27.6/app.go:333 +0x5a5 github.com/urfave/cli/v2.(*App).Run(0xc000521f30?, {0xc0000500a0?, 0x1?, 0x0?}) github.com/urfave/cli/v2@v2.27.6/app.go:307 +0x2f main.main() github.com/NVIDIA/dcgm-exporter/cmd/dcgm-exporter/main.go:30 +0x5f goroutine 8 [IO wait, 169 minutes]: internal/poll.runtime_pollWait(0x749e375aafc0, 0x72) runtime/netpoll.go:351 +0x85 internal/poll.(*pollDesc).wait(0xc000391560?, 0xc000218000?, 0x1) internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc000391560, {0xc000218000, 0x1000, 0x1000}) internal/poll/fd_unix.go:165 +0x27a os.(*File).read(...) os/file_posix.go:29 os.(*File).Read(0xc00008f320, {0xc000218000?, 0x0?, 0x0?}) os/file.go:124 +0x4f bufio.(*Scanner).Scan(0xc00015a100) bufio/scan.go:219 +0x81e github.com/NVIDIA/dcgm-exporter/internal/pkg/stdout.Capture.func2() github.com/NVIDIA/dcgm-exporter/internal/pkg/stdout/capture.go:58 +0x47 created by github.com/NVIDIA/dcgm-exporter/internal/pkg/stdout.Capture in goroutine 1 github.com/NVIDIA/dcgm-exporter/internal/pkg/stdout/capture.go:57 +0x1c9 goroutine 10 [syscall, 169 minutes]: os/signal.signal_recv() runtime/sigqueue.go:152 +0x29 os/signal.loop() os/signal/signal_unix.go:23 +0x13 created by os/signal.Notify.func1.1 in goroutine 1 os/signal/signal.go:152 +0x1f goroutine 66 [chan receive, 169 minutes]: github.com/NVIDIA/dcgm-exporter/internal/pkg/server.(*MetricsServer).Run(0xc00015a080, {0x2631090, 0x3948900}, 0xc0004362a0) github.com/NVIDIA/dcgm-exporter/internal/pkg/server/server.go:219 +0x17e github.com/NVIDIA/dcgm-exporter/pkg/cmd.StartDCGMExporterWithSignalSource.func2() github.com/NVIDIA/dcgm-exporter/pkg/cmd/app.go:513 +0x53 created by github.com/NVIDIA/dcgm-exporter/pkg/cmd.StartDCGMExporterWithSignalSource in goroutine 1 github.com/NVIDIA/dcgm-exporter/pkg/cmd/app.go:511 +0x751 goroutine 67 [select, 169 minutes]: github.com/NVIDIA/dcgm-exporter/internal/pkg/watcher.(*FileWatcher).Watch(0xc0003703c0, {0x2631330, 0xc000024690}, 0xc00029aed0) github.com/NVIDIA/dcgm-exporter/internal/pkg/watcher/file.go:110 +0x6a6 github.com/NVIDIA/dcgm-exporter/pkg/cmd.runWatcher.func1() github.com/NVIDIA/dcgm-exporter/pkg/cmd/app.go:1131 +0x73 created by github.com/NVIDIA/dcgm-exporter/pkg/cmd.runWatcher in goroutine 1 github.com/NVIDIA/dcgm-exporter/pkg/cmd/app.go:1129 +0xc5 goroutine 37 [sync.WaitGroup.Wait, 169 minutes]: sync.runtime_SemacquireWaitGroup(0xc000010120?) runtime/sema.go:110 +0x25 sync.(*WaitGroup).Wait(0x2132520?) sync/waitgroup.go:118 +0x48 golang.org/x/sync/errgroup.(*Group).Wait(0xc00020c2c0) golang.org/x/sync@v0.18.0/errgroup/errgroup.go:56 +0x1e github.com/prometheus/exporter-toolkit/web.ServeMultiple({0xc0003b0090, 0x1, 0x22ec114?}, 0xc00015c000, 0xc000580060, 0xc000411450) github.com/prometheus/exporter-toolkit@v0.14.0/web/tls_config.go:277 +0x117 github.com/prometheus/exporter-toolkit/web.ListenAndServe(0xc00015c000, 0xc000580060, 0xc000411450) github.com/prometheus/exporter-toolkit@v0.14.0/web/tls_config.go:325 +0x412 github.com/NVIDIA/dcgm-exporter/internal/pkg/server.(*MetricsServer).Run.func1() github.com/NVIDIA/dcgm-exporter/internal/pkg/server/server.go:190 +0x345 created by github.com/NVIDIA/dcgm-exporter/internal/pkg/server.(*MetricsServer).Run in goroutine 66 github.com/NVIDIA/dcgm-exporter/internal/pkg/server/server.go:175 +0xc5 goroutine 38 [select, 52 minutes]: github.com/NVIDIA/dcgm-exporter/internal/pkg/server.(*MetricsServer).Run.func2() github.com/NVIDIA/dcgm-exporter/internal/pkg/server/server.go:204 +0x145 created by github.com/NVIDIA/dcgm-exporter/internal/pkg/server.(*MetricsServer).Run in goroutine 66 github.com/NVIDIA/dcgm-exporter/internal/pkg/server/server.go:197 +0x16f goroutine 68 [IO wait, 169 minutes]: internal/poll.runtime_pollWait(0x749e375aaea8, 0x72) runtime/netpoll.go:351 +0x85 internal/poll.(*pollDesc).wait(0xc0000682a0?, 0xc0005a3e3b?, 0x1) internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Read(0xc0000682a0, {0xc0005a3e3b, 0x10000, 0x10000}) internal/poll/fd_unix.go:165 +0x27a os.(*File).read(...) os/file_posix.go:29 os.(*File).Read(0xc000316130, {0xc0005a3e3b?, 0x0?, 0x0?}) os/file.go:124 +0x4f github.com/fsnotify/fsnotify.(*Watcher).readEvents(0xc000448540) github.com/fsnotify/fsnotify@v1.7.0/backend_inotify.go:483 +0xc5 created by github.com/fsnotify/fsnotify.NewBufferedWatcher in goroutine 67 github.com/fsnotify/fsnotify@v1.7.0/backend_inotify.go:270 +0x1c5 goroutine 39 [IO wait]: internal/poll.runtime_pollWait(0x749e375aad90, 0x72) runtime/netpoll.go:351 +0x85 internal/poll.(*pollDesc).wait(0xc000318080?, 0x90042a7de?, 0x0) internal/poll/fd_poll_runtime.go:84 +0x27 internal/poll.(*pollDesc).waitRead(...) internal/poll/fd_poll_runtime.go:89 internal/poll.(*FD).Accept(0xc000318080) internal/poll/fd_unix.go:620 +0x295 net.(*netFD).accept(0xc000318080) net/fd_unix.go:172 +0x29 net.(*TCPListener).accept(0xc00020c280) net/tcpsock_posix.go:159 +0x1b net.(*TCPListener).Accept(0xc00020c280) net/tcpsock.go:380 +0x30 net/http.(*Server).Serve(0xc00015c000, {0x261fee8, 0xc00020c280}) net/http/server.go:3424 +0x30c github.com/prometheus/exporter-toolkit/web.Serve({0x261fee8, 0xc00020c280}, 0xc00015c000, 0xc000580060, 0xc000411450) github.com/prometheus/exporter-toolkit@v0.14.0/web/tls_config.go:351 +0x756 github.com/prometheus/exporter-toolkit/web.ServeMultiple.func1() github.com/prometheus/exporter-toolkit@v0.14.0/web/tls_config.go:274 +0x27 golang.org/x/sync/errgroup.(*Group).Go.func1() golang.org/x/sync@v0.18.0/errgroup/errgroup.go:93 +0x50 created by golang.org/x/sync/errgroup.(*Group).Go in goroutine 37 golang.org/x/sync@v0.18.0/errgroup/errgroup.go:78 +0x93 goroutine 2851 [runnable]: syscall.Syscall(0x0, 0x19, 0xc0003d2101, 0x1) syscall/syscall_linux.go:73 +0x25 syscall.read(0xc00015a000?, {0xc0003d2101?, 0x394a300?, 0xc00077ded0?}) syscall/zsyscall_linux_amd64.go:736 +0x38 syscall.Read(...) syscall/syscall_unix.go:183 internal/poll.ignoringEINTRIO(...) internal/poll/fd_unix.go:745 internal/poll.(*FD).Read(0xc00015a000, {0xc0003d2101, 0x1, 0x1}) internal/poll/fd_unix.go:161 +0x2ae net.(*netFD).Read(0xc00015a000, {0xc0003d2101?, 0xc000730118?, 0xc00077df70?}) net/fd_posix.go:55 +0x25 net.(*conn).Read(0xc000396060, {0xc0003d2101?, 0xc0003f5080?, 0x19a74e0?}) net/net.go:194 +0x45 net/http.(*connReader).backgroundRead(0xc0003d20f0) net/http/server.go:690 +0x37 created by net/http.(*connReader).startBackgroundRead in goroutine 2850 net/http/server.go:686 +0xb6