Add Pyroscope to config (#1252)

.golangci.yml

@@ -75,6 +75,8 @@ linters-settings:
           - "k8s.io/api/core"
           - "k8s.io/apimachinery/pkg"
           - "k8s.io/client-go"
+          - "github.com/grafana/pyroscope-go"
+          - "github.com/sirupsen/logrus"
       tests:
         files:
           - "$test"

cmd/config.go

@@ -218,6 +218,7 @@ func RunConfigV1() {
 	cl := cmdline.NewCmdline()
 	cl.AddConfigType("node", "Specifies the node configuration of this instance", types.NodeCfg{}, cmdline.Required, cmdline.Singleton)
 	cl.AddConfigType("local-only", "Runs a self-contained node with no backend", backends.NullBackendCfg{}, cmdline.Singleton)
+	cl.AddConfigType("pyroscope-client", "Profile Receptor using Pyroscope, client ", types.ReceptorPyroscopeCfg{}, cmdline.Singleton)
 
 	// Add registered config types from imported modules
 	for _, appName := range []string{

docs/source/user_guide/configuration_options.rst

@@ -117,6 +117,74 @@ Run qvis locally:
 - Qvis is now served on port 8080
 - Upload qlogs to qvis and navigate to Sequence
 
+^^^^^^^^^^^^^^^^
+Pyroscope Client
+^^^^^^^^^^^^^^^^
+
+Use pyroscope-client to implement pyroscope push-mode to profile receptor and push to a Pyroscope server.
+
+.. list-table:: Pyroscope client
+    :header-rows: 1
+    :widths: auto
+
+    * - Parameter
+      - Description
+      - Default value
+      - Type
+    * - ``applicationName``
+      - Name of application used in Pyroscope UI
+      - No default value.
+      - string
+    * - ``tags``
+      -  Map of static tags.
+      - No default value.
+      - map of string
+    * - ``serverAddress``
+      - Address of pyroscope server
+      - No default value.
+      - string
+    * - ``basicAuthUser``
+      - Http basic auth user
+      - No default value.
+      - string
+    * - ``basicAuthPassword``
+      - Http basic auth password
+      - No default value.
+      - string
+    * - ``tenantID``
+      - Specify TenantId when using phlare multi-tenancy
+      - No default value.
+      - string
+    * - ``uploadRate``
+      - Upload rate e.g. ``10s``
+      - 15s
+      - string
+    * - ``profileTypes``
+      - Optional profile types- ProfileGoroutines, ProfileMutexCount, ProfileMutexDuration, ProfileBlockCount, ProfileBlockDuration.
+      - Profile types set by default- ProfileCPU, ProfileAllocObjects, ProfileAlloSpace, ProfileInuseObjects, ProfileInuseSpace.
+      - list of sting
+    * - ``disableGCRuns``
+      - This will disable automatic runtime.GC runs between getting the heap profiles
+      - true
+      - bool
+    * - ``HTTPHeaders``
+      - Set the Authorization header manually
+      - No default value.
+      - map of string
+
+
+.. code-block:: yaml
+
+    pyroscope-client:
+      - applicationName: "receptor"
+        serverAddress: "http://localhost:4040"
+        profileTypes: 
+          - ProfileGoroutines
+          - ProfileMutexCount
+          - ProfileMutexDuration
+          - ProfileBlockCount
+          - ProfileBlockDuration
+
 ^^^^
 Node
 ^^^^

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/prep/socketpair v0.0.0-20171228153254-c2c6a7f821c2
 	github.com/quic-go/quic-go v0.48.2
 	github.com/rogpeppe/go-internal v1.13.1
+	github.com/sirupsen/logrus v1.9.3
 	github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/viper v1.19.0
@@ -31,6 +32,8 @@ require (
 	k8s.io/client-go v0.31.3
 )
 
+require github.com/stretchr/testify v1.10.0 // indirect
+
 require (
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
@@ -47,11 +50,14 @@ require (
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20241101162523-b92577c0c142 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/grafana/pyroscope-go v1.2.0
+	github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
@@ -82,7 +88,7 @@ require (
 	golang.org/x/text v0.21.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.27.0 // indirect
-	google.golang.org/protobuf v1.34.2 // indirect
+	google.golang.org/protobuf v1.35.2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -93,6 +93,10 @@ github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE0
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
 github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/grafana/pyroscope-go v1.2.0 h1:aILLKjTj8CS8f/24OPMGPewQSYlhmdQMBmol1d3KGj8=
+github.com/grafana/pyroscope-go v1.2.0/go.mod h1:2GHr28Nr05bg2pElS+dDsc98f3JTUh2f6Fz1hWXrqwk=
+github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg=
+github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
@@ -112,6 +116,8 @@ github.com/jupp0r/go-priority-queue v0.0.0-20160601094913-ab1073853bde h1:+5PMaa
 github.com/jupp0r/go-priority-queue v0.0.0-20160601094913-ab1073853bde/go.mod h1:RDgD/dfPmIwFH0qdUOjw71HjtWg56CtyLIoHL+R1wJw=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
+github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -201,6 +207,8 @@ github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1l
 github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4=
 github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8=
 github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E=
 github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
@@ -220,15 +228,18 @@ github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
@@ -307,6 +318,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
@@ -353,8 +365,8 @@ google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmE
 google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
-google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
+google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
+google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=

pkg/types/main.go

@@ -5,11 +5,16 @@ import (
 	"fmt"
 	"os"
 	"regexp"
+	"runtime"
 	"strings"
+	"time"
 
 	"github.com/ansible/receptor/pkg/controlsvc"
 	"github.com/ansible/receptor/pkg/netceptor"
 	"github.com/ansible/receptor/pkg/workceptor"
+	"github.com/grafana/pyroscope-go"
+	"github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
 )
 
 type NodeCfg struct {
@@ -23,6 +28,8 @@ type NodeCfg struct {
 	ReceptorKubeClientsetRateLimiter string
 }
 
+var receptorDataDir string
+
 func (cfg NodeCfg) Init() error {
 	var err error
 	if cfg.ID == "" {
@@ -46,6 +53,8 @@ func (cfg NodeCfg) Init() error {
 		return fmt.Errorf("node ID \"localhost\" is reserved")
 	}
 
+	receptorDataDir = cfg.DataDir
+
 	netceptor.MainInstance = netceptor.New(context.Background(), cfg.ID)
 
 	if len(cfg.FirewallRules) > 0 {
@@ -69,7 +78,7 @@ func (cfg NodeCfg) Init() error {
 		}
 	}
 
-	workceptor.MainInstance, err = workceptor.New(context.Background(), netceptor.MainInstance, cfg.DataDir)
+	workceptor.MainInstance, err = workceptor.New(context.Background(), netceptor.MainInstance, receptorDataDir)
 	if err != nil {
 		return err
 	}
@@ -87,3 +96,109 @@ func (cfg NodeCfg) Run() error {
 
 	return nil
 }
+
+type ReceptorPyroscopeCfg struct {
+	ApplicationName   string
+	Tags              map[string]string
+	ServerAddress     string // e.g http://pyroscope.services.internal:4040
+	BasicAuthUser     string // http basic auth user
+	BasicAuthPassword string // http basic auth password
+	TenantID          string // specify TenantId when using phlare multi-tenancy
+	UploadRate        string
+	ProfileTypes      []string
+	DisableGCRuns     bool // this will disable automatic runtime.GC runs between getting the heap profiles
+	HTTPHeaders       map[string]string
+}
+
+type UploadRate struct {
+	UploadRate time.Duration `yaml:"uploadRate"`
+}
+
+func (pyroscopeCfg ReceptorPyroscopeCfg) Init() error {
+	if pyroscopeCfg.ApplicationName == "" {
+		return nil
+	}
+
+	runtime.SetMutexProfileFraction(5)
+	runtime.SetBlockProfileRate(5)
+
+	pyroscopeLogger := logrus.New()
+	pyroscopeLogger.SetLevel(logrus.DebugLevel)
+
+	if _, err := os.Stat(receptorDataDir); os.IsNotExist(err) {
+		err := os.MkdirAll(receptorDataDir, 0o700)
+		if err != nil {
+			fmt.Printf("error creating directory: %v", err)
+		}
+	}
+
+	logFile, err := os.OpenFile(fmt.Sprintf("%s/pyroscope.log", receptorDataDir), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o666)
+	if err != nil {
+		pyroscopeLogger.Fatalf("Error opening log file: %v", err)
+	}
+	pyroscopeLogger.SetOutput(logFile)
+
+	pyroscopeLogger.SetFormatter(&logrus.JSONFormatter{})
+
+	_, err = pyroscope.Start(pyroscope.Config{
+		ApplicationName:   pyroscopeCfg.ApplicationName,
+		Tags:              pyroscopeCfg.Tags,
+		ServerAddress:     pyroscopeCfg.ServerAddress,
+		BasicAuthUser:     pyroscopeCfg.BasicAuthUser,
+		BasicAuthPassword: pyroscopeCfg.BasicAuthPassword,
+		TenantID:          pyroscopeCfg.TenantID,
+		UploadRate:        getUploadRate(pyroscopeCfg),
+		Logger:            pyroscopeLogger,
+		ProfileTypes:      getProfileTypes(pyroscopeCfg),
+		DisableGCRuns:     pyroscopeCfg.DisableGCRuns,
+		HTTPHeaders:       pyroscopeCfg.HTTPHeaders,
+	})
+
+	if err != nil {
+		return err
+	} else {
+		return nil
+	}
+}
+
+func getUploadRate(cfg ReceptorPyroscopeCfg) time.Duration {
+	if cfg.UploadRate == "" {
+		return 15 * time.Second
+	}
+	var uploadRate UploadRate
+	err := yaml.Unmarshal([]byte(cfg.UploadRate), &uploadRate)
+	if err != nil {
+		fmt.Println("failed to parse uploadRate from config file")
+	}
+
+	return uploadRate.UploadRate
+}
+
+func getProfileTypes(cfg ReceptorPyroscopeCfg) []pyroscope.ProfileType {
+	profileType := []pyroscope.ProfileType{
+		pyroscope.ProfileCPU,
+		pyroscope.ProfileAllocObjects,
+		pyroscope.ProfileAllocSpace,
+		pyroscope.ProfileInuseObjects,
+		pyroscope.ProfileInuseSpace,
+	}
+	if len(cfg.ProfileTypes) == 0 {
+		return profileType
+	}
+	for _, pt := range cfg.ProfileTypes {
+		switch pt {
+		case "ProfileGoroutines":
+			profileType = append(profileType, pyroscope.ProfileGoroutines)
+		case "ProfileMutexCount":
+			profileType = append(profileType, pyroscope.ProfileMutexCount)
+		case "ProfileMutexDuration":
+			profileType = append(profileType, pyroscope.ProfileMutexDuration)
+		case "ProfileBlockCount":
+			profileType = append(profileType, pyroscope.ProfileBlockCount)
+		case "ProfileBlockDuration":
+			profileType = append(profileType, pyroscope.ProfileBlockDuration)
+		}
+	}
+
+	return profileType
+}