diff --git a/go.mod b/go.mod index 1dc6758e0..ffdb04b52 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/mozilla-services/autograph go 1.23.4 require ( - github.com/DataDog/datadog-go v4.8.3+incompatible + github.com/DataDog/datadog-go/v5 v5.6.0 github.com/aws/aws-lambda-go v1.47.0 github.com/aws/aws-sdk-go-v2 v1.32.8 github.com/aws/aws-sdk-go-v2/config v1.28.10 diff --git a/go.sum b/go.sum index fe9324cba..3a9914c58 100644 --- a/go.sum +++ b/go.sum @@ -47,8 +47,8 @@ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mo github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 h1:kYRSnvJju5gYVyhkij+RTJ/VR6QIUaCfWeaFm2ycsjQ= github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= -github.com/DataDog/datadog-go v4.8.3+incompatible h1:fNGaYSuObuQb5nzeTQqowRAd9bpDIRRV4/gUtIBjh8Q= -github.com/DataDog/datadog-go v4.8.3+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/DataDog/datadog-go/v5 v5.6.0 h1:2oCLxjF/4htd55piM75baflj/KoE6VYS7alEUqFvRDw= +github.com/DataDog/datadog-go/v5 v5.6.0/go.mod h1:K9kcYBlxkcPP8tvvjZZKs/m1edNAUFzBbdpTUKfCsuw= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 h1:3c8yed4lgqTt+oTQ+JNMDo+F4xprBf+O/il4ZC0nRLw= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0/go.mod h1:obipzmGjfSjam60XLwGfqUkJsfiheAl+TUjG+4yzyPM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.49.0 h1:o90wcURuxekmXrtxmYWTyNla0+ZEHhud6DI1ZTxd1vI= @@ -57,6 +57,7 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.49.0/go.mod h1:l2fIqmwB+FKSfvn3bAD/0i+AXAxhIZjTK2svT/mgUXs= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.49.0 h1:GYUJLfvd++4DMuMhCFLgLXvFwofIxh/qOwoGuS/LTew= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.49.0/go.mod h1:wRbFgBQUVm1YXrvWKofAEmq9HNJTDphbAaJSSX01KUI= +github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= @@ -269,6 +270,7 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -277,9 +279,11 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= @@ -349,7 +353,9 @@ golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/mockstatsd/clientinterface.go b/internal/mockstatsd/clientinterface.go index 38717cfcb..904033173 100644 --- a/internal/mockstatsd/clientinterface.go +++ b/internal/mockstatsd/clientinterface.go @@ -1,9 +1,9 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: vendor/github.com/DataDog/datadog-go/statsd/statsd.go +// Source: vendor/github.com/DataDog/datadog-go/v5/statsd/statsd.go // // Generated by this command: // -// mockgen -package mockstatsd -source vendor/github.com/DataDog/datadog-go/statsd/statsd.go -destination internal/mockstatsd/clientinterface.go -imports github.com/mozilla-services/autograph/vendor/github.com/DataDog/datadog-go/statsd=github.com/DataDog/datadog-go/statsd ClientInterface +// mockgen -package mockstatsd -source vendor/github.com/DataDog/datadog-go/v5/statsd/statsd.go -destination internal/mockstatsd/clientinterface.go -imports github.com/mozilla-services/autograph/vendor/github.com/DataDog/datadog-go/v5/statsd=github.com/DataDog/datadog-go/v5/statsd ClientInterface // // Package mockstatsd is a generated GoMock package. @@ -13,7 +13,7 @@ import ( reflect "reflect" time "time" - statsd "github.com/DataDog/datadog-go/statsd" + statsd "github.com/DataDog/datadog-go/v5/statsd" gomock "go.uber.org/mock/gomock" ) @@ -68,6 +68,20 @@ func (mr *MockClientInterfaceMockRecorder) Count(name, value, tags, rate any) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Count", reflect.TypeOf((*MockClientInterface)(nil).Count), name, value, tags, rate) } +// CountWithTimestamp mocks base method. +func (m *MockClientInterface) CountWithTimestamp(name string, value int64, tags []string, rate float64, timestamp time.Time) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CountWithTimestamp", name, value, tags, rate, timestamp) + ret0, _ := ret[0].(error) + return ret0 +} + +// CountWithTimestamp indicates an expected call of CountWithTimestamp. +func (mr *MockClientInterfaceMockRecorder) CountWithTimestamp(name, value, tags, rate, timestamp any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountWithTimestamp", reflect.TypeOf((*MockClientInterface)(nil).CountWithTimestamp), name, value, tags, rate, timestamp) +} + // Decr mocks base method. func (m *MockClientInterface) Decr(name string, tags []string, rate float64) error { m.ctrl.T.Helper() @@ -138,6 +152,34 @@ func (mr *MockClientInterfaceMockRecorder) Gauge(name, value, tags, rate any) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Gauge", reflect.TypeOf((*MockClientInterface)(nil).Gauge), name, value, tags, rate) } +// GaugeWithTimestamp mocks base method. +func (m *MockClientInterface) GaugeWithTimestamp(name string, value float64, tags []string, rate float64, timestamp time.Time) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GaugeWithTimestamp", name, value, tags, rate, timestamp) + ret0, _ := ret[0].(error) + return ret0 +} + +// GaugeWithTimestamp indicates an expected call of GaugeWithTimestamp. +func (mr *MockClientInterfaceMockRecorder) GaugeWithTimestamp(name, value, tags, rate, timestamp any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GaugeWithTimestamp", reflect.TypeOf((*MockClientInterface)(nil).GaugeWithTimestamp), name, value, tags, rate, timestamp) +} + +// GetTelemetry mocks base method. +func (m *MockClientInterface) GetTelemetry() statsd.Telemetry { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTelemetry") + ret0, _ := ret[0].(statsd.Telemetry) + return ret0 +} + +// GetTelemetry indicates an expected call of GetTelemetry. +func (mr *MockClientInterfaceMockRecorder) GetTelemetry() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTelemetry", reflect.TypeOf((*MockClientInterface)(nil).GetTelemetry)) +} + // Histogram mocks base method. func (m *MockClientInterface) Histogram(name string, value float64, tags []string, rate float64) error { m.ctrl.T.Helper() @@ -166,6 +208,20 @@ func (mr *MockClientInterfaceMockRecorder) Incr(name, tags, rate any) *gomock.Ca return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Incr", reflect.TypeOf((*MockClientInterface)(nil).Incr), name, tags, rate) } +// IsClosed mocks base method. +func (m *MockClientInterface) IsClosed() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsClosed") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsClosed indicates an expected call of IsClosed. +func (mr *MockClientInterfaceMockRecorder) IsClosed() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsClosed", reflect.TypeOf((*MockClientInterface)(nil).IsClosed)) +} + // ServiceCheck mocks base method. func (m *MockClientInterface) ServiceCheck(sc *statsd.ServiceCheck) error { m.ctrl.T.Helper() @@ -194,20 +250,6 @@ func (mr *MockClientInterfaceMockRecorder) Set(name, value, tags, rate any) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Set", reflect.TypeOf((*MockClientInterface)(nil).Set), name, value, tags, rate) } -// SetWriteTimeout mocks base method. -func (m *MockClientInterface) SetWriteTimeout(d time.Duration) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetWriteTimeout", d) - ret0, _ := ret[0].(error) - return ret0 -} - -// SetWriteTimeout indicates an expected call of SetWriteTimeout. -func (mr *MockClientInterfaceMockRecorder) SetWriteTimeout(d any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetWriteTimeout", reflect.TypeOf((*MockClientInterface)(nil).SetWriteTimeout), d) -} - // SimpleEvent mocks base method. func (m *MockClientInterface) SimpleEvent(title, text string) error { m.ctrl.T.Helper() diff --git a/main.go b/main.go index 9fa4128f0..41abbe60b 100644 --- a/main.go +++ b/main.go @@ -41,7 +41,7 @@ import ( sops "github.com/getsops/sops/v3" "github.com/getsops/sops/v3/decrypt" - "github.com/DataDog/datadog-go/statsd" + "github.com/DataDog/datadog-go/v5/statsd" "github.com/mozilla-services/autograph/crypto11" ) diff --git a/monitor.go b/monitor.go index 3bb133193..659904716 100644 --- a/monitor.go +++ b/monitor.go @@ -7,7 +7,7 @@ import ( "sync" "time" - "github.com/DataDog/datadog-go/statsd" + "github.com/DataDog/datadog-go/v5/statsd" "github.com/mozilla-services/autograph/formats" "github.com/mozilla-services/autograph/signer" log "github.com/sirupsen/logrus" diff --git a/signer/signer.go b/signer/signer.go index b548c2303..e2067c040 100644 --- a/signer/signer.go +++ b/signer/signer.go @@ -25,7 +25,7 @@ import ( "github.com/mozilla-services/autograph/database" "github.com/mozilla-services/autograph/formats" - "github.com/DataDog/datadog-go/statsd" + "github.com/DataDog/datadog-go/v5/statsd" "github.com/mozilla-services/autograph/crypto11" log "github.com/sirupsen/logrus" diff --git a/signer/xpi/xpi_test.go b/signer/xpi/xpi_test.go index c795aa4a8..df4344ab5 100644 --- a/signer/xpi/xpi_test.go +++ b/signer/xpi/xpi_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/DataDog/datadog-go/statsd" + "github.com/DataDog/datadog-go/v5/statsd" "github.com/mozilla-services/autograph/signer" "go.mozilla.org/pkcs7" ) diff --git a/stats.go b/stats.go index 441002675..090c86b44 100644 --- a/stats.go +++ b/stats.go @@ -5,17 +5,15 @@ import ( "net/http" "sync/atomic" - "github.com/DataDog/datadog-go/statsd" - + "github.com/DataDog/datadog-go/v5/statsd" log "github.com/sirupsen/logrus" ) func loadStatsd(conf configuration) (*statsd.Client, error) { - statsdClient, err := statsd.NewBuffered(conf.Statsd.Addr, conf.Statsd.Buflen) + statsdClient, err := statsd.New(conf.Statsd.Addr, statsd.WithNamespace(conf.Statsd.Namespace)) if err != nil { return nil, fmt.Errorf("error constructing statsdClient: %w", err) } - statsdClient.Namespace = conf.Statsd.Namespace return statsdClient, nil } diff --git a/vendor/github.com/DataDog/datadog-go/statsd/buffered_metric_context.go b/vendor/github.com/DataDog/datadog-go/statsd/buffered_metric_context.go deleted file mode 100644 index 15bba2861..000000000 --- a/vendor/github.com/DataDog/datadog-go/statsd/buffered_metric_context.go +++ /dev/null @@ -1,82 +0,0 @@ -package statsd - -import ( - "math/rand" - "sync" - "sync/atomic" - "time" -) - -// bufferedMetricContexts represent the contexts for Histograms, Distributions -// and Timing. Since those 3 metric types behave the same way and are sampled -// with the same type they're represented by the same class. -type bufferedMetricContexts struct { - nbContext int32 - mutex sync.RWMutex - values bufferedMetricMap - newMetric func(string, float64, string) *bufferedMetric - - // Each bufferedMetricContexts uses its own random source and random - // lock to prevent goroutines from contending for the lock on the - // "math/rand" package-global random source (e.g. calls like - // "rand.Float64()" must acquire a shared lock to get the next - // pseudorandom number). - random *rand.Rand - randomLock sync.Mutex -} - -func newBufferedContexts(newMetric func(string, float64, string) *bufferedMetric) bufferedMetricContexts { - return bufferedMetricContexts{ - values: bufferedMetricMap{}, - newMetric: newMetric, - // Note that calling "time.Now().UnixNano()" repeatedly quickly may return - // very similar values. That's fine for seeding the worker-specific random - // source because we just need an evenly distributed stream of float values. - // Do not use this random source for cryptographic randomness. - random: rand.New(rand.NewSource(time.Now().UnixNano())), - } -} - -func (bc *bufferedMetricContexts) flush(metrics []metric) []metric { - bc.mutex.Lock() - values := bc.values - bc.values = bufferedMetricMap{} - bc.mutex.Unlock() - - for _, d := range values { - metrics = append(metrics, d.flushUnsafe()) - } - atomic.AddInt32(&bc.nbContext, int32(len(values))) - return metrics -} - -func (bc *bufferedMetricContexts) sample(name string, value float64, tags []string, rate float64) error { - if !shouldSample(rate, bc.random, &bc.randomLock) { - return nil - } - - context, stringTags := getContextAndTags(name, tags) - - bc.mutex.RLock() - if v, found := bc.values[context]; found { - v.sample(value) - bc.mutex.RUnlock() - return nil - } - bc.mutex.RUnlock() - - bc.mutex.Lock() - // Check if another goroutines hasn't created the value betwen the 'RUnlock' and 'Lock' - if v, found := bc.values[context]; found { - v.sample(value) - bc.mutex.Unlock() - return nil - } - bc.values[context] = bc.newMetric(name, value, stringTags) - bc.mutex.Unlock() - return nil -} - -func (bc *bufferedMetricContexts) resetAndGetNbContext() int32 { - return atomic.SwapInt32(&bc.nbContext, 0) -} diff --git a/vendor/github.com/DataDog/datadog-go/statsd/metrics.go b/vendor/github.com/DataDog/datadog-go/statsd/metrics.go deleted file mode 100644 index 99ed4da53..000000000 --- a/vendor/github.com/DataDog/datadog-go/statsd/metrics.go +++ /dev/null @@ -1,181 +0,0 @@ -package statsd - -import ( - "math" - "sync" - "sync/atomic" -) - -/* -Those are metrics type that can be aggregated on the client side: - - Gauge - - Count - - Set -*/ - -type countMetric struct { - value int64 - name string - tags []string -} - -func newCountMetric(name string, value int64, tags []string) *countMetric { - return &countMetric{ - value: value, - name: name, - tags: tags, - } -} - -func (c *countMetric) sample(v int64) { - atomic.AddInt64(&c.value, v) -} - -func (c *countMetric) flushUnsafe() metric { - return metric{ - metricType: count, - name: c.name, - tags: c.tags, - rate: 1, - ivalue: c.value, - } -} - -// Gauge - -type gaugeMetric struct { - value uint64 - name string - tags []string -} - -func newGaugeMetric(name string, value float64, tags []string) *gaugeMetric { - return &gaugeMetric{ - value: math.Float64bits(value), - name: name, - tags: tags, - } -} - -func (g *gaugeMetric) sample(v float64) { - atomic.StoreUint64(&g.value, math.Float64bits(v)) -} - -func (g *gaugeMetric) flushUnsafe() metric { - return metric{ - metricType: gauge, - name: g.name, - tags: g.tags, - rate: 1, - fvalue: math.Float64frombits(g.value), - } -} - -// Set - -type setMetric struct { - data map[string]struct{} - name string - tags []string - sync.Mutex -} - -func newSetMetric(name string, value string, tags []string) *setMetric { - set := &setMetric{ - data: map[string]struct{}{}, - name: name, - tags: tags, - } - set.data[value] = struct{}{} - return set -} - -func (s *setMetric) sample(v string) { - s.Lock() - defer s.Unlock() - s.data[v] = struct{}{} -} - -// Sets are aggregated on the agent side too. We flush the keys so a set from -// multiple application can be correctly aggregated on the agent side. -func (s *setMetric) flushUnsafe() []metric { - if len(s.data) == 0 { - return nil - } - - metrics := make([]metric, len(s.data)) - i := 0 - for value := range s.data { - metrics[i] = metric{ - metricType: set, - name: s.name, - tags: s.tags, - rate: 1, - svalue: value, - } - i++ - } - return metrics -} - -// Histograms, Distributions and Timings - -type bufferedMetric struct { - sync.Mutex - - data []float64 - name string - // Histograms and Distributions store tags as one string since we need - // to compute its size multiple time when serializing. - tags string - mtype metricType -} - -func (s *bufferedMetric) sample(v float64) { - s.Lock() - defer s.Unlock() - s.data = append(s.data, v) -} - -func (s *bufferedMetric) flushUnsafe() metric { - return metric{ - metricType: s.mtype, - name: s.name, - stags: s.tags, - rate: 1, - fvalues: s.data, - } -} - -type histogramMetric = bufferedMetric - -func newHistogramMetric(name string, value float64, stringTags string) *histogramMetric { - return &histogramMetric{ - data: []float64{value}, - name: name, - tags: stringTags, - mtype: histogramAggregated, - } -} - -type distributionMetric = bufferedMetric - -func newDistributionMetric(name string, value float64, stringTags string) *distributionMetric { - return &distributionMetric{ - data: []float64{value}, - name: name, - tags: stringTags, - mtype: distributionAggregated, - } -} - -type timingMetric = bufferedMetric - -func newTimingMetric(name string, value float64, stringTags string) *timingMetric { - return &timingMetric{ - data: []float64{value}, - name: name, - tags: stringTags, - mtype: timingAggregated, - } -} diff --git a/vendor/github.com/DataDog/datadog-go/statsd/options.go b/vendor/github.com/DataDog/datadog-go/statsd/options.go deleted file mode 100644 index 9db27039c..000000000 --- a/vendor/github.com/DataDog/datadog-go/statsd/options.go +++ /dev/null @@ -1,323 +0,0 @@ -package statsd - -import ( - "fmt" - "math" - "strings" - "time" -) - -var ( - // DefaultNamespace is the default value for the Namespace option - DefaultNamespace = "" - // DefaultTags is the default value for the Tags option - DefaultTags = []string{} - // DefaultMaxBytesPerPayload is the default value for the MaxBytesPerPayload option - DefaultMaxBytesPerPayload = 0 - // DefaultMaxMessagesPerPayload is the default value for the MaxMessagesPerPayload option - DefaultMaxMessagesPerPayload = math.MaxInt32 - // DefaultBufferPoolSize is the default value for the DefaultBufferPoolSize option - DefaultBufferPoolSize = 0 - // DefaultBufferFlushInterval is the default value for the BufferFlushInterval option - DefaultBufferFlushInterval = 100 * time.Millisecond - // DefaultBufferShardCount is the default value for the BufferShardCount option - DefaultBufferShardCount = 32 - // DefaultSenderQueueSize is the default value for the DefaultSenderQueueSize option - DefaultSenderQueueSize = 0 - // DefaultWriteTimeoutUDS is the default value for the WriteTimeoutUDS option - DefaultWriteTimeoutUDS = 100 * time.Millisecond - // DefaultTelemetry is the default value for the Telemetry option - DefaultTelemetry = true - // DefaultReceivingMode is the default behavior when sending metrics - DefaultReceivingMode = MutexMode - // DefaultChannelModeBufferSize is the default size of the channel holding incoming metrics - DefaultChannelModeBufferSize = 4096 - // DefaultAggregationFlushInterval is the default interval for the aggregator to flush metrics. - // This should divide the Agent reporting period (default=10s) evenly to reduce "aliasing" that - // can cause values to appear irregular. - DefaultAggregationFlushInterval = 2 * time.Second - // DefaultAggregation - DefaultAggregation = false - // DefaultExtendedAggregation - DefaultExtendedAggregation = false - // DefaultDevMode - DefaultDevMode = false -) - -// Options contains the configuration options for a client. -type Options struct { - // Namespace to prepend to all metrics, events and service checks name. - Namespace string - // Tags are global tags to be applied to every metrics, events and service checks. - Tags []string - // MaxBytesPerPayload is the maximum number of bytes a single payload will contain. - // The magic value 0 will set the option to the optimal size for the transport - // protocol used when creating the client: 1432 for UDP and 8192 for UDS. - MaxBytesPerPayload int - // MaxMessagesPerPayload is the maximum number of metrics, events and/or service checks a single payload will contain. - // This option can be set to `1` to create an unbuffered client. - MaxMessagesPerPayload int - // BufferPoolSize is the size of the pool of buffers in number of buffers. - // The magic value 0 will set the option to the optimal size for the transport - // protocol used when creating the client: 2048 for UDP and 512 for UDS. - BufferPoolSize int - // BufferFlushInterval is the interval after which the current buffer will get flushed. - BufferFlushInterval time.Duration - // BufferShardCount is the number of buffer "shards" that will be used. - // Those shards allows the use of multiple buffers at the same time to reduce - // lock contention. - BufferShardCount int - // SenderQueueSize is the size of the sender queue in number of buffers. - // The magic value 0 will set the option to the optimal size for the transport - // protocol used when creating the client: 2048 for UDP and 512 for UDS. - SenderQueueSize int - // WriteTimeoutUDS is the timeout after which a UDS packet is dropped. - WriteTimeoutUDS time.Duration - // Telemetry is a set of metrics automatically injected by the client in the - // dogstatsd stream to be able to monitor the client itself. - Telemetry bool - // ReceiveMode determins the behavior of the client when receiving to many - // metrics. The client will either drop the metrics if its buffers are - // full (ChannelMode mode) or block the caller until the metric can be - // handled (MutexMode mode). By default the client will MutexMode. This - // option should be set to ChannelMode only when use under very high - // load. - // - // MutexMode uses a mutex internally which is much faster than - // channel but causes some lock contention when used with a high number - // of threads. Mutex are sharded based on the metrics name which - // limit mutex contention when goroutines send different metrics. - // - // ChannelMode: uses channel (of ChannelModeBufferSize size) to send - // metrics and drop metrics if the channel is full. Sending metrics in - // this mode is slower that MutexMode (because of the channel), but - // will not block the application. This mode is made for application - // using many goroutines, sending the same metrics at a very high - // volume. The goal is to not slow down the application at the cost of - // dropping metrics and having a lower max throughput. - ReceiveMode ReceivingMode - // ChannelModeBufferSize is the size of the channel holding incoming metrics - ChannelModeBufferSize int - // AggregationFlushInterval is the interval for the aggregator to flush metrics - AggregationFlushInterval time.Duration - // [beta] Aggregation enables/disables client side aggregation for - // Gauges, Counts and Sets (compatible with every Agent's version). - Aggregation bool - // [beta] Extended aggregation enables/disables client side aggregation - // for all types. This feature is only compatible with Agent's versions - // >=7.25.0 or Agent's version >=6.25.0 && < 7.0.0. - ExtendedAggregation bool - // TelemetryAddr specify a different endpoint for telemetry metrics. - TelemetryAddr string - // DevMode enables the "dev" mode where the client sends much more - // telemetry metrics to help troubleshooting the client behavior. - DevMode bool -} - -func resolveOptions(options []Option) (*Options, error) { - o := &Options{ - Namespace: DefaultNamespace, - Tags: DefaultTags, - MaxBytesPerPayload: DefaultMaxBytesPerPayload, - MaxMessagesPerPayload: DefaultMaxMessagesPerPayload, - BufferPoolSize: DefaultBufferPoolSize, - BufferFlushInterval: DefaultBufferFlushInterval, - BufferShardCount: DefaultBufferShardCount, - SenderQueueSize: DefaultSenderQueueSize, - WriteTimeoutUDS: DefaultWriteTimeoutUDS, - Telemetry: DefaultTelemetry, - ReceiveMode: DefaultReceivingMode, - ChannelModeBufferSize: DefaultChannelModeBufferSize, - AggregationFlushInterval: DefaultAggregationFlushInterval, - Aggregation: DefaultAggregation, - ExtendedAggregation: DefaultExtendedAggregation, - DevMode: DefaultDevMode, - } - - for _, option := range options { - err := option(o) - if err != nil { - return nil, err - } - } - - return o, nil -} - -// Option is a client option. Can return an error if validation fails. -type Option func(*Options) error - -// WithNamespace sets the Namespace option. -func WithNamespace(namespace string) Option { - return func(o *Options) error { - if strings.HasSuffix(namespace, ".") { - o.Namespace = namespace - } else { - o.Namespace = namespace + "." - } - return nil - } -} - -// WithTags sets the Tags option. -func WithTags(tags []string) Option { - return func(o *Options) error { - o.Tags = tags - return nil - } -} - -// WithMaxMessagesPerPayload sets the MaxMessagesPerPayload option. -func WithMaxMessagesPerPayload(maxMessagesPerPayload int) Option { - return func(o *Options) error { - o.MaxMessagesPerPayload = maxMessagesPerPayload - return nil - } -} - -// WithMaxBytesPerPayload sets the MaxBytesPerPayload option. -func WithMaxBytesPerPayload(MaxBytesPerPayload int) Option { - return func(o *Options) error { - o.MaxBytesPerPayload = MaxBytesPerPayload - return nil - } -} - -// WithBufferPoolSize sets the BufferPoolSize option. -func WithBufferPoolSize(bufferPoolSize int) Option { - return func(o *Options) error { - o.BufferPoolSize = bufferPoolSize - return nil - } -} - -// WithBufferFlushInterval sets the BufferFlushInterval option. -func WithBufferFlushInterval(bufferFlushInterval time.Duration) Option { - return func(o *Options) error { - o.BufferFlushInterval = bufferFlushInterval - return nil - } -} - -// WithBufferShardCount sets the BufferShardCount option. -func WithBufferShardCount(bufferShardCount int) Option { - return func(o *Options) error { - if bufferShardCount < 1 { - return fmt.Errorf("BufferShardCount must be a positive integer") - } - o.BufferShardCount = bufferShardCount - return nil - } -} - -// WithSenderQueueSize sets the SenderQueueSize option. -func WithSenderQueueSize(senderQueueSize int) Option { - return func(o *Options) error { - o.SenderQueueSize = senderQueueSize - return nil - } -} - -// WithWriteTimeoutUDS sets the WriteTimeoutUDS option. -func WithWriteTimeoutUDS(writeTimeoutUDS time.Duration) Option { - return func(o *Options) error { - o.WriteTimeoutUDS = writeTimeoutUDS - return nil - } -} - -// WithoutTelemetry disables the telemetry -func WithoutTelemetry() Option { - return func(o *Options) error { - o.Telemetry = false - return nil - } -} - -// WithChannelMode will use channel to receive metrics -func WithChannelMode() Option { - return func(o *Options) error { - o.ReceiveMode = ChannelMode - return nil - } -} - -// WithMutexMode will use mutex to receive metrics -func WithMutexMode() Option { - return func(o *Options) error { - o.ReceiveMode = MutexMode - return nil - } -} - -// WithChannelModeBufferSize the channel buffer size when using "drop mode" -func WithChannelModeBufferSize(bufferSize int) Option { - return func(o *Options) error { - o.ChannelModeBufferSize = bufferSize - return nil - } -} - -// WithAggregationInterval set the aggregation interval -func WithAggregationInterval(interval time.Duration) Option { - return func(o *Options) error { - o.AggregationFlushInterval = interval - return nil - } -} - -// WithClientSideAggregation enables client side aggregation for Gauges, Counts -// and Sets. Client side aggregation is a beta feature. -func WithClientSideAggregation() Option { - return func(o *Options) error { - o.Aggregation = true - return nil - } -} - -// WithoutClientSideAggregation disables client side aggregation. -func WithoutClientSideAggregation() Option { - return func(o *Options) error { - o.Aggregation = false - o.ExtendedAggregation = false - return nil - } -} - -// WithExtendedClientSideAggregation enables client side aggregation for all -// types. This feature is only compatible with Agent's version >=6.25.0 && -// <7.0.0 or Agent's versions >=7.25.0. Client side aggregation is a beta -// feature. -func WithExtendedClientSideAggregation() Option { - return func(o *Options) error { - o.Aggregation = true - o.ExtendedAggregation = true - return nil - } -} - -// WithTelemetryAddr specify a different address for telemetry metrics. -func WithTelemetryAddr(addr string) Option { - return func(o *Options) error { - o.TelemetryAddr = addr - return nil - } -} - -// WithDevMode enables client "dev" mode, sending more Telemetry metrics to -// help troubleshoot client behavior. -func WithDevMode() Option { - return func(o *Options) error { - o.DevMode = true - return nil - } -} - -// WithoutDevMode disables client "dev" mode, sending more Telemetry metrics to -// help troubleshoot client behavior. -func WithoutDevMode() Option { - return func(o *Options) error { - o.DevMode = false - return nil - } -} diff --git a/vendor/github.com/DataDog/datadog-go/statsd/pipe.go b/vendor/github.com/DataDog/datadog-go/statsd/pipe.go deleted file mode 100644 index 0d098a182..000000000 --- a/vendor/github.com/DataDog/datadog-go/statsd/pipe.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build !windows - -package statsd - -import "errors" - -func newWindowsPipeWriter(pipepath string) (statsdWriter, error) { - return nil, errors.New("Windows Named Pipes are only supported on Windows") -} diff --git a/vendor/github.com/DataDog/datadog-go/statsd/sender.go b/vendor/github.com/DataDog/datadog-go/statsd/sender.go deleted file mode 100644 index 4c8eb2696..000000000 --- a/vendor/github.com/DataDog/datadog-go/statsd/sender.go +++ /dev/null @@ -1,130 +0,0 @@ -package statsd - -import ( - "sync/atomic" - "time" -) - -// A statsdWriter offers a standard interface regardless of the underlying -// protocol. For now UDS and UPD writers are available. -// Attention: the underlying buffer of `data` is reused after a `statsdWriter.Write` call. -// `statsdWriter.Write` must be synchronous. -type statsdWriter interface { - Write(data []byte) (n int, err error) - SetWriteTimeout(time.Duration) error - Close() error -} - -// SenderMetrics contains metrics about the health of the sender -type SenderMetrics struct { - TotalSentBytes uint64 - TotalSentPayloads uint64 - TotalDroppedPayloads uint64 - TotalDroppedBytes uint64 - TotalDroppedPayloadsQueueFull uint64 - TotalDroppedBytesQueueFull uint64 - TotalDroppedPayloadsWriter uint64 - TotalDroppedBytesWriter uint64 -} - -type sender struct { - transport statsdWriter - pool *bufferPool - queue chan *statsdBuffer - metrics *SenderMetrics - stop chan struct{} - flushSignal chan struct{} -} - -func newSender(transport statsdWriter, queueSize int, pool *bufferPool) *sender { - sender := &sender{ - transport: transport, - pool: pool, - queue: make(chan *statsdBuffer, queueSize), - metrics: &SenderMetrics{}, - stop: make(chan struct{}), - flushSignal: make(chan struct{}), - } - - go sender.sendLoop() - return sender -} - -func (s *sender) send(buffer *statsdBuffer) { - select { - case s.queue <- buffer: - default: - atomic.AddUint64(&s.metrics.TotalDroppedPayloads, 1) - atomic.AddUint64(&s.metrics.TotalDroppedBytes, uint64(len(buffer.bytes()))) - atomic.AddUint64(&s.metrics.TotalDroppedPayloadsQueueFull, 1) - atomic.AddUint64(&s.metrics.TotalDroppedBytesQueueFull, uint64(len(buffer.bytes()))) - s.pool.returnBuffer(buffer) - } -} - -func (s *sender) write(buffer *statsdBuffer) { - _, err := s.transport.Write(buffer.bytes()) - if err != nil { - atomic.AddUint64(&s.metrics.TotalDroppedPayloads, 1) - atomic.AddUint64(&s.metrics.TotalDroppedBytes, uint64(len(buffer.bytes()))) - atomic.AddUint64(&s.metrics.TotalDroppedPayloadsWriter, 1) - atomic.AddUint64(&s.metrics.TotalDroppedBytesWriter, uint64(len(buffer.bytes()))) - } else { - atomic.AddUint64(&s.metrics.TotalSentPayloads, 1) - atomic.AddUint64(&s.metrics.TotalSentBytes, uint64(len(buffer.bytes()))) - } - s.pool.returnBuffer(buffer) -} - -func (s *sender) flushTelemetryMetrics() SenderMetrics { - return SenderMetrics{ - TotalSentBytes: atomic.SwapUint64(&s.metrics.TotalSentBytes, 0), - TotalSentPayloads: atomic.SwapUint64(&s.metrics.TotalSentPayloads, 0), - TotalDroppedPayloads: atomic.SwapUint64(&s.metrics.TotalDroppedPayloads, 0), - TotalDroppedBytes: atomic.SwapUint64(&s.metrics.TotalDroppedBytes, 0), - TotalDroppedPayloadsQueueFull: atomic.SwapUint64(&s.metrics.TotalDroppedPayloadsQueueFull, 0), - TotalDroppedBytesQueueFull: atomic.SwapUint64(&s.metrics.TotalDroppedBytesQueueFull, 0), - TotalDroppedPayloadsWriter: atomic.SwapUint64(&s.metrics.TotalDroppedPayloadsWriter, 0), - TotalDroppedBytesWriter: atomic.SwapUint64(&s.metrics.TotalDroppedBytesWriter, 0), - } -} - -func (s *sender) sendLoop() { - defer close(s.stop) - for { - select { - case buffer := <-s.queue: - s.write(buffer) - case <-s.stop: - return - case <-s.flushSignal: - // At that point we know that the workers are paused (the statsd client - // will pause them before calling sender.flush()). - // So we can fully flush the input queue - s.flushInputQueue() - s.flushSignal <- struct{}{} - } - } -} - -func (s *sender) flushInputQueue() { - for { - select { - case buffer := <-s.queue: - s.write(buffer) - default: - return - } - } -} -func (s *sender) flush() { - s.flushSignal <- struct{}{} - <-s.flushSignal -} - -func (s *sender) close() error { - s.stop <- struct{}{} - <-s.stop - s.flushInputQueue() - return s.transport.Close() -} diff --git a/vendor/github.com/DataDog/datadog-go/statsd/statsd.go b/vendor/github.com/DataDog/datadog-go/statsd/statsd.go deleted file mode 100644 index 19b5515a5..000000000 --- a/vendor/github.com/DataDog/datadog-go/statsd/statsd.go +++ /dev/null @@ -1,687 +0,0 @@ -// Copyright 2013 Ooyala, Inc. - -/* -Package statsd provides a Go dogstatsd client. Dogstatsd extends the popular statsd, -adding tags and histograms and pushing upstream to Datadog. - -Refer to http://docs.datadoghq.com/guides/dogstatsd/ for information about DogStatsD. - -statsd is based on go-statsd-client. -*/ -package statsd - -import ( - "errors" - "fmt" - "os" - "strings" - "sync" - "sync/atomic" - "time" -) - -/* -OptimalUDPPayloadSize defines the optimal payload size for a UDP datagram, 1432 bytes -is optimal for regular networks with an MTU of 1500 so datagrams don't get -fragmented. It's generally recommended not to fragment UDP datagrams as losing -a single fragment will cause the entire datagram to be lost. -*/ -const OptimalUDPPayloadSize = 1432 - -/* -MaxUDPPayloadSize defines the maximum payload size for a UDP datagram. -Its value comes from the calculation: 65535 bytes Max UDP datagram size - -8byte UDP header - 60byte max IP headers -any number greater than that will see frames being cut out. -*/ -const MaxUDPPayloadSize = 65467 - -// DefaultUDPBufferPoolSize is the default size of the buffer pool for UDP clients. -const DefaultUDPBufferPoolSize = 2048 - -// DefaultUDSBufferPoolSize is the default size of the buffer pool for UDS clients. -const DefaultUDSBufferPoolSize = 512 - -/* -DefaultMaxAgentPayloadSize is the default maximum payload size the agent -can receive. This can be adjusted by changing dogstatsd_buffer_size in the -agent configuration file datadog.yaml. This is also used as the optimal payload size -for UDS datagrams. -*/ -const DefaultMaxAgentPayloadSize = 8192 - -/* -UnixAddressPrefix holds the prefix to use to enable Unix Domain Socket -traffic instead of UDP. -*/ -const UnixAddressPrefix = "unix://" - -/* -WindowsPipeAddressPrefix holds the prefix to use to enable Windows Named Pipes -traffic instead of UDP. -*/ -const WindowsPipeAddressPrefix = `\\.\pipe\` - -const ( - agentHostEnvVarName = "DD_AGENT_HOST" - agentPortEnvVarName = "DD_DOGSTATSD_PORT" - defaultUDPPort = "8125" -) - -/* -ddEnvTagsMapping is a mapping of each "DD_" prefixed environment variable -to a specific tag name. We use a slice to keep the order and simplify tests. -*/ -var ddEnvTagsMapping = []struct{ envName, tagName string }{ - {"DD_ENTITY_ID", "dd.internal.entity_id"}, // Client-side entity ID injection for container tagging. - {"DD_ENV", "env"}, // The name of the env in which the service runs. - {"DD_SERVICE", "service"}, // The name of the running service. - {"DD_VERSION", "version"}, // The current version of the running service. -} - -type metricType int - -const ( - gauge metricType = iota - count - histogram - histogramAggregated - distribution - distributionAggregated - set - timing - timingAggregated - event - serviceCheck -) - -type ReceivingMode int - -const ( - MutexMode ReceivingMode = iota - ChannelMode -) - -const ( - WriterNameUDP string = "udp" - WriterNameUDS string = "uds" - WriterWindowsPipe string = "pipe" -) - -type metric struct { - metricType metricType - namespace string - globalTags []string - name string - fvalue float64 - fvalues []float64 - ivalue int64 - svalue string - evalue *Event - scvalue *ServiceCheck - tags []string - stags string - rate float64 -} - -type noClientErr string - -// ErrNoClient is returned if statsd reporting methods are invoked on -// a nil client. -const ErrNoClient = noClientErr("statsd client is nil") - -func (e noClientErr) Error() string { - return string(e) -} - -// ClientInterface is an interface that exposes the common client functions for the -// purpose of being able to provide a no-op client or even mocking. This can aid -// downstream users' with their testing. -type ClientInterface interface { - // Gauge measures the value of a metric at a particular time. - Gauge(name string, value float64, tags []string, rate float64) error - - // Count tracks how many times something happened per second. - Count(name string, value int64, tags []string, rate float64) error - - // Histogram tracks the statistical distribution of a set of values on each host. - Histogram(name string, value float64, tags []string, rate float64) error - - // Distribution tracks the statistical distribution of a set of values across your infrastructure. - Distribution(name string, value float64, tags []string, rate float64) error - - // Decr is just Count of -1 - Decr(name string, tags []string, rate float64) error - - // Incr is just Count of 1 - Incr(name string, tags []string, rate float64) error - - // Set counts the number of unique elements in a group. - Set(name string, value string, tags []string, rate float64) error - - // Timing sends timing information, it is an alias for TimeInMilliseconds - Timing(name string, value time.Duration, tags []string, rate float64) error - - // TimeInMilliseconds sends timing information in milliseconds. - // It is flushed by statsd with percentiles, mean and other info (https://github.com/etsy/statsd/blob/master/docs/metric_types.md#timing) - TimeInMilliseconds(name string, value float64, tags []string, rate float64) error - - // Event sends the provided Event. - Event(e *Event) error - - // SimpleEvent sends an event with the provided title and text. - SimpleEvent(title, text string) error - - // ServiceCheck sends the provided ServiceCheck. - ServiceCheck(sc *ServiceCheck) error - - // SimpleServiceCheck sends an serviceCheck with the provided name and status. - SimpleServiceCheck(name string, status ServiceCheckStatus) error - - // Close the client connection. - Close() error - - // Flush forces a flush of all the queued dogstatsd payloads. - Flush() error - - // SetWriteTimeout allows the user to set a custom write timeout. - SetWriteTimeout(d time.Duration) error -} - -// A Client is a handle for sending messages to dogstatsd. It is safe to -// use one Client from multiple goroutines simultaneously. -type Client struct { - // Sender handles the underlying networking protocol - sender *sender - // Namespace to prepend to all statsd calls - Namespace string - // Tags are global tags to be added to every statsd call - Tags []string - // skipErrors turns off error passing and allows UDS to emulate UDP behaviour - SkipErrors bool - flushTime time.Duration - metrics *ClientMetrics - telemetry *telemetryClient - stop chan struct{} - wg sync.WaitGroup - workers []*worker - closerLock sync.Mutex - workersMode ReceivingMode - aggregatorMode ReceivingMode - agg *aggregator - aggExtended *aggregator - options []Option - addrOption string -} - -// ClientMetrics contains metrics about the client -type ClientMetrics struct { - TotalMetrics uint64 - TotalMetricsGauge uint64 - TotalMetricsCount uint64 - TotalMetricsHistogram uint64 - TotalMetricsDistribution uint64 - TotalMetricsSet uint64 - TotalMetricsTiming uint64 - TotalEvents uint64 - TotalServiceChecks uint64 - TotalDroppedOnReceive uint64 -} - -// Verify that Client implements the ClientInterface. -// https://golang.org/doc/faq#guarantee_satisfies_interface -var _ ClientInterface = &Client{} - -func resolveAddr(addr string) string { - envPort := "" - if addr == "" { - addr = os.Getenv(agentHostEnvVarName) - envPort = os.Getenv(agentPortEnvVarName) - } - - if addr == "" { - return "" - } - - if !strings.HasPrefix(addr, WindowsPipeAddressPrefix) && !strings.HasPrefix(addr, UnixAddressPrefix) { - if !strings.Contains(addr, ":") { - if envPort != "" { - addr = fmt.Sprintf("%s:%s", addr, envPort) - } else { - addr = fmt.Sprintf("%s:%s", addr, defaultUDPPort) - } - } - } - return addr -} - -func createWriter(addr string) (statsdWriter, string, error) { - addr = resolveAddr(addr) - if addr == "" { - return nil, "", errors.New("No address passed and autodetection from environment failed") - } - - switch { - case strings.HasPrefix(addr, WindowsPipeAddressPrefix): - w, err := newWindowsPipeWriter(addr) - return w, WriterWindowsPipe, err - case strings.HasPrefix(addr, UnixAddressPrefix): - w, err := newUDSWriter(addr[len(UnixAddressPrefix):]) - return w, WriterNameUDS, err - default: - w, err := newUDPWriter(addr) - return w, WriterNameUDP, err - } -} - -// New returns a pointer to a new Client given an addr in the format "hostname:port" for UDP, -// "unix:///path/to/socket" for UDS or "\\.\pipe\path\to\pipe" for Windows Named Pipes. -func New(addr string, options ...Option) (*Client, error) { - o, err := resolveOptions(options) - if err != nil { - return nil, err - } - - w, writerType, err := createWriter(addr) - if err != nil { - return nil, err - } - - client, err := newWithWriter(w, o, writerType) - if err == nil { - client.options = append(client.options, options...) - client.addrOption = addr - } - return client, err -} - -// NewWithWriter creates a new Client with given writer. Writer is a -// io.WriteCloser + SetWriteTimeout(time.Duration) error -func NewWithWriter(w statsdWriter, options ...Option) (*Client, error) { - o, err := resolveOptions(options) - if err != nil { - return nil, err - } - return newWithWriter(w, o, "custom") -} - -// CloneWithExtraOptions create a new Client with extra options -func CloneWithExtraOptions(c *Client, options ...Option) (*Client, error) { - if c == nil { - return nil, ErrNoClient - } - - if c.addrOption == "" { - return nil, fmt.Errorf("can't clone client with no addrOption") - } - opt := append(c.options, options...) - return New(c.addrOption, opt...) -} - -func newWithWriter(w statsdWriter, o *Options, writerName string) (*Client, error) { - - w.SetWriteTimeout(o.WriteTimeoutUDS) - - c := Client{ - Namespace: o.Namespace, - Tags: o.Tags, - metrics: &ClientMetrics{}, - } - // Inject values of DD_* environment variables as global tags. - for _, mapping := range ddEnvTagsMapping { - if value := os.Getenv(mapping.envName); value != "" { - c.Tags = append(c.Tags, fmt.Sprintf("%s:%s", mapping.tagName, value)) - } - } - - if o.MaxBytesPerPayload == 0 { - if writerName == WriterNameUDS { - o.MaxBytesPerPayload = DefaultMaxAgentPayloadSize - } else { - o.MaxBytesPerPayload = OptimalUDPPayloadSize - } - } - if o.BufferPoolSize == 0 { - if writerName == WriterNameUDS { - o.BufferPoolSize = DefaultUDSBufferPoolSize - } else { - o.BufferPoolSize = DefaultUDPBufferPoolSize - } - } - if o.SenderQueueSize == 0 { - if writerName == WriterNameUDS { - o.SenderQueueSize = DefaultUDSBufferPoolSize - } else { - o.SenderQueueSize = DefaultUDPBufferPoolSize - } - } - - bufferPool := newBufferPool(o.BufferPoolSize, o.MaxBytesPerPayload, o.MaxMessagesPerPayload) - c.sender = newSender(w, o.SenderQueueSize, bufferPool) - c.aggregatorMode = o.ReceiveMode - - c.workersMode = o.ReceiveMode - // ChannelMode mode at the worker level is not enabled when - // ExtendedAggregation is since the user app will not directly - // use the worker (the aggregator sit between the app and the - // workers). - if o.ExtendedAggregation { - c.workersMode = MutexMode - } - - if o.Aggregation || o.ExtendedAggregation { - c.agg = newAggregator(&c) - c.agg.start(o.AggregationFlushInterval) - - if o.ExtendedAggregation { - c.aggExtended = c.agg - - if c.aggregatorMode == ChannelMode { - c.agg.startReceivingMetric(o.ChannelModeBufferSize, o.BufferShardCount) - } - } - } - - for i := 0; i < o.BufferShardCount; i++ { - w := newWorker(bufferPool, c.sender) - c.workers = append(c.workers, w) - - if c.workersMode == ChannelMode { - w.startReceivingMetric(o.ChannelModeBufferSize) - } - } - - c.flushTime = o.BufferFlushInterval - c.stop = make(chan struct{}, 1) - - c.wg.Add(1) - go func() { - defer c.wg.Done() - c.watch() - }() - - if o.Telemetry { - if o.TelemetryAddr == "" { - c.telemetry = newTelemetryClient(&c, writerName, o.DevMode) - } else { - var err error - c.telemetry, err = newTelemetryClientWithCustomAddr(&c, writerName, o.DevMode, o.TelemetryAddr, bufferPool) - if err != nil { - return nil, err - } - } - c.telemetry.run(&c.wg, c.stop) - } - - return &c, nil -} - -// NewBuffered returns a Client that buffers its output and sends it in chunks. -// Buflen is the length of the buffer in number of commands. -// -// When addr is empty, the client will default to a UDP client and use the DD_AGENT_HOST -// and (optionally) the DD_DOGSTATSD_PORT environment variables to build the target address. -func NewBuffered(addr string, buflen int) (*Client, error) { - return New(addr, WithMaxMessagesPerPayload(buflen)) -} - -// SetWriteTimeout allows the user to set a custom UDS write timeout. Not supported for UDP -// or Windows Pipes. -func (c *Client) SetWriteTimeout(d time.Duration) error { - if c == nil { - return ErrNoClient - } - return c.sender.transport.SetWriteTimeout(d) -} - -func (c *Client) watch() { - ticker := time.NewTicker(c.flushTime) - - for { - select { - case <-ticker.C: - for _, w := range c.workers { - w.flush() - } - case <-c.stop: - ticker.Stop() - return - } - } -} - -// Flush forces a flush of all the queued dogstatsd payloads This method is -// blocking and will not return until everything is sent through the network. -// In MutexMode, this will also block sampling new data to the client while the -// workers and sender are flushed. -func (c *Client) Flush() error { - if c == nil { - return ErrNoClient - } - if c.agg != nil { - c.agg.flush() - } - for _, w := range c.workers { - w.pause() - defer w.unpause() - w.flushUnsafe() - } - // Now that the worker are pause the sender can flush the queue between - // worker and senders - c.sender.flush() - return nil -} - -func (c *Client) FlushTelemetryMetrics() ClientMetrics { - cm := ClientMetrics{ - TotalMetricsGauge: atomic.SwapUint64(&c.metrics.TotalMetricsGauge, 0), - TotalMetricsCount: atomic.SwapUint64(&c.metrics.TotalMetricsCount, 0), - TotalMetricsSet: atomic.SwapUint64(&c.metrics.TotalMetricsSet, 0), - TotalMetricsHistogram: atomic.SwapUint64(&c.metrics.TotalMetricsHistogram, 0), - TotalMetricsDistribution: atomic.SwapUint64(&c.metrics.TotalMetricsDistribution, 0), - TotalMetricsTiming: atomic.SwapUint64(&c.metrics.TotalMetricsTiming, 0), - TotalEvents: atomic.SwapUint64(&c.metrics.TotalEvents, 0), - TotalServiceChecks: atomic.SwapUint64(&c.metrics.TotalServiceChecks, 0), - TotalDroppedOnReceive: atomic.SwapUint64(&c.metrics.TotalDroppedOnReceive, 0), - } - - cm.TotalMetrics = cm.TotalMetricsGauge + cm.TotalMetricsCount + - cm.TotalMetricsSet + cm.TotalMetricsHistogram + - cm.TotalMetricsDistribution + cm.TotalMetricsTiming - - return cm -} - -func (c *Client) send(m metric) error { - h := hashString32(m.name) - worker := c.workers[h%uint32(len(c.workers))] - - if c.workersMode == ChannelMode { - select { - case worker.inputMetrics <- m: - default: - atomic.AddUint64(&c.metrics.TotalDroppedOnReceive, 1) - } - return nil - } - return worker.processMetric(m) -} - -// sendBlocking is used by the aggregator to inject aggregated metrics. -func (c *Client) sendBlocking(m metric) error { - m.globalTags = c.Tags - m.namespace = c.Namespace - - h := hashString32(m.name) - worker := c.workers[h%uint32(len(c.workers))] - return worker.processMetric(m) -} - -func (c *Client) sendToAggregator(mType metricType, name string, value float64, tags []string, rate float64, f bufferedMetricSampleFunc) error { - if c.aggregatorMode == ChannelMode { - select { - case c.aggExtended.inputMetrics <- metric{metricType: mType, name: name, fvalue: value, tags: tags, rate: rate}: - default: - atomic.AddUint64(&c.metrics.TotalDroppedOnReceive, 1) - } - return nil - } - return f(name, value, tags, rate) -} - -// Gauge measures the value of a metric at a particular time. -func (c *Client) Gauge(name string, value float64, tags []string, rate float64) error { - if c == nil { - return ErrNoClient - } - atomic.AddUint64(&c.metrics.TotalMetricsGauge, 1) - if c.agg != nil { - return c.agg.gauge(name, value, tags) - } - return c.send(metric{metricType: gauge, name: name, fvalue: value, tags: tags, rate: rate, globalTags: c.Tags, namespace: c.Namespace}) -} - -// Count tracks how many times something happened per second. -func (c *Client) Count(name string, value int64, tags []string, rate float64) error { - if c == nil { - return ErrNoClient - } - atomic.AddUint64(&c.metrics.TotalMetricsCount, 1) - if c.agg != nil { - return c.agg.count(name, value, tags) - } - return c.send(metric{metricType: count, name: name, ivalue: value, tags: tags, rate: rate, globalTags: c.Tags, namespace: c.Namespace}) -} - -// Histogram tracks the statistical distribution of a set of values on each host. -func (c *Client) Histogram(name string, value float64, tags []string, rate float64) error { - if c == nil { - return ErrNoClient - } - atomic.AddUint64(&c.metrics.TotalMetricsHistogram, 1) - if c.aggExtended != nil { - return c.sendToAggregator(histogram, name, value, tags, rate, c.aggExtended.histogram) - } - return c.send(metric{metricType: histogram, name: name, fvalue: value, tags: tags, rate: rate, globalTags: c.Tags, namespace: c.Namespace}) -} - -// Distribution tracks the statistical distribution of a set of values across your infrastructure. -func (c *Client) Distribution(name string, value float64, tags []string, rate float64) error { - if c == nil { - return ErrNoClient - } - atomic.AddUint64(&c.metrics.TotalMetricsDistribution, 1) - if c.aggExtended != nil { - return c.sendToAggregator(distribution, name, value, tags, rate, c.aggExtended.distribution) - } - return c.send(metric{metricType: distribution, name: name, fvalue: value, tags: tags, rate: rate, globalTags: c.Tags, namespace: c.Namespace}) -} - -// Decr is just Count of -1 -func (c *Client) Decr(name string, tags []string, rate float64) error { - return c.Count(name, -1, tags, rate) -} - -// Incr is just Count of 1 -func (c *Client) Incr(name string, tags []string, rate float64) error { - return c.Count(name, 1, tags, rate) -} - -// Set counts the number of unique elements in a group. -func (c *Client) Set(name string, value string, tags []string, rate float64) error { - if c == nil { - return ErrNoClient - } - atomic.AddUint64(&c.metrics.TotalMetricsSet, 1) - if c.agg != nil { - return c.agg.set(name, value, tags) - } - return c.send(metric{metricType: set, name: name, svalue: value, tags: tags, rate: rate, globalTags: c.Tags, namespace: c.Namespace}) -} - -// Timing sends timing information, it is an alias for TimeInMilliseconds -func (c *Client) Timing(name string, value time.Duration, tags []string, rate float64) error { - return c.TimeInMilliseconds(name, value.Seconds()*1000, tags, rate) -} - -// TimeInMilliseconds sends timing information in milliseconds. -// It is flushed by statsd with percentiles, mean and other info (https://github.com/etsy/statsd/blob/master/docs/metric_types.md#timing) -func (c *Client) TimeInMilliseconds(name string, value float64, tags []string, rate float64) error { - if c == nil { - return ErrNoClient - } - atomic.AddUint64(&c.metrics.TotalMetricsTiming, 1) - if c.aggExtended != nil { - return c.sendToAggregator(timing, name, value, tags, rate, c.aggExtended.timing) - } - return c.send(metric{metricType: timing, name: name, fvalue: value, tags: tags, rate: rate, globalTags: c.Tags, namespace: c.Namespace}) -} - -// Event sends the provided Event. -func (c *Client) Event(e *Event) error { - if c == nil { - return ErrNoClient - } - atomic.AddUint64(&c.metrics.TotalEvents, 1) - return c.send(metric{metricType: event, evalue: e, rate: 1, globalTags: c.Tags, namespace: c.Namespace}) -} - -// SimpleEvent sends an event with the provided title and text. -func (c *Client) SimpleEvent(title, text string) error { - e := NewEvent(title, text) - return c.Event(e) -} - -// ServiceCheck sends the provided ServiceCheck. -func (c *Client) ServiceCheck(sc *ServiceCheck) error { - if c == nil { - return ErrNoClient - } - atomic.AddUint64(&c.metrics.TotalServiceChecks, 1) - return c.send(metric{metricType: serviceCheck, scvalue: sc, rate: 1, globalTags: c.Tags, namespace: c.Namespace}) -} - -// SimpleServiceCheck sends an serviceCheck with the provided name and status. -func (c *Client) SimpleServiceCheck(name string, status ServiceCheckStatus) error { - sc := NewServiceCheck(name, status) - return c.ServiceCheck(sc) -} - -// Close the client connection. -func (c *Client) Close() error { - if c == nil { - return ErrNoClient - } - - // Acquire closer lock to ensure only one thread can close the stop channel - c.closerLock.Lock() - defer c.closerLock.Unlock() - - // Notify all other threads that they should stop - select { - case <-c.stop: - return nil - default: - } - close(c.stop) - - if c.workersMode == ChannelMode { - for _, w := range c.workers { - w.stopReceivingMetric() - } - } - - // flush the aggregator first - if c.agg != nil { - if c.aggExtended != nil && c.aggregatorMode == ChannelMode { - c.agg.stopReceivingMetric() - } - c.agg.stop() - } - - // Wait for the threads to stop - c.wg.Wait() - - c.Flush() - return c.sender.close() -} diff --git a/vendor/github.com/DataDog/datadog-go/statsd/telemetry.go b/vendor/github.com/DataDog/datadog-go/statsd/telemetry.go deleted file mode 100644 index eef5c2731..000000000 --- a/vendor/github.com/DataDog/datadog-go/statsd/telemetry.go +++ /dev/null @@ -1,151 +0,0 @@ -package statsd - -import ( - "fmt" - "sync" - "time" -) - -/* -TelemetryInterval is the interval at which telemetry will be sent by the client. -*/ -const TelemetryInterval = 10 * time.Second - -/* -clientTelemetryTag is a tag identifying this specific client. -*/ -var clientTelemetryTag = "client:go" - -/* -clientVersionTelemetryTag is a tag identifying this specific client version. -*/ -var clientVersionTelemetryTag = "client_version:4.8.3" - -type telemetryClient struct { - c *Client - tags []string - tagsByType map[metricType][]string - sender *sender - worker *worker - devMode bool -} - -func newTelemetryClient(c *Client, transport string, devMode bool) *telemetryClient { - t := &telemetryClient{ - c: c, - tags: append(c.Tags, clientTelemetryTag, clientVersionTelemetryTag, "client_transport:"+transport), - tagsByType: map[metricType][]string{}, - devMode: devMode, - } - - if devMode { - t.tagsByType[gauge] = append(append([]string{}, t.tags...), "metrics_type:gauge") - t.tagsByType[count] = append(append([]string{}, t.tags...), "metrics_type:count") - t.tagsByType[set] = append(append([]string{}, t.tags...), "metrics_type:set") - t.tagsByType[timing] = append(append([]string{}, t.tags...), "metrics_type:timing") - t.tagsByType[histogram] = append(append([]string{}, t.tags...), "metrics_type:histogram") - t.tagsByType[distribution] = append(append([]string{}, t.tags...), "metrics_type:distribution") - t.tagsByType[timing] = append(append([]string{}, t.tags...), "metrics_type:timing") - } - return t -} - -func newTelemetryClientWithCustomAddr(c *Client, transport string, devMode bool, telemetryAddr string, pool *bufferPool) (*telemetryClient, error) { - telemetryWriter, _, err := createWriter(telemetryAddr) - if err != nil { - return nil, fmt.Errorf("Could not resolve telemetry address: %v", err) - } - - t := newTelemetryClient(c, transport, devMode) - - // Creating a custom sender/worker with 1 worker in mutex mode for the - // telemetry that share the same bufferPool. - // FIXME due to performance pitfall, we're always using UDP defaults - // even for UDS. - t.sender = newSender(telemetryWriter, DefaultUDPBufferPoolSize, pool) - t.worker = newWorker(pool, t.sender) - return t, nil -} - -func (t *telemetryClient) run(wg *sync.WaitGroup, stop chan struct{}) { - wg.Add(1) - go func() { - defer wg.Done() - ticker := time.NewTicker(TelemetryInterval) - for { - select { - case <-ticker.C: - t.sendTelemetry() - case <-stop: - ticker.Stop() - if t.sender != nil { - t.sender.close() - } - return - } - } - }() -} - -func (t *telemetryClient) sendTelemetry() { - for _, m := range t.flush() { - if t.worker != nil { - t.worker.processMetric(m) - } else { - t.c.send(m) - } - } - - if t.worker != nil { - t.worker.flush() - } -} - -// flushTelemetry returns Telemetry metrics to be flushed. It's its own function to ease testing. -func (t *telemetryClient) flush() []metric { - m := []metric{} - - // same as Count but without global namespace - telemetryCount := func(name string, value int64, tags []string) { - m = append(m, metric{metricType: count, name: name, ivalue: value, tags: tags, rate: 1}) - } - - clientMetrics := t.c.FlushTelemetryMetrics() - telemetryCount("datadog.dogstatsd.client.metrics", int64(clientMetrics.TotalMetrics), t.tags) - if t.devMode { - telemetryCount("datadog.dogstatsd.client.metrics_by_type", int64(clientMetrics.TotalMetricsGauge), t.tagsByType[gauge]) - telemetryCount("datadog.dogstatsd.client.metrics_by_type", int64(clientMetrics.TotalMetricsCount), t.tagsByType[count]) - telemetryCount("datadog.dogstatsd.client.metrics_by_type", int64(clientMetrics.TotalMetricsHistogram), t.tagsByType[histogram]) - telemetryCount("datadog.dogstatsd.client.metrics_by_type", int64(clientMetrics.TotalMetricsDistribution), t.tagsByType[distribution]) - telemetryCount("datadog.dogstatsd.client.metrics_by_type", int64(clientMetrics.TotalMetricsSet), t.tagsByType[set]) - telemetryCount("datadog.dogstatsd.client.metrics_by_type", int64(clientMetrics.TotalMetricsTiming), t.tagsByType[timing]) - } - - telemetryCount("datadog.dogstatsd.client.events", int64(clientMetrics.TotalEvents), t.tags) - telemetryCount("datadog.dogstatsd.client.service_checks", int64(clientMetrics.TotalServiceChecks), t.tags) - telemetryCount("datadog.dogstatsd.client.metric_dropped_on_receive", int64(clientMetrics.TotalDroppedOnReceive), t.tags) - - senderMetrics := t.c.sender.flushTelemetryMetrics() - telemetryCount("datadog.dogstatsd.client.packets_sent", int64(senderMetrics.TotalSentPayloads), t.tags) - telemetryCount("datadog.dogstatsd.client.bytes_sent", int64(senderMetrics.TotalSentBytes), t.tags) - telemetryCount("datadog.dogstatsd.client.packets_dropped", int64(senderMetrics.TotalDroppedPayloads), t.tags) - telemetryCount("datadog.dogstatsd.client.bytes_dropped", int64(senderMetrics.TotalDroppedBytes), t.tags) - telemetryCount("datadog.dogstatsd.client.packets_dropped_queue", int64(senderMetrics.TotalDroppedPayloadsQueueFull), t.tags) - telemetryCount("datadog.dogstatsd.client.bytes_dropped_queue", int64(senderMetrics.TotalDroppedBytesQueueFull), t.tags) - telemetryCount("datadog.dogstatsd.client.packets_dropped_writer", int64(senderMetrics.TotalDroppedPayloadsWriter), t.tags) - telemetryCount("datadog.dogstatsd.client.bytes_dropped_writer", int64(senderMetrics.TotalDroppedBytesWriter), t.tags) - - if aggMetrics := t.c.agg.flushTelemetryMetrics(); aggMetrics != nil { - telemetryCount("datadog.dogstatsd.client.aggregated_context", int64(aggMetrics.nbContext), t.tags) - if t.devMode { - telemetryCount("datadog.dogstatsd.client.aggregated_context_by_type", int64(aggMetrics.nbContextGauge), t.tagsByType[gauge]) - telemetryCount("datadog.dogstatsd.client.aggregated_context_by_type", int64(aggMetrics.nbContextSet), t.tagsByType[set]) - telemetryCount("datadog.dogstatsd.client.aggregated_context_by_type", int64(aggMetrics.nbContextCount), t.tagsByType[count]) - telemetryCount("datadog.dogstatsd.client.aggregated_context_by_type", int64(aggMetrics.nbContextHistogram), t.tagsByType[histogram]) - telemetryCount("datadog.dogstatsd.client.aggregated_context_by_type", int64(aggMetrics.nbContextDistribution), t.tagsByType[distribution]) - telemetryCount("datadog.dogstatsd.client.aggregated_context_by_type", int64(aggMetrics.nbContextTiming), t.tagsByType[timing]) - } - } - - return m -} diff --git a/vendor/github.com/DataDog/datadog-go/statsd/uds.go b/vendor/github.com/DataDog/datadog-go/statsd/uds.go deleted file mode 100644 index 6c52261bd..000000000 --- a/vendor/github.com/DataDog/datadog-go/statsd/uds.go +++ /dev/null @@ -1,100 +0,0 @@ -// +build !windows - -package statsd - -import ( - "net" - "sync" - "time" -) - -/* -UDSTimeout holds the default timeout for UDS socket writes, as they can get -blocking when the receiving buffer is full. -*/ -const defaultUDSTimeout = 100 * time.Millisecond - -// udsWriter is an internal class wrapping around management of UDS connection -type udsWriter struct { - // Address to send metrics to, needed to allow reconnection on error - addr net.Addr - // Established connection object, or nil if not connected yet - conn net.Conn - // write timeout - writeTimeout time.Duration - sync.RWMutex // used to lock conn / writer can replace it -} - -// newUDSWriter returns a pointer to a new udsWriter given a socket file path as addr. -func newUDSWriter(addr string) (*udsWriter, error) { - udsAddr, err := net.ResolveUnixAddr("unixgram", addr) - if err != nil { - return nil, err - } - // Defer connection to first Write - writer := &udsWriter{addr: udsAddr, conn: nil, writeTimeout: defaultUDSTimeout} - return writer, nil -} - -// SetWriteTimeout allows the user to set a custom write timeout -func (w *udsWriter) SetWriteTimeout(d time.Duration) error { - w.writeTimeout = d - return nil -} - -// Write data to the UDS connection with write timeout and minimal error handling: -// create the connection if nil, and destroy it if the statsd server has disconnected -func (w *udsWriter) Write(data []byte) (int, error) { - conn, err := w.ensureConnection() - if err != nil { - return 0, err - } - - conn.SetWriteDeadline(time.Now().Add(w.writeTimeout)) - n, e := conn.Write(data) - - if err, isNetworkErr := e.(net.Error); err != nil && (!isNetworkErr || !err.Temporary()) { - // Statsd server disconnected, retry connecting at next packet - w.unsetConnection() - return 0, e - } - return n, e -} - -func (w *udsWriter) Close() error { - if w.conn != nil { - return w.conn.Close() - } - return nil -} - -func (w *udsWriter) ensureConnection() (net.Conn, error) { - // Check if we've already got a socket we can use - w.RLock() - currentConn := w.conn - w.RUnlock() - - if currentConn != nil { - return currentConn, nil - } - - // Looks like we might need to connect - try again with write locking. - w.Lock() - defer w.Unlock() - if w.conn != nil { - return w.conn, nil - } - - newConn, err := net.Dial(w.addr.Network(), w.addr.String()) - if err != nil { - return nil, err - } - w.conn = newConn - return newConn, nil -} - -func (w *udsWriter) unsetConnection() { - w.Lock() - defer w.Unlock() - w.conn = nil -} diff --git a/vendor/github.com/DataDog/datadog-go/statsd/uds_windows.go b/vendor/github.com/DataDog/datadog-go/statsd/uds_windows.go deleted file mode 100644 index 9c97dfd4e..000000000 --- a/vendor/github.com/DataDog/datadog-go/statsd/uds_windows.go +++ /dev/null @@ -1,10 +0,0 @@ -// +build windows - -package statsd - -import "fmt" - -// newUDSWriter is disable on windows as unix sockets are not available -func newUDSWriter(addr string) (statsdWriter, error) { - return nil, fmt.Errorf("unix socket is not available on windows") -} diff --git a/vendor/github.com/DataDog/datadog-go/LICENSE.txt b/vendor/github.com/DataDog/datadog-go/v5/LICENSE.txt similarity index 100% rename from vendor/github.com/DataDog/datadog-go/LICENSE.txt rename to vendor/github.com/DataDog/datadog-go/v5/LICENSE.txt diff --git a/vendor/github.com/DataDog/datadog-go/statsd/README.md b/vendor/github.com/DataDog/datadog-go/v5/statsd/README.md similarity index 100% rename from vendor/github.com/DataDog/datadog-go/statsd/README.md rename to vendor/github.com/DataDog/datadog-go/v5/statsd/README.md diff --git a/vendor/github.com/DataDog/datadog-go/statsd/aggregator.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/aggregator.go similarity index 72% rename from vendor/github.com/DataDog/datadog-go/statsd/aggregator.go rename to vendor/github.com/DataDog/datadog-go/v5/statsd/aggregator.go index c4446a23b..33eb930ae 100644 --- a/vendor/github.com/DataDog/datadog-go/statsd/aggregator.go +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/aggregator.go @@ -15,9 +15,9 @@ type ( ) type aggregator struct { - nbContextGauge int32 - nbContextCount int32 - nbContextSet int32 + nbContextGauge uint64 + nbContextCount uint64 + nbContextSet uint64 countsM sync.RWMutex gaugesM sync.RWMutex @@ -34,34 +34,24 @@ type aggregator struct { client *Client - // aggregator implements ChannelMode mechanism to receive histograms, + // aggregator implements channelMode mechanism to receive histograms, // distributions and timings. Since they need sampling they need to - // lock for random. When using both ChannelMode and ExtendedAggregation + // lock for random. When using both channelMode and ExtendedAggregation // we don't want goroutine to fight over the lock. inputMetrics chan metric stopChannelMode chan struct{} wg sync.WaitGroup } -type aggregatorMetrics struct { - nbContext int32 - nbContextGauge int32 - nbContextCount int32 - nbContextSet int32 - nbContextHistogram int32 - nbContextDistribution int32 - nbContextTiming int32 -} - -func newAggregator(c *Client) *aggregator { +func newAggregator(c *Client, maxSamplesPerContext int64) *aggregator { return &aggregator{ client: c, counts: countsMap{}, gauges: gaugesMap{}, sets: setsMap{}, - histograms: newBufferedContexts(newHistogramMetric), - distributions: newBufferedContexts(newDistributionMetric), - timings: newBufferedContexts(newTimingMetric), + histograms: newBufferedContexts(newHistogramMetric, maxSamplesPerContext), + distributions: newBufferedContexts(newDistributionMetric, maxSamplesPerContext), + timings: newBufferedContexts(newTimingMetric, maxSamplesPerContext), closed: make(chan struct{}), stopChannelMode: make(chan struct{}), } @@ -76,6 +66,7 @@ func (a *aggregator) start(flushInterval time.Duration) { case <-ticker.C: a.flush() case <-a.closed: + ticker.Stop() return } } @@ -124,22 +115,18 @@ func (a *aggregator) flush() { } } -func (a *aggregator) flushTelemetryMetrics() *aggregatorMetrics { +func (a *aggregator) flushTelemetryMetrics(t *Telemetry) { if a == nil { - return nil + // aggregation is disabled + return } - am := &aggregatorMetrics{ - nbContextGauge: atomic.SwapInt32(&a.nbContextGauge, 0), - nbContextCount: atomic.SwapInt32(&a.nbContextCount, 0), - nbContextSet: atomic.SwapInt32(&a.nbContextSet, 0), - nbContextHistogram: a.histograms.resetAndGetNbContext(), - nbContextDistribution: a.distributions.resetAndGetNbContext(), - nbContextTiming: a.timings.resetAndGetNbContext(), - } - - am.nbContext = am.nbContextGauge + am.nbContextCount + am.nbContextSet + am.nbContextHistogram + am.nbContextDistribution + am.nbContextTiming - return am + t.AggregationNbContextGauge = atomic.LoadUint64(&a.nbContextGauge) + t.AggregationNbContextCount = atomic.LoadUint64(&a.nbContextCount) + t.AggregationNbContextSet = atomic.LoadUint64(&a.nbContextSet) + t.AggregationNbContextHistogram = a.histograms.getNbContext() + t.AggregationNbContextDistribution = a.distributions.getNbContext() + t.AggregationNbContextTiming = a.timings.getNbContext() } func (a *aggregator) flushMetrics() []metric { @@ -179,19 +166,47 @@ func (a *aggregator) flushMetrics() []metric { metrics = a.distributions.flush(metrics) metrics = a.timings.flush(metrics) - atomic.AddInt32(&a.nbContextCount, int32(len(counts))) - atomic.AddInt32(&a.nbContextGauge, int32(len(gauges))) - atomic.AddInt32(&a.nbContextSet, int32(len(sets))) + atomic.AddUint64(&a.nbContextCount, uint64(len(counts))) + atomic.AddUint64(&a.nbContextGauge, uint64(len(gauges))) + atomic.AddUint64(&a.nbContextSet, uint64(len(sets))) return metrics } +// getContext returns the context for a metric name and tags. +// +// The context is the metric name and tags separated by a separator symbol. +// It is not intended to be used as a metric name but as a unique key to aggregate func getContext(name string, tags []string) string { - return name + ":" + strings.Join(tags, tagSeparatorSymbol) + c, _ := getContextAndTags(name, tags) + return c } +// getContextAndTags returns the context and tags for a metric name and tags. +// +// See getContext for usage for context +// The tags are the tags separated by a separator symbol and can be re-used to pass down to the writer func getContextAndTags(name string, tags []string) (string, string) { - stringTags := strings.Join(tags, tagSeparatorSymbol) - return name + ":" + stringTags, stringTags + if len(tags) == 0 { + return name, "" + } + n := len(name) + len(nameSeparatorSymbol) + len(tagSeparatorSymbol)*(len(tags)-1) + for _, s := range tags { + n += len(s) + } + + var sb strings.Builder + sb.Grow(n) + sb.WriteString(name) + sb.WriteString(nameSeparatorSymbol) + sb.WriteString(tags[0]) + for _, s := range tags[1:] { + sb.WriteString(tagSeparatorSymbol) + sb.WriteString(s) + } + + s := sb.String() + + return s, s[len(name)+len(nameSeparatorSymbol):] } func (a *aggregator) count(name string, value int64, tags []string) error { diff --git a/vendor/github.com/DataDog/datadog-go/statsd/buffer.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/buffer.go similarity index 84% rename from vendor/github.com/DataDog/datadog-go/statsd/buffer.go rename to vendor/github.com/DataDog/datadog-go/v5/statsd/buffer.go index 34dc0d3fb..91f2e32b9 100644 --- a/vendor/github.com/DataDog/datadog-go/statsd/buffer.go +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/buffer.go @@ -4,11 +4,15 @@ import ( "strconv" ) -type bufferFullError string +// MessageTooLongError is an error returned when a sample, event or service check is too large once serialized. See +// WithMaxBytesPerPayload option for more details. +type MessageTooLongError struct{} -func (e bufferFullError) Error() string { return string(e) } +func (e MessageTooLongError) Error() string { + return "message too long. See 'WithMaxBytesPerPayload' documentation." +} -const errBufferFull = bufferFullError("statsd buffer is full") +var errBufferFull = MessageTooLongError{} type partialWriteError string @@ -19,7 +23,7 @@ const errPartialWrite = partialWriteError("value partially written") const metricOverhead = 512 // statsdBuffer is a buffer containing statsd messages -// this struct methods are NOT safe for concurent use +// this struct methods are NOT safe for concurrent use type statsdBuffer struct { buffer []byte maxSize int @@ -35,22 +39,24 @@ func newStatsdBuffer(maxSize, maxElements int) *statsdBuffer { } } -func (b *statsdBuffer) writeGauge(namespace string, globalTags []string, name string, value float64, tags []string, rate float64) error { +func (b *statsdBuffer) writeGauge(namespace string, globalTags []string, name string, value float64, tags []string, rate float64, timestamp int64) error { if b.elementCount >= b.maxElements { return errBufferFull } originalBuffer := b.buffer b.buffer = appendGauge(b.buffer, namespace, globalTags, name, value, tags, rate) + b.buffer = appendTimestamp(b.buffer, timestamp) b.writeSeparator() return b.validateNewElement(originalBuffer) } -func (b *statsdBuffer) writeCount(namespace string, globalTags []string, name string, value int64, tags []string, rate float64) error { +func (b *statsdBuffer) writeCount(namespace string, globalTags []string, name string, value int64, tags []string, rate float64, timestamp int64) error { if b.elementCount >= b.maxElements { return errBufferFull } originalBuffer := b.buffer b.buffer = appendCount(b.buffer, namespace, globalTags, name, value, tags, rate) + b.buffer = appendTimestamp(b.buffer, timestamp) b.writeSeparator() return b.validateNewElement(originalBuffer) } @@ -66,7 +72,7 @@ func (b *statsdBuffer) writeHistogram(namespace string, globalTags []string, nam } // writeAggregated serialized as many values as possible in the current buffer and return the position in values where it stopped. -func (b *statsdBuffer) writeAggregated(metricSymbol []byte, namespace string, globalTags []string, name string, values []float64, tags string, tagSize int, precision int) (int, error) { +func (b *statsdBuffer) writeAggregated(metricSymbol []byte, namespace string, globalTags []string, name string, values []float64, tags string, tagSize int, precision int, rate float64) (int, error) { if b.elementCount >= b.maxElements { return 0, errBufferFull } @@ -106,7 +112,9 @@ func (b *statsdBuffer) writeAggregated(metricSymbol []byte, namespace string, gl b.buffer = append(b.buffer, '|') b.buffer = append(b.buffer, metricSymbol...) + b.buffer = appendRate(b.buffer, rate) b.buffer = appendTagsAggregated(b.buffer, globalTags, tags) + b.buffer = appendContainerID(b.buffer) b.writeSeparator() b.elementCount++ @@ -147,7 +155,7 @@ func (b *statsdBuffer) writeTiming(namespace string, globalTags []string, name s return b.validateNewElement(originalBuffer) } -func (b *statsdBuffer) writeEvent(event Event, globalTags []string) error { +func (b *statsdBuffer) writeEvent(event *Event, globalTags []string) error { if b.elementCount >= b.maxElements { return errBufferFull } @@ -157,7 +165,7 @@ func (b *statsdBuffer) writeEvent(event Event, globalTags []string) error { return b.validateNewElement(originalBuffer) } -func (b *statsdBuffer) writeServiceCheck(serviceCheck ServiceCheck, globalTags []string) error { +func (b *statsdBuffer) writeServiceCheck(serviceCheck *ServiceCheck, globalTags []string) error { if b.elementCount >= b.maxElements { return errBufferFull } diff --git a/vendor/github.com/DataDog/datadog-go/statsd/buffer_pool.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/buffer_pool.go similarity index 100% rename from vendor/github.com/DataDog/datadog-go/statsd/buffer_pool.go rename to vendor/github.com/DataDog/datadog-go/v5/statsd/buffer_pool.go diff --git a/vendor/github.com/DataDog/datadog-go/v5/statsd/buffered_metric_context.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/buffered_metric_context.go new file mode 100644 index 000000000..94b31fe5b --- /dev/null +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/buffered_metric_context.go @@ -0,0 +1,104 @@ +package statsd + +import ( + "math/rand" + "sync" + "sync/atomic" + "time" +) + +// bufferedMetricContexts represent the contexts for Histograms, Distributions +// and Timing. Since those 3 metric types behave the same way and are sampled +// with the same type they're represented by the same class. +type bufferedMetricContexts struct { + nbContext uint64 + mutex sync.RWMutex + values bufferedMetricMap + newMetric func(string, float64, string, float64) *bufferedMetric + + // Each bufferedMetricContexts uses its own random source and random + // lock to prevent goroutines from contending for the lock on the + // "math/rand" package-global random source (e.g. calls like + // "rand.Float64()" must acquire a shared lock to get the next + // pseudorandom number). + random *rand.Rand + randomLock sync.Mutex +} + +func newBufferedContexts(newMetric func(string, float64, string, int64, float64) *bufferedMetric, maxSamples int64) bufferedMetricContexts { + return bufferedMetricContexts{ + values: bufferedMetricMap{}, + newMetric: func(name string, value float64, stringTags string, rate float64) *bufferedMetric { + return newMetric(name, value, stringTags, maxSamples, rate) + }, + // Note that calling "time.Now().UnixNano()" repeatedly quickly may return + // very similar values. That's fine for seeding the worker-specific random + // source because we just need an evenly distributed stream of float values. + // Do not use this random source for cryptographic randomness. + random: rand.New(rand.NewSource(time.Now().UnixNano())), + } +} + +func (bc *bufferedMetricContexts) flush(metrics []metric) []metric { + bc.mutex.Lock() + values := bc.values + bc.values = bufferedMetricMap{} + bc.mutex.Unlock() + + for _, d := range values { + d.Lock() + metrics = append(metrics, d.flushUnsafe()) + d.Unlock() + } + atomic.AddUint64(&bc.nbContext, uint64(len(values))) + return metrics +} + +func (bc *bufferedMetricContexts) sample(name string, value float64, tags []string, rate float64) error { + keepingSample := shouldSample(rate, bc.random, &bc.randomLock) + + // If we don't keep the sample, return early. If we do keep the sample + // we end up storing the *first* observed sampling rate in the metric. + // This is the *wrong* behavior but it's the one we had before and the alternative would increase lock contention too + // much with the current code. + // TODO: change this behavior in the future, probably by introducing thread-local storage and lockless stuctures. + // If this code is removed, also remove the observed sampling rate in the metric and fix `bufferedMetric.flushUnsafe()` + if !keepingSample { + return nil + } + + context, stringTags := getContextAndTags(name, tags) + var v *bufferedMetric + + bc.mutex.RLock() + v, _ = bc.values[context] + bc.mutex.RUnlock() + + // Create it if it wasn't found + if v == nil { + bc.mutex.Lock() + // It might have been created by another goroutine since last call + v, _ = bc.values[context] + if v == nil { + // If we might keep a sample that we should have skipped, but that should not drastically affect performances. + bc.values[context] = bc.newMetric(name, value, stringTags, rate) + // We added a new value, we need to unlock the mutex and quit + bc.mutex.Unlock() + return nil + } + bc.mutex.Unlock() + } + + // Now we can keep the sample or skip it + if keepingSample { + v.maybeKeepSample(value, bc.random, &bc.randomLock) + } else { + v.skipSample() + } + + return nil +} + +func (bc *bufferedMetricContexts) getNbContext() uint64 { + return atomic.LoadUint64(&bc.nbContext) +} diff --git a/vendor/github.com/DataDog/datadog-go/v5/statsd/container.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/container.go new file mode 100644 index 000000000..20d69ef63 --- /dev/null +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/container.go @@ -0,0 +1,19 @@ +package statsd + +import ( + "sync" +) + +var ( + // containerID holds the container ID. + containerID = "" + + initOnce sync.Once +) + +// getContainerID returns the container ID configured at the client creation +// It can either be auto-discovered with origin detection or provided by the user. +// User-defined container ID is prioritized. +func getContainerID() string { + return containerID +} diff --git a/vendor/github.com/DataDog/datadog-go/v5/statsd/container_linux.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/container_linux.go new file mode 100644 index 000000000..125132349 --- /dev/null +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/container_linux.go @@ -0,0 +1,219 @@ +//go:build linux +// +build linux + +package statsd + +import ( + "bufio" + "fmt" + "io" + "os" + "path" + "regexp" + "strings" + "syscall" +) + +const ( + // cgroupPath is the path to the cgroup file where we can find the container id if one exists. + cgroupPath = "/proc/self/cgroup" + + // selfMountinfo is the path to the mountinfo path where we can find the container id in case cgroup namespace is preventing the use of /proc/self/cgroup + selfMountInfoPath = "/proc/self/mountinfo" + + // defaultCgroupMountPath is the default path to the cgroup mount point. + defaultCgroupMountPath = "/sys/fs/cgroup" + + // cgroupV1BaseController is the controller used to identify the container-id for cgroup v1 + cgroupV1BaseController = "memory" + + uuidSource = "[0-9a-f]{8}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{12}" + containerSource = "[0-9a-f]{64}" + taskSource = "[0-9a-f]{32}-\\d+" + + containerdSandboxPrefix = "sandboxes" + + // ContainerRegexpStr defines the regexp used to match container IDs + // ([0-9a-f]{64}) is standard container id used pretty much everywhere + // ([0-9a-f]{32}-\d+) is container id used by AWS ECS + // ([0-9a-f]{8}(-[0-9a-f]{4}){4}$) is container id used by Garden + containerRegexpStr = "([0-9a-f]{64})|([0-9a-f]{32}-\\d+)|([0-9a-f]{8}(-[0-9a-f]{4}){4}$)" + // cIDRegexpStr defines the regexp used to match container IDs in /proc/self/mountinfo + cIDRegexpStr = `.*/([^\s/]+)/(` + containerRegexpStr + `)/[\S]*hostname` + + // From https://github.com/torvalds/linux/blob/5859a2b1991101d6b978f3feb5325dad39421f29/include/linux/proc_ns.h#L41-L49 + // Currently, host namespace inode number are hardcoded, which can be used to detect + // if we're running in host namespace or not (does not work when running in DinD) + hostCgroupNamespaceInode = 0xEFFFFFFB +) + +var ( + // expLine matches a line in the /proc/self/cgroup file. It has a submatch for the last element (path), which contains the container ID. + expLine = regexp.MustCompile(`^\d+:[^:]*:(.+)$`) + + // expContainerID matches contained IDs and sources. Source: https://github.com/Qard/container-info/blob/master/index.js + expContainerID = regexp.MustCompile(fmt.Sprintf(`(%s|%s|%s)(?:.scope)?$`, uuidSource, containerSource, taskSource)) + + cIDMountInfoRegexp = regexp.MustCompile(cIDRegexpStr) + + // initContainerID initializes the container ID. + initContainerID = internalInitContainerID +) + +// parseContainerID finds the first container ID reading from r and returns it. +func parseContainerID(r io.Reader) string { + scn := bufio.NewScanner(r) + for scn.Scan() { + path := expLine.FindStringSubmatch(scn.Text()) + if len(path) != 2 { + // invalid entry, continue + continue + } + if parts := expContainerID.FindStringSubmatch(path[1]); len(parts) == 2 { + return parts[1] + } + } + return "" +} + +// readContainerID attempts to return the container ID from the provided file path or empty on failure. +func readContainerID(fpath string) string { + f, err := os.Open(fpath) + if err != nil { + return "" + } + defer f.Close() + return parseContainerID(f) +} + +// Parsing /proc/self/mountinfo is not always reliable in Kubernetes+containerd (at least) +// We're still trying to use it as it may help in some cgroupv2 configurations (Docker, ECS, raw containerd) +func parseMountinfo(r io.Reader) string { + scn := bufio.NewScanner(r) + for scn.Scan() { + line := scn.Text() + allMatches := cIDMountInfoRegexp.FindAllStringSubmatch(line, -1) + if len(allMatches) == 0 { + continue + } + + // We're interest in rightmost match + matches := allMatches[len(allMatches)-1] + if len(matches) > 0 && matches[1] != containerdSandboxPrefix { + return matches[2] + } + } + + return "" +} + +func readMountinfo(path string) string { + f, err := os.Open(path) + if err != nil { + return "" + } + defer f.Close() + return parseMountinfo(f) +} + +func isHostCgroupNamespace() bool { + fi, err := os.Stat("/proc/self/ns/cgroup") + if err != nil { + return false + } + + inode := fi.Sys().(*syscall.Stat_t).Ino + + return inode == hostCgroupNamespaceInode +} + +// parseCgroupNodePath parses /proc/self/cgroup and returns a map of controller to its associated cgroup node path. +func parseCgroupNodePath(r io.Reader) map[string]string { + res := make(map[string]string) + scn := bufio.NewScanner(r) + for scn.Scan() { + line := scn.Text() + tokens := strings.Split(line, ":") + if len(tokens) != 3 { + continue + } + if tokens[1] == cgroupV1BaseController || tokens[1] == "" { + res[tokens[1]] = tokens[2] + } + } + return res +} + +// getCgroupInode returns the cgroup controller inode if it exists otherwise an empty string. +// The inode is prefixed by "in-" and is used by the agent to retrieve the container ID. +// For cgroup v1, we use the memory controller. +func getCgroupInode(cgroupMountPath, procSelfCgroupPath string) string { + // Parse /proc/self/cgroup to retrieve the paths to the memory controller (cgroupv1) and the cgroup node (cgroupv2) + f, err := os.Open(procSelfCgroupPath) + if err != nil { + return "" + } + defer f.Close() + cgroupControllersPaths := parseCgroupNodePath(f) + // Retrieve the cgroup inode from /sys/fs/cgroup+controller+cgroupNodePath + for _, controller := range []string{cgroupV1BaseController, ""} { + cgroupNodePath, ok := cgroupControllersPaths[controller] + if !ok { + continue + } + inode := inodeForPath(path.Join(cgroupMountPath, controller, cgroupNodePath)) + if inode != "" { + return inode + } + } + return "" +} + +// inodeForPath returns the inode for the provided path or empty on failure. +func inodeForPath(path string) string { + fi, err := os.Stat(path) + if err != nil { + return "" + } + stats, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return "" + } + return fmt.Sprintf("in-%d", stats.Ino) +} + +// internalInitContainerID initializes the container ID. +// It can either be provided by the user or read from cgroups. +func internalInitContainerID(userProvidedID string, cgroupFallback, isHostCgroupNs bool) { + initOnce.Do(func() { + readCIDOrInode(userProvidedID, cgroupPath, selfMountInfoPath, defaultCgroupMountPath, cgroupFallback, isHostCgroupNs) + }) +} + +// readCIDOrInode reads the container ID from the user provided ID, cgroups or mountinfo. +func readCIDOrInode(userProvidedID, cgroupPath, selfMountInfoPath, defaultCgroupMountPath string, cgroupFallback, isHostCgroupNs bool) { + if userProvidedID != "" { + containerID = userProvidedID + return + } + + if cgroupFallback { + containerID = readContainerID(cgroupPath) + if containerID != "" { + return + } + + containerID = readMountinfo(selfMountInfoPath) + if containerID != "" { + return + } + + // If we're in the host cgroup namespace, the cid should be retrievable in /proc/self/cgroup + // In private cgroup namespace, we can retrieve the cgroup controller inode. + if containerID == "" && isHostCgroupNs { + return + } + + containerID = getCgroupInode(defaultCgroupMountPath, cgroupPath) + } +} diff --git a/vendor/github.com/DataDog/datadog-go/v5/statsd/container_stub.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/container_stub.go new file mode 100644 index 000000000..29ab7f2c9 --- /dev/null +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/container_stub.go @@ -0,0 +1,17 @@ +//go:build !linux +// +build !linux + +package statsd + +func isHostCgroupNamespace() bool { + return false +} + +var initContainerID = func(userProvidedID string, _, _ bool) { + initOnce.Do(func() { + if userProvidedID != "" { + containerID = userProvidedID + return + } + }) +} diff --git a/vendor/github.com/DataDog/datadog-go/v5/statsd/error_handler.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/error_handler.go new file mode 100644 index 000000000..007626273 --- /dev/null +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/error_handler.go @@ -0,0 +1,22 @@ +package statsd + +import ( + "log" +) + +func LoggingErrorHandler(err error) { + if e, ok := err.(*ErrorInputChannelFull); ok { + log.Printf( + "Input Queue is full (%d elements): %s %s dropped - %s - increase channel buffer size with `WithChannelModeBufferSize()`", + e.ChannelSize, e.Metric.name, e.Metric.tags, e.Msg, + ) + return + } else if e, ok := err.(*ErrorSenderChannelFull); ok { + log.Printf( + "Sender Queue is full (%d elements): %d metrics dropped - %s - increase sender queue size with `WithSenderQueueSize()`", + e.ChannelSize, e.LostElements, e.Msg, + ) + } else { + log.Printf("Error: %v", err) + } +} diff --git a/vendor/github.com/DataDog/datadog-go/statsd/event.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/event.go similarity index 81% rename from vendor/github.com/DataDog/datadog-go/statsd/event.go rename to vendor/github.com/DataDog/datadog-go/v5/statsd/event.go index 35c535362..a2ca4faf7 100644 --- a/vendor/github.com/DataDog/datadog-go/statsd/event.go +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/event.go @@ -37,7 +37,7 @@ const ( type Event struct { // Title of the event. Required. Title string - // Text is the description of the event. Required. + // Text is the description of the event. Text string // Timestamp is a timestamp for the event. If not provided, the dogstatsd // server will set this to the current time. @@ -67,22 +67,9 @@ func NewEvent(title, text string) *Event { } // Check verifies that an event is valid. -func (e Event) Check() error { +func (e *Event) Check() error { if len(e.Title) == 0 { return fmt.Errorf("statsd.Event title is required") } return nil } - -// Encode returns the dogstatsd wire protocol representation for an event. -// Tags may be passed which will be added to the encoded output but not to -// the Event's list of tags, eg. for default tags. -func (e Event) Encode(tags ...string) (string, error) { - err := e.Check() - if err != nil { - return "", err - } - var buffer []byte - buffer = appendEvent(buffer, e, tags) - return string(buffer), nil -} diff --git a/vendor/github.com/DataDog/datadog-go/statsd/fnv1a.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/fnv1a.go similarity index 100% rename from vendor/github.com/DataDog/datadog-go/statsd/fnv1a.go rename to vendor/github.com/DataDog/datadog-go/v5/statsd/fnv1a.go diff --git a/vendor/github.com/DataDog/datadog-go/statsd/format.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/format.go similarity index 87% rename from vendor/github.com/DataDog/datadog-go/statsd/format.go rename to vendor/github.com/DataDog/datadog-go/v5/statsd/format.go index 8d62aa7ba..f3ab9231f 100644 --- a/vendor/github.com/DataDog/datadog-go/statsd/format.go +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/format.go @@ -6,13 +6,14 @@ import ( ) var ( - gaugeSymbol = []byte("g") - countSymbol = []byte("c") - histogramSymbol = []byte("h") - distributionSymbol = []byte("d") - setSymbol = []byte("s") - timingSymbol = []byte("ms") - tagSeparatorSymbol = "," + gaugeSymbol = []byte("g") + countSymbol = []byte("c") + histogramSymbol = []byte("h") + distributionSymbol = []byte("d") + setSymbol = []byte("s") + timingSymbol = []byte("ms") + tagSeparatorSymbol = "," + nameSeparatorSymbol = ":" ) func appendHeader(buffer []byte, namespace string, name string) []byte { @@ -101,6 +102,7 @@ func appendFloatMetric(buffer []byte, typeSymbol []byte, namespace string, globa buffer = append(buffer, typeSymbol...) buffer = appendRate(buffer, rate) buffer = appendTags(buffer, globalTags, tags) + buffer = appendContainerID(buffer) return buffer } @@ -111,6 +113,7 @@ func appendIntegerMetric(buffer []byte, typeSymbol []byte, namespace string, glo buffer = append(buffer, typeSymbol...) buffer = appendRate(buffer, rate) buffer = appendTags(buffer, globalTags, tags) + buffer = appendContainerID(buffer) return buffer } @@ -121,6 +124,7 @@ func appendStringMetric(buffer []byte, typeSymbol []byte, namespace string, glob buffer = append(buffer, typeSymbol...) buffer = appendRate(buffer, rate) buffer = appendTags(buffer, globalTags, tags) + buffer = appendContainerID(buffer) return buffer } @@ -163,7 +167,7 @@ func appendEscapedEventText(buffer []byte, text string) []byte { return buffer } -func appendEvent(buffer []byte, event Event, globalTags []string) []byte { +func appendEvent(buffer []byte, event *Event, globalTags []string) []byte { escapedTextLen := escapedEventTextLen(event.Text) buffer = append(buffer, "_e{"...) @@ -210,6 +214,7 @@ func appendEvent(buffer []byte, event Event, globalTags []string) []byte { } buffer = appendTags(buffer, globalTags, event.Tags) + buffer = appendContainerID(buffer) return buffer } @@ -227,7 +232,7 @@ func appendEscapedServiceCheckText(buffer []byte, text string) []byte { return buffer } -func appendServiceCheck(buffer []byte, serviceCheck ServiceCheck, globalTags []string) []byte { +func appendServiceCheck(buffer []byte, serviceCheck *ServiceCheck, globalTags []string) []byte { buffer = append(buffer, "_sc|"...) buffer = append(buffer, serviceCheck.Name...) buffer = append(buffer, '|') @@ -249,9 +254,27 @@ func appendServiceCheck(buffer []byte, serviceCheck ServiceCheck, globalTags []s buffer = append(buffer, "|m:"...) buffer = appendEscapedServiceCheckText(buffer, serviceCheck.Message) } + + buffer = appendContainerID(buffer) return buffer } func appendSeparator(buffer []byte) []byte { return append(buffer, '\n') } + +func appendContainerID(buffer []byte) []byte { + if containerID := getContainerID(); len(containerID) > 0 { + buffer = append(buffer, "|c:"...) + buffer = append(buffer, containerID...) + } + return buffer +} + +func appendTimestamp(buffer []byte, timestamp int64) []byte { + if timestamp > noTimestamp { + buffer = append(buffer, "|T"...) + buffer = strconv.AppendInt(buffer, timestamp, 10) + } + return buffer +} diff --git a/vendor/github.com/DataDog/datadog-go/v5/statsd/metrics.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/metrics.go new file mode 100644 index 000000000..3d243b7a6 --- /dev/null +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/metrics.go @@ -0,0 +1,268 @@ +package statsd + +import ( + "math" + "math/rand" + "sync" + "sync/atomic" +) + +/* +Those are metrics type that can be aggregated on the client side: + - Gauge + - Count + - Set +*/ + +type countMetric struct { + value int64 + name string + tags []string +} + +func newCountMetric(name string, value int64, tags []string) *countMetric { + return &countMetric{ + value: value, + name: name, + tags: copySlice(tags), + } +} + +func (c *countMetric) sample(v int64) { + atomic.AddInt64(&c.value, v) +} + +func (c *countMetric) flushUnsafe() metric { + return metric{ + metricType: count, + name: c.name, + tags: c.tags, + rate: 1, + ivalue: c.value, + } +} + +// Gauge + +type gaugeMetric struct { + value uint64 + name string + tags []string +} + +func newGaugeMetric(name string, value float64, tags []string) *gaugeMetric { + return &gaugeMetric{ + value: math.Float64bits(value), + name: name, + tags: copySlice(tags), + } +} + +func (g *gaugeMetric) sample(v float64) { + atomic.StoreUint64(&g.value, math.Float64bits(v)) +} + +func (g *gaugeMetric) flushUnsafe() metric { + return metric{ + metricType: gauge, + name: g.name, + tags: g.tags, + rate: 1, + fvalue: math.Float64frombits(g.value), + } +} + +// Set + +type setMetric struct { + data map[string]struct{} + name string + tags []string + sync.Mutex +} + +func newSetMetric(name string, value string, tags []string) *setMetric { + set := &setMetric{ + data: map[string]struct{}{}, + name: name, + tags: copySlice(tags), + } + set.data[value] = struct{}{} + return set +} + +func (s *setMetric) sample(v string) { + s.Lock() + defer s.Unlock() + s.data[v] = struct{}{} +} + +// Sets are aggregated on the agent side too. We flush the keys so a set from +// multiple application can be correctly aggregated on the agent side. +func (s *setMetric) flushUnsafe() []metric { + if len(s.data) == 0 { + return nil + } + + metrics := make([]metric, len(s.data)) + i := 0 + for value := range s.data { + metrics[i] = metric{ + metricType: set, + name: s.name, + tags: s.tags, + rate: 1, + svalue: value, + } + i++ + } + return metrics +} + +// Histograms, Distributions and Timings + +type bufferedMetric struct { + sync.Mutex + + // Kept samples (after sampling) + data []float64 + // Total stored samples (after sampling) + storedSamples int64 + // Total number of observed samples (before sampling). This is used to keep + // the sampling rate correct. + totalSamples int64 + + name string + // Histograms and Distributions store tags as one string since we need + // to compute its size multiple time when serializing. + tags string + mtype metricType + + // maxSamples is the maximum number of samples we keep in memory + maxSamples int64 + + // The first observed user-specified sample rate. When specified + // it is used because we don't know better. + specifiedRate float64 +} + +func (s *bufferedMetric) sample(v float64) { + s.Lock() + defer s.Unlock() + s.sampleUnsafe(v) +} + +func (s *bufferedMetric) sampleUnsafe(v float64) { + s.data = append(s.data, v) + s.storedSamples++ + // Total samples needs to be incremented though an atomic because it can be accessed without the lock. + atomic.AddInt64(&s.totalSamples, 1) +} + +func (s *bufferedMetric) maybeKeepSample(v float64, rand *rand.Rand, randLock *sync.Mutex) { + s.Lock() + defer s.Unlock() + if s.maxSamples > 0 { + if s.storedSamples >= s.maxSamples { + // We reached the maximum number of samples we can keep in memory, so we randomly + // replace a sample. + randLock.Lock() + i := rand.Int63n(atomic.LoadInt64(&s.totalSamples)) + randLock.Unlock() + if i < s.maxSamples { + s.data[i] = v + } + } else { + s.data[s.storedSamples] = v + s.storedSamples++ + } + s.totalSamples++ + } else { + // This code path appends to the slice since we did not pre-allocate memory in this case. + s.sampleUnsafe(v) + } +} + +func (s *bufferedMetric) skipSample() { + atomic.AddInt64(&s.totalSamples, 1) +} + +func (s *bufferedMetric) flushUnsafe() metric { + totalSamples := atomic.LoadInt64(&s.totalSamples) + var rate float64 + + // If the user had a specified rate send it because we don't know better. + // This code should be removed once we can also remove the early return at the top of + // `bufferedMetricContexts.sample` + if s.specifiedRate != 1.0 { + rate = s.specifiedRate + } else { + rate = float64(s.storedSamples) / float64(totalSamples) + } + + return metric{ + metricType: s.mtype, + name: s.name, + stags: s.tags, + rate: rate, + fvalues: s.data[:s.storedSamples], + } +} + +type histogramMetric = bufferedMetric + +func newHistogramMetric(name string, value float64, stringTags string, maxSamples int64, rate float64) *histogramMetric { + return &histogramMetric{ + data: newData(value, maxSamples), + totalSamples: 1, + storedSamples: 1, + name: name, + tags: stringTags, + mtype: histogramAggregated, + maxSamples: maxSamples, + specifiedRate: rate, + } +} + +type distributionMetric = bufferedMetric + +func newDistributionMetric(name string, value float64, stringTags string, maxSamples int64, rate float64) *distributionMetric { + return &distributionMetric{ + data: newData(value, maxSamples), + totalSamples: 1, + storedSamples: 1, + name: name, + tags: stringTags, + mtype: distributionAggregated, + maxSamples: maxSamples, + specifiedRate: rate, + } +} + +type timingMetric = bufferedMetric + +func newTimingMetric(name string, value float64, stringTags string, maxSamples int64, rate float64) *timingMetric { + return &timingMetric{ + data: newData(value, maxSamples), + totalSamples: 1, + storedSamples: 1, + name: name, + tags: stringTags, + mtype: timingAggregated, + maxSamples: maxSamples, + specifiedRate: rate, + } +} + +// newData creates a new slice of float64 with the given capacity. If maxSample +// is less than or equal to 0, it returns a slice with the given value as the +// only element. +func newData(value float64, maxSample int64) []float64 { + if maxSample <= 0 { + return []float64{value} + } else { + data := make([]float64, maxSample) + data[0] = value + return data + } +} diff --git a/vendor/github.com/DataDog/datadog-go/statsd/noop.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/noop.go similarity index 71% rename from vendor/github.com/DataDog/datadog-go/statsd/noop.go rename to vendor/github.com/DataDog/datadog-go/v5/statsd/noop.go index 010783333..6500cde9a 100644 --- a/vendor/github.com/DataDog/datadog-go/statsd/noop.go +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/noop.go @@ -11,11 +11,21 @@ func (n *NoOpClient) Gauge(name string, value float64, tags []string, rate float return nil } +// GaugeWithTimestamp does nothing and returns nil +func (n *NoOpClient) GaugeWithTimestamp(name string, value float64, tags []string, rate float64, timestamp time.Time) error { + return nil +} + // Count does nothing and returns nil func (n *NoOpClient) Count(name string, value int64, tags []string, rate float64) error { return nil } +// CountWithTimestamp does nothing and returns nil +func (n *NoOpClient) CountWithTimestamp(name string, value int64, tags []string, rate float64, timestamp time.Time) error { + return nil +} + // Histogram does nothing and returns nil func (n *NoOpClient) Histogram(name string, value float64, tags []string, rate float64) error { return nil @@ -81,11 +91,28 @@ func (n *NoOpClient) Flush() error { return nil } -// SetWriteTimeout does nothing and returns nil -func (n *NoOpClient) SetWriteTimeout(d time.Duration) error { - return nil +// IsClosed does nothing and return false +func (n *NoOpClient) IsClosed() bool { + return false +} + +// GetTelemetry does nothing and returns an empty Telemetry +func (n *NoOpClient) GetTelemetry() Telemetry { + return Telemetry{} } // Verify that NoOpClient implements the ClientInterface. // https://golang.org/doc/faq#guarantee_satisfies_interface var _ ClientInterface = &NoOpClient{} + +// NoOpClientDirect implements ClientDirectInterface and does nothing. +type NoOpClientDirect struct { + NoOpClient +} + +// DistributionSamples does nothing and returns nil +func (n *NoOpClientDirect) DistributionSamples(name string, values []float64, tags []string, rate float64) error { + return nil +} + +var _ ClientDirectInterface = &NoOpClientDirect{} diff --git a/vendor/github.com/DataDog/datadog-go/v5/statsd/options.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/options.go new file mode 100644 index 000000000..e007505a6 --- /dev/null +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/options.go @@ -0,0 +1,414 @@ +package statsd + +import ( + "fmt" + "math" + "strings" + "time" +) + +var ( + defaultNamespace = "" + defaultTags = []string{} + defaultMaxBytesPerPayload = 0 + defaultMaxMessagesPerPayload = math.MaxInt32 + defaultBufferPoolSize = 0 + defaultBufferFlushInterval = 100 * time.Millisecond + defaultWorkerCount = 32 + defaultSenderQueueSize = 0 + defaultWriteTimeout = 100 * time.Millisecond + defaultConnectTimeout = 1000 * time.Millisecond + defaultTelemetry = true + defaultReceivingMode = mutexMode + defaultChannelModeBufferSize = 4096 + defaultAggregationFlushInterval = 2 * time.Second + defaultAggregation = true + defaultExtendedAggregation = false + defaultMaxBufferedSamplesPerContext = -1 + defaultOriginDetection = true + defaultChannelModeErrorsWhenFull = false + defaultErrorHandler = func(error) {} +) + +// Options contains the configuration options for a client. +type Options struct { + namespace string + tags []string + maxBytesPerPayload int + maxMessagesPerPayload int + bufferPoolSize int + bufferFlushInterval time.Duration + workersCount int + senderQueueSize int + writeTimeout time.Duration + connectTimeout time.Duration + telemetry bool + receiveMode receivingMode + channelModeBufferSize int + aggregationFlushInterval time.Duration + aggregation bool + extendedAggregation bool + maxBufferedSamplesPerContext int + telemetryAddr string + originDetection bool + containerID string + channelModeErrorsWhenFull bool + errorHandler ErrorHandler +} + +func resolveOptions(options []Option) (*Options, error) { + o := &Options{ + namespace: defaultNamespace, + tags: defaultTags, + maxBytesPerPayload: defaultMaxBytesPerPayload, + maxMessagesPerPayload: defaultMaxMessagesPerPayload, + bufferPoolSize: defaultBufferPoolSize, + bufferFlushInterval: defaultBufferFlushInterval, + workersCount: defaultWorkerCount, + senderQueueSize: defaultSenderQueueSize, + writeTimeout: defaultWriteTimeout, + connectTimeout: defaultConnectTimeout, + telemetry: defaultTelemetry, + receiveMode: defaultReceivingMode, + channelModeBufferSize: defaultChannelModeBufferSize, + aggregationFlushInterval: defaultAggregationFlushInterval, + aggregation: defaultAggregation, + extendedAggregation: defaultExtendedAggregation, + maxBufferedSamplesPerContext: defaultMaxBufferedSamplesPerContext, + originDetection: defaultOriginDetection, + channelModeErrorsWhenFull: defaultChannelModeErrorsWhenFull, + errorHandler: defaultErrorHandler, + } + + for _, option := range options { + err := option(o) + if err != nil { + return nil, err + } + } + + return o, nil +} + +// Option is a client option. Can return an error if validation fails. +type Option func(*Options) error + +// WithNamespace sets a string to be prepend to all metrics, events and service checks name. +// +// A '.' will automatically be added after the namespace if needed. For example a metrics 'test' with a namespace 'prod' +// will produce a final metric named 'prod.test'. +func WithNamespace(namespace string) Option { + return func(o *Options) error { + if strings.HasSuffix(namespace, ".") { + o.namespace = namespace + } else { + o.namespace = namespace + "." + } + return nil + } +} + +// WithTags sets global tags to be applied to every metrics, events and service checks. +func WithTags(tags []string) Option { + return func(o *Options) error { + o.tags = tags + return nil + } +} + +// WithMaxMessagesPerPayload sets the maximum number of metrics, events and/or service checks that a single payload can +// contain. +// +// The default is 'math.MaxInt32' which will most likely let the WithMaxBytesPerPayload option take precedence. This +// option can be set to `1` to create an unbuffered client (each metrics/event/service check will be send in its own +// payload to the agent). +func WithMaxMessagesPerPayload(maxMessagesPerPayload int) Option { + return func(o *Options) error { + o.maxMessagesPerPayload = maxMessagesPerPayload + return nil + } +} + +// WithMaxBytesPerPayload sets the maximum number of bytes a single payload can contain. Each sample, even and service +// check must be lower than this value once serialized or an `MessageTooLongError` is returned. +// +// The default value 0 which will set the option to the optimal size for the transport protocol used: 1432 for UDP and +// named pipe and 8192 for UDS. Those values offer the best performances. +// Be careful when changing this option, see +// https://docs.datadoghq.com/developers/dogstatsd/high_throughput/#ensure-proper-packet-sizes. +func WithMaxBytesPerPayload(MaxBytesPerPayload int) Option { + return func(o *Options) error { + o.maxBytesPerPayload = MaxBytesPerPayload + return nil + } +} + +// WithBufferPoolSize sets the size of the pool of buffers used to serialized metrics, events and service_checks. +// +// The default, 0, will set the option to the optimal size for the transport protocol used: 2048 for UDP and named pipe +// and 512 for UDS. +func WithBufferPoolSize(bufferPoolSize int) Option { + return func(o *Options) error { + o.bufferPoolSize = bufferPoolSize + return nil + } +} + +// WithBufferFlushInterval sets the interval after which the current buffer is flushed. +// +// A buffers are used to serialized data, they're flushed either when full (see WithMaxBytesPerPayload) or when it's +// been open for longer than this interval. +// +// With apps sending a high number of metrics/events/service_checks the interval rarely timeout. But with slow sending +// apps increasing this value will reduce the number of payload sent on the wire as more data is serialized in the same +// payload. +// +// Default is 100ms +func WithBufferFlushInterval(bufferFlushInterval time.Duration) Option { + return func(o *Options) error { + o.bufferFlushInterval = bufferFlushInterval + return nil + } +} + +// WithWorkersCount sets the number of workers that will be used to serialized data. +// +// Those workers allow the use of multiple buffers at the same time (see WithBufferPoolSize) to reduce lock contention. +// +// Default is 32. +func WithWorkersCount(workersCount int) Option { + return func(o *Options) error { + if workersCount < 1 { + return fmt.Errorf("workersCount must be a positive integer") + } + o.workersCount = workersCount + return nil + } +} + +// WithSenderQueueSize sets the size of the sender queue in number of buffers. +// +// After data has been serialized in a buffer they're pushed to a queue that the sender will consume and then each one +// ot the agent. +// +// The default value 0 will set the option to the optimal size for the transport protocol used: 2048 for UDP and named +// pipe and 512 for UDS. +func WithSenderQueueSize(senderQueueSize int) Option { + return func(o *Options) error { + o.senderQueueSize = senderQueueSize + return nil + } +} + +// WithWriteTimeout sets the timeout for network communication with the Agent, after this interval a payload is +// dropped. This is only used for UDS and named pipes connection. +func WithWriteTimeout(writeTimeout time.Duration) Option { + return func(o *Options) error { + o.writeTimeout = writeTimeout + return nil + } +} + +// WithConnectTimeout sets the timeout for network connection with the Agent, after this interval the connection +// attempt is aborted. This is only used for UDS connection. This will also reset the connection if nothing can be +// written to it for this duration. +func WithConnectTimeout(connectTimeout time.Duration) Option { + return func(o *Options) error { + o.connectTimeout = connectTimeout + return nil + } +} + +// WithChannelMode make the client use channels to receive metrics +// +// This determines how the client receive metrics from the app (for example when calling the `Gauge()` method). +// The client will either drop the metrics if its buffers are full (WithChannelMode option) or block the caller until the +// metric can be handled (WithMutexMode option). By default, the client use mutexes. +// +// WithChannelMode uses a channel (see WithChannelModeBufferSize to configure its size) to receive metrics and drops metrics if +// the channel is full. Sending metrics in this mode is much slower that WithMutexMode (because of the channel), but will not +// block the application. This mode is made for application using statsd directly into the application code instead of +// a separated periodic reporter. The goal is to not slow down the application at the cost of dropping metrics and having a lower max +// throughput. +func WithChannelMode() Option { + return func(o *Options) error { + o.receiveMode = channelMode + return nil + } +} + +// WithMutexMode will use mutex to receive metrics from the app through the API. +// +// This determines how the client receive metrics from the app (for example when calling the `Gauge()` method). +// The client will either drop the metrics if its buffers are full (WithChannelMode option) or block the caller until the +// metric can be handled (WithMutexMode option). By default the client use mutexes. +// +// WithMutexMode uses mutexes to receive metrics which is much faster than channels but can cause some lock contention +// when used with a high number of goroutines sending the same metrics. Mutexes are sharded based on the metrics name +// which limit mutex contention when multiple goroutines send different metrics (see WithWorkersCount). This is the +// default behavior which will produce the best throughput. +func WithMutexMode() Option { + return func(o *Options) error { + o.receiveMode = mutexMode + return nil + } +} + +// WithChannelModeBufferSize sets the size of the channel holding incoming metrics when WithChannelMode is used. +func WithChannelModeBufferSize(bufferSize int) Option { + return func(o *Options) error { + o.channelModeBufferSize = bufferSize + return nil + } +} + +// WithChannelModeErrorsWhenFull makes the client return an error when the channel is full. +// This should be enabled if you want to be notified when the client is dropping metrics. You +// will also need to set `WithErrorHandler` to be notified of sender error. This might have +// a small performance impact. +func WithChannelModeErrorsWhenFull() Option { + return func(o *Options) error { + o.channelModeErrorsWhenFull = true + return nil + } +} + +// WithoutChannelModeErrorsWhenFull makes the client not return an error when the channel is full. +func WithoutChannelModeErrorsWhenFull() Option { + return func(o *Options) error { + o.channelModeErrorsWhenFull = false + return nil + } +} + +// WithErrorHandler sets a function that will be called when an error occurs. +func WithErrorHandler(errorHandler ErrorHandler) Option { + return func(o *Options) error { + o.errorHandler = errorHandler + return nil + } +} + +// WithAggregationInterval sets the interval at which aggregated metrics are flushed. See WithClientSideAggregation and +// WithExtendedClientSideAggregation for more. +// +// The default interval is 2s. The interval must divide the Agent reporting period (default=10s) evenly to reduce "aliasing" +// that can cause values to appear irregular/spiky. +// +// For example a 3s aggregation interval will create spikes in the final graph: a application sending a count metric +// that increments at a constant 1000 time per second will appear noisy with an interval of 3s. This is because +// client-side aggregation would report every 3 seconds, while the agent is reporting every 10 seconds. This means in +// each agent bucket, the values are: 9000, 9000, 12000. +func WithAggregationInterval(interval time.Duration) Option { + return func(o *Options) error { + o.aggregationFlushInterval = interval + return nil + } +} + +// WithClientSideAggregation enables client side aggregation for Gauges, Counts and Sets. +func WithClientSideAggregation() Option { + return func(o *Options) error { + o.aggregation = true + return nil + } +} + +// WithoutClientSideAggregation disables client side aggregation. +func WithoutClientSideAggregation() Option { + return func(o *Options) error { + o.aggregation = false + o.extendedAggregation = false + return nil + } +} + +// WithExtendedClientSideAggregation enables client side aggregation for all types. This feature is only compatible with +// Agent's version >=6.25.0 && <7.0.0 or Agent's versions >=7.25.0. +// When enabled, the use of `rate` with distribution is discouraged and `WithMaxSamplesPerContext()` should be used. +// If `rate` is used with different values of `rate` the resulting rate is not guaranteed to be correct. +func WithExtendedClientSideAggregation() Option { + return func(o *Options) error { + o.aggregation = true + o.extendedAggregation = true + return nil + } +} + +// WithMaxSamplesPerContext limits the number of sample for metric types that require multiple samples to be send +// over statsd to the agent, such as distributions or timings. This limits the number of sample per +// context for a distribution to a given number. Gauges and counts will not be affected as a single sample per context +// is sent with client side aggregation. +// - This will enable client side aggregation for all metrics. +// - This feature should be used with `WithExtendedClientSideAggregation` for optimal results. +func WithMaxSamplesPerContext(maxSamplesPerDistribution int) Option { + return func(o *Options) error { + o.aggregation = true + o.maxBufferedSamplesPerContext = maxSamplesPerDistribution + return nil + } +} + +// WithoutTelemetry disables the client telemetry. +// +// More on this here: https://docs.datadoghq.com/developers/dogstatsd/high_throughput/#client-side-telemetry +func WithoutTelemetry() Option { + return func(o *Options) error { + o.telemetry = false + return nil + } +} + +// WithTelemetryAddr sets a different address for telemetry metrics. By default the same address as the client is used +// for telemetry. +// +// More on this here: https://docs.datadoghq.com/developers/dogstatsd/high_throughput/#client-side-telemetry +func WithTelemetryAddr(addr string) Option { + return func(o *Options) error { + o.telemetryAddr = addr + return nil + } +} + +// WithoutOriginDetection disables the client origin detection. +// When enabled, the client tries to discover its container ID and sends it to the Agent +// to enrich the metrics with container tags. +// If the container id is not found and the client is running in a private cgroup namespace, the client +// sends the base cgroup controller inode. +// Origin detection can also be disabled by configuring the environment variabe DD_ORIGIN_DETECTION_ENABLED=false +// The client tries to read the container ID by parsing the file /proc/self/cgroup, this is not supported on Windows. +// +// More on this here: https://docs.datadoghq.com/developers/dogstatsd/?tab=kubernetes#origin-detection-over-udp +func WithoutOriginDetection() Option { + return func(o *Options) error { + o.originDetection = false + return nil + } +} + +// WithOriginDetection enables the client origin detection. +// This feature requires Datadog Agent version >=6.35.0 && <7.0.0 or Agent versions >=7.35.0. +// When enabled, the client tries to discover its container ID and sends it to the Agent +// to enrich the metrics with container tags. +// If the container id is not found and the client is running in a private cgroup namespace, the client +// sends the base cgroup controller inode. +// Origin detection can be disabled by configuring the environment variable DD_ORIGIN_DETECTION_ENABLED=false +// +// More on this here: https://docs.datadoghq.com/developers/dogstatsd/?tab=kubernetes#origin-detection-over-udp +func WithOriginDetection() Option { + return func(o *Options) error { + o.originDetection = true + return nil + } +} + +// WithContainerID allows passing the container ID, this will be used by the Agent to enrich metrics with container tags. +// This feature requires Datadog Agent version >=6.35.0 && <7.0.0 or Agent versions >=7.35.0. +// When configured, the provided container ID is prioritized over the container ID discovered via Origin Detection. +// The client prioritizes the value passed via DD_ENTITY_ID (if set) over the container ID. +func WithContainerID(id string) Option { + return func(o *Options) error { + o.containerID = id + return nil + } +} diff --git a/vendor/github.com/DataDog/datadog-go/v5/statsd/pipe.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/pipe.go new file mode 100644 index 000000000..1188b00f3 --- /dev/null +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/pipe.go @@ -0,0 +1,13 @@ +//go:build !windows +// +build !windows + +package statsd + +import ( + "errors" + "time" +) + +func newWindowsPipeWriter(pipepath string, writeTimeout time.Duration) (Transport, error) { + return nil, errors.New("Windows Named Pipes are only supported on Windows") +} diff --git a/vendor/github.com/DataDog/datadog-go/statsd/pipe_windows.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/pipe_windows.go similarity index 82% rename from vendor/github.com/DataDog/datadog-go/statsd/pipe_windows.go rename to vendor/github.com/DataDog/datadog-go/v5/statsd/pipe_windows.go index f533b0248..c27434ccf 100644 --- a/vendor/github.com/DataDog/datadog-go/statsd/pipe_windows.go +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/pipe_windows.go @@ -1,3 +1,4 @@ +//go:build windows // +build windows package statsd @@ -10,8 +11,6 @@ import ( "github.com/Microsoft/go-winio" ) -const defaultPipeTimeout = 1 * time.Millisecond - type pipeWriter struct { mu sync.RWMutex conn net.Conn @@ -19,13 +18,6 @@ type pipeWriter struct { pipepath string } -func (p *pipeWriter) SetWriteTimeout(d time.Duration) error { - p.mu.Lock() - p.timeout = d - p.mu.Unlock() - return nil -} - func (p *pipeWriter) Write(data []byte) (n int, err error) { conn, err := p.ensureConnection() if err != nil { @@ -74,11 +66,16 @@ func (p *pipeWriter) Close() error { return p.conn.Close() } -func newWindowsPipeWriter(pipepath string) (*pipeWriter, error) { +// GetTransportName returns the name of the transport +func (p *pipeWriter) GetTransportName() string { + return writerWindowsPipe +} + +func newWindowsPipeWriter(pipepath string, writeTimeout time.Duration) (*pipeWriter, error) { // Defer connection establishment to first write return &pipeWriter{ conn: nil, - timeout: defaultPipeTimeout, + timeout: writeTimeout, pipepath: pipepath, }, nil } diff --git a/vendor/github.com/DataDog/datadog-go/v5/statsd/sender.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/sender.go new file mode 100644 index 000000000..fc80395c3 --- /dev/null +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/sender.go @@ -0,0 +1,145 @@ +package statsd + +import ( + "io" + "sync/atomic" +) + +// senderTelemetry contains telemetry about the health of the sender +type senderTelemetry struct { + totalPayloadsSent uint64 + totalPayloadsDroppedQueueFull uint64 + totalPayloadsDroppedWriter uint64 + totalBytesSent uint64 + totalBytesDroppedQueueFull uint64 + totalBytesDroppedWriter uint64 +} + +type Transport interface { + io.WriteCloser + + // GetTransportName returns the name of the transport + GetTransportName() string +} + +type sender struct { + transport Transport + pool *bufferPool + queue chan *statsdBuffer + telemetry *senderTelemetry + stop chan struct{} + flushSignal chan struct{} + errorHandler ErrorHandler +} + +type ErrorSenderChannelFull struct { + LostElements int + ChannelSize int + Msg string +} + +func (e *ErrorSenderChannelFull) Error() string { + return e.Msg +} + +func newSender(transport Transport, queueSize int, pool *bufferPool, errorHandler ErrorHandler) *sender { + sender := &sender{ + transport: transport, + pool: pool, + queue: make(chan *statsdBuffer, queueSize), + telemetry: &senderTelemetry{}, + stop: make(chan struct{}), + flushSignal: make(chan struct{}), + errorHandler: errorHandler, + } + + go sender.sendLoop() + return sender +} + +func (s *sender) send(buffer *statsdBuffer) { + select { + case s.queue <- buffer: + default: + if s.errorHandler != nil { + err := &ErrorSenderChannelFull{ + LostElements: buffer.elementCount, + ChannelSize: len(s.queue), + Msg: "Sender queue is full", + } + s.errorHandler(err) + } + atomic.AddUint64(&s.telemetry.totalPayloadsDroppedQueueFull, 1) + atomic.AddUint64(&s.telemetry.totalBytesDroppedQueueFull, uint64(len(buffer.bytes()))) + s.pool.returnBuffer(buffer) + } +} + +func (s *sender) write(buffer *statsdBuffer) { + _, err := s.transport.Write(buffer.bytes()) + if err != nil { + atomic.AddUint64(&s.telemetry.totalPayloadsDroppedWriter, 1) + atomic.AddUint64(&s.telemetry.totalBytesDroppedWriter, uint64(len(buffer.bytes()))) + if s.errorHandler != nil { + s.errorHandler(err) + } + } else { + atomic.AddUint64(&s.telemetry.totalPayloadsSent, 1) + atomic.AddUint64(&s.telemetry.totalBytesSent, uint64(len(buffer.bytes()))) + } + s.pool.returnBuffer(buffer) +} + +func (s *sender) flushTelemetryMetrics(t *Telemetry) { + t.TotalPayloadsSent = atomic.LoadUint64(&s.telemetry.totalPayloadsSent) + t.TotalPayloadsDroppedQueueFull = atomic.LoadUint64(&s.telemetry.totalPayloadsDroppedQueueFull) + t.TotalPayloadsDroppedWriter = atomic.LoadUint64(&s.telemetry.totalPayloadsDroppedWriter) + + t.TotalBytesSent = atomic.LoadUint64(&s.telemetry.totalBytesSent) + t.TotalBytesDroppedQueueFull = atomic.LoadUint64(&s.telemetry.totalBytesDroppedQueueFull) + t.TotalBytesDroppedWriter = atomic.LoadUint64(&s.telemetry.totalBytesDroppedWriter) +} + +func (s *sender) sendLoop() { + defer close(s.stop) + for { + select { + case buffer := <-s.queue: + s.write(buffer) + case <-s.stop: + return + case <-s.flushSignal: + // At that point we know that the workers are paused (the statsd client + // will pause them before calling sender.flush()). + // So we can fully flush the input queue + s.flushInputQueue() + s.flushSignal <- struct{}{} + } + } +} + +func (s *sender) flushInputQueue() { + for { + select { + case buffer := <-s.queue: + s.write(buffer) + default: + return + } + } +} +func (s *sender) flush() { + s.flushSignal <- struct{}{} + <-s.flushSignal +} + +func (s *sender) close() error { + s.stop <- struct{}{} + <-s.stop + s.flushInputQueue() + return s.transport.Close() +} + +func (s *sender) getTransportName() string { + return s.transport.GetTransportName() +} diff --git a/vendor/github.com/DataDog/datadog-go/statsd/service_check.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/service_check.go similarity index 76% rename from vendor/github.com/DataDog/datadog-go/statsd/service_check.go rename to vendor/github.com/DataDog/datadog-go/v5/statsd/service_check.go index fce86755f..e2850465c 100644 --- a/vendor/github.com/DataDog/datadog-go/statsd/service_check.go +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/service_check.go @@ -46,7 +46,7 @@ func NewServiceCheck(name string, status ServiceCheckStatus) *ServiceCheck { } // Check verifies that a service check is valid. -func (sc ServiceCheck) Check() error { +func (sc *ServiceCheck) Check() error { if len(sc.Name) == 0 { return fmt.Errorf("statsd.ServiceCheck name is required") } @@ -55,16 +55,3 @@ func (sc ServiceCheck) Check() error { } return nil } - -// Encode returns the dogstatsd wire protocol representation for a service check. -// Tags may be passed which will be added to the encoded output but not to -// the Service Check's list of tags, eg. for default tags. -func (sc ServiceCheck) Encode(tags ...string) (string, error) { - err := sc.Check() - if err != nil { - return "", err - } - var buffer []byte - buffer = appendServiceCheck(buffer, sc, tags) - return string(buffer), nil -} diff --git a/vendor/github.com/DataDog/datadog-go/v5/statsd/statsd.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/statsd.go new file mode 100644 index 000000000..c0137b523 --- /dev/null +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/statsd.go @@ -0,0 +1,907 @@ +// Copyright 2013 Ooyala, Inc. + +/* +Package statsd provides a Go dogstatsd client. Dogstatsd extends the popular statsd, +adding tags and histograms and pushing upstream to Datadog. + +Refer to http://docs.datadoghq.com/guides/dogstatsd/ for information about DogStatsD. + +statsd is based on go-statsd-client. +*/ +package statsd + +//go:generate mockgen -source=statsd.go -destination=mocks/statsd.go + +import ( + "errors" + "fmt" + "io" + "net/url" + "os" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +/* +OptimalUDPPayloadSize defines the optimal payload size for a UDP datagram, 1432 bytes +is optimal for regular networks with an MTU of 1500 so datagrams don't get +fragmented. It's generally recommended not to fragment UDP datagrams as losing +a single fragment will cause the entire datagram to be lost. +*/ +const OptimalUDPPayloadSize = 1432 + +/* +MaxUDPPayloadSize defines the maximum payload size for a UDP datagram. +Its value comes from the calculation: 65535 bytes Max UDP datagram size - +8byte UDP header - 60byte max IP headers +any number greater than that will see frames being cut out. +*/ +const MaxUDPPayloadSize = 65467 + +// DefaultUDPBufferPoolSize is the default size of the buffer pool for UDP clients. +const DefaultUDPBufferPoolSize = 2048 + +// DefaultUDSBufferPoolSize is the default size of the buffer pool for UDS clients. +const DefaultUDSBufferPoolSize = 512 + +/* +DefaultMaxAgentPayloadSize is the default maximum payload size the agent +can receive. This can be adjusted by changing dogstatsd_buffer_size in the +agent configuration file datadog.yaml. This is also used as the optimal payload size +for UDS datagrams. +*/ +const DefaultMaxAgentPayloadSize = 8192 + +/* +UnixAddressPrefix holds the prefix to use to enable Unix Domain Socket +traffic instead of UDP. The type of the socket will be guessed. +*/ +const UnixAddressPrefix = "unix://" + +/* +UnixDatagramAddressPrefix holds the prefix to use to enable Unix Domain Socket +datagram traffic instead of UDP. +*/ +const UnixAddressDatagramPrefix = "unixgram://" + +/* +UnixAddressStreamPrefix holds the prefix to use to enable Unix Domain Socket +stream traffic instead of UDP. +*/ +const UnixAddressStreamPrefix = "unixstream://" + +/* +WindowsPipeAddressPrefix holds the prefix to use to enable Windows Named Pipes +traffic instead of UDP. +*/ +const WindowsPipeAddressPrefix = `\\.\pipe\` + +var ( + AddressPrefixes = []string{UnixAddressPrefix, UnixAddressDatagramPrefix, UnixAddressStreamPrefix, WindowsPipeAddressPrefix} +) + +const ( + agentHostEnvVarName = "DD_AGENT_HOST" + agentPortEnvVarName = "DD_DOGSTATSD_PORT" + agentURLEnvVarName = "DD_DOGSTATSD_URL" + defaultUDPPort = "8125" +) + +const ( + // ddEntityID specifies client-side user-specified entity ID injection. + // This env var can be set to the Pod UID on Kubernetes via the downward API. + // Docs: https://docs.datadoghq.com/developers/dogstatsd/?tab=kubernetes#origin-detection-over-udp + ddEntityID = "DD_ENTITY_ID" + + // ddEntityIDTag specifies the tag name for the client-side entity ID injection + // The Agent expects this tag to contain a non-prefixed Kubernetes Pod UID. + ddEntityIDTag = "dd.internal.entity_id" + + // originDetectionEnabled specifies the env var to enable/disable sending the container ID field. + originDetectionEnabled = "DD_ORIGIN_DETECTION_ENABLED" +) + +/* +ddEnvTagsMapping is a mapping of each "DD_" prefixed environment variable +to a specific tag name. We use a slice to keep the order and simplify tests. +*/ +var ddEnvTagsMapping = []struct{ envName, tagName string }{ + {ddEntityID, ddEntityIDTag}, // Client-side entity ID injection for container tagging. + {"DD_ENV", "env"}, // The name of the env in which the service runs. + {"DD_SERVICE", "service"}, // The name of the running service. + {"DD_VERSION", "version"}, // The current version of the running service. +} + +type metricType int + +const ( + gauge metricType = iota + count + histogram + histogramAggregated + distribution + distributionAggregated + set + timing + timingAggregated + event + serviceCheck +) + +type receivingMode int + +const ( + mutexMode receivingMode = iota + channelMode +) + +const ( + writerNameUDP string = "udp" + writerNameUDS string = "uds" + writerNameUDSStream string = "uds-stream" + writerWindowsPipe string = "pipe" + writerNameCustom string = "custom" +) + +// noTimestamp is used as a value for metric without a given timestamp. +const noTimestamp = int64(0) + +type metric struct { + metricType metricType + namespace string + globalTags []string + name string + fvalue float64 + fvalues []float64 + ivalue int64 + svalue string + evalue *Event + scvalue *ServiceCheck + tags []string + stags string + rate float64 + timestamp int64 +} + +type noClientErr string + +// ErrNoClient is returned if statsd reporting methods are invoked on +// a nil client. +const ErrNoClient = noClientErr("statsd client is nil") + +func (e noClientErr) Error() string { + return string(e) +} + +type invalidTimestampErr string + +// InvalidTimestamp is returned if a provided timestamp is invalid. +const InvalidTimestamp = invalidTimestampErr("invalid timestamp") + +func (e invalidTimestampErr) Error() string { + return string(e) +} + +// ClientInterface is an interface that exposes the common client functions for the +// purpose of being able to provide a no-op client or even mocking. This can aid +// downstream users' with their testing. +type ClientInterface interface { + // Gauge measures the value of a metric at a particular time. + Gauge(name string, value float64, tags []string, rate float64) error + + // GaugeWithTimestamp measures the value of a metric at a given time. + // BETA - Please contact our support team for more information to use this feature: https://www.datadoghq.com/support/ + // The value will bypass any aggregation on the client side and agent side, this is + // useful when sending points in the past. + // + // Minimum Datadog Agent version: 7.40.0 + GaugeWithTimestamp(name string, value float64, tags []string, rate float64, timestamp time.Time) error + + // Count tracks how many times something happened per second. + Count(name string, value int64, tags []string, rate float64) error + + // CountWithTimestamp tracks how many times something happened at the given second. + // BETA - Please contact our support team for more information to use this feature: https://www.datadoghq.com/support/ + // The value will bypass any aggregation on the client side and agent side, this is + // useful when sending points in the past. + // + // Minimum Datadog Agent version: 7.40.0 + CountWithTimestamp(name string, value int64, tags []string, rate float64, timestamp time.Time) error + + // Histogram tracks the statistical distribution of a set of values on each host. + Histogram(name string, value float64, tags []string, rate float64) error + + // Distribution tracks the statistical distribution of a set of values across your infrastructure. + // + // It is recommended to use `WithMaxBufferedMetricsPerContext` to avoid dropping metrics at high throughput, `rate` can + // also be used to limit the load. Both options can *not* be used together. + Distribution(name string, value float64, tags []string, rate float64) error + + // Decr is just Count of -1 + Decr(name string, tags []string, rate float64) error + + // Incr is just Count of 1 + Incr(name string, tags []string, rate float64) error + + // Set counts the number of unique elements in a group. + Set(name string, value string, tags []string, rate float64) error + + // Timing sends timing information, it is an alias for TimeInMilliseconds + Timing(name string, value time.Duration, tags []string, rate float64) error + + // TimeInMilliseconds sends timing information in milliseconds. + // It is flushed by statsd with percentiles, mean and other info (https://github.com/etsy/statsd/blob/master/docs/metric_types.md#timing) + TimeInMilliseconds(name string, value float64, tags []string, rate float64) error + + // Event sends the provided Event. + Event(e *Event) error + + // SimpleEvent sends an event with the provided title and text. + SimpleEvent(title, text string) error + + // ServiceCheck sends the provided ServiceCheck. + ServiceCheck(sc *ServiceCheck) error + + // SimpleServiceCheck sends an serviceCheck with the provided name and status. + SimpleServiceCheck(name string, status ServiceCheckStatus) error + + // Close the client connection. + Close() error + + // Flush forces a flush of all the queued dogstatsd payloads. + Flush() error + + // IsClosed returns if the client has been closed. + IsClosed() bool + + // GetTelemetry return the telemetry metrics for the client since it started. + GetTelemetry() Telemetry +} + +type ErrorHandler func(error) + +// A Client is a handle for sending messages to dogstatsd. It is safe to +// use one Client from multiple goroutines simultaneously. +type Client struct { + // Sender handles the underlying networking protocol + sender *sender + // namespace to prepend to all statsd calls + namespace string + // tags are global tags to be added to every statsd call + tags []string + flushTime time.Duration + telemetry *statsdTelemetry + telemetryClient *telemetryClient + stop chan struct{} + wg sync.WaitGroup + workers []*worker + closerLock sync.Mutex + workersMode receivingMode + aggregatorMode receivingMode + agg *aggregator + aggExtended *aggregator + options []Option + addrOption string + isClosed bool + errorOnBlockedChannel bool + errorHandler ErrorHandler +} + +// statsdTelemetry contains telemetry metrics about the client +type statsdTelemetry struct { + totalMetricsGauge uint64 + totalMetricsCount uint64 + totalMetricsHistogram uint64 + totalMetricsDistribution uint64 + totalMetricsSet uint64 + totalMetricsTiming uint64 + totalEvents uint64 + totalServiceChecks uint64 + totalDroppedOnReceive uint64 +} + +// Verify that Client implements the ClientInterface. +// https://golang.org/doc/faq#guarantee_satisfies_interface +var _ ClientInterface = &Client{} + +func resolveAddr(addr string) string { + envPort := "" + + if addr == "" { + addr = os.Getenv(agentHostEnvVarName) + envPort = os.Getenv(agentPortEnvVarName) + agentURL, _ := os.LookupEnv(agentURLEnvVarName) + agentURL = parseAgentURL(agentURL) + + // agentURLEnvVarName has priority over agentHostEnvVarName + if agentURL != "" { + return agentURL + } + } + + if addr == "" { + return "" + } + + for _, prefix := range AddressPrefixes { + if strings.HasPrefix(addr, prefix) { + return addr + } + } + // TODO: How does this work for IPv6? + if strings.Contains(addr, ":") { + return addr + } + if envPort != "" { + addr = fmt.Sprintf("%s:%s", addr, envPort) + } else { + addr = fmt.Sprintf("%s:%s", addr, defaultUDPPort) + } + return addr +} + +func parseAgentURL(agentURL string) string { + if agentURL != "" { + if strings.HasPrefix(agentURL, WindowsPipeAddressPrefix) { + return agentURL + } + + parsedURL, err := url.Parse(agentURL) + if err != nil { + return "" + } + + if parsedURL.Scheme == "udp" { + if strings.Contains(parsedURL.Host, ":") { + return parsedURL.Host + } + return fmt.Sprintf("%s:%s", parsedURL.Host, defaultUDPPort) + } + + if parsedURL.Scheme == "unix" { + return agentURL + } + } + return "" +} + +func createWriter(addr string, writeTimeout time.Duration, connectTimeout time.Duration) (Transport, string, error) { + if addr == "" { + return nil, "", errors.New("No address passed and autodetection from environment failed") + } + + switch { + case strings.HasPrefix(addr, WindowsPipeAddressPrefix): + w, err := newWindowsPipeWriter(addr, writeTimeout) + return w, writerWindowsPipe, err + case strings.HasPrefix(addr, UnixAddressPrefix): + w, err := newUDSWriter(addr[len(UnixAddressPrefix):], writeTimeout, connectTimeout, "") + return w, writerNameUDS, err + case strings.HasPrefix(addr, UnixAddressDatagramPrefix): + w, err := newUDSWriter(addr[len(UnixAddressDatagramPrefix):], writeTimeout, connectTimeout, "unixgram") + return w, writerNameUDS, err + case strings.HasPrefix(addr, UnixAddressStreamPrefix): + w, err := newUDSWriter(addr[len(UnixAddressStreamPrefix):], writeTimeout, connectTimeout, "unix") + return w, writerNameUDS, err + default: + w, err := newUDPWriter(addr, writeTimeout) + return w, writerNameUDP, err + } +} + +// New returns a pointer to a new Client given an addr in the format "hostname:port" for UDP, +// "unix:///path/to/socket" for UDS or "\\.\pipe\path\to\pipe" for Windows Named Pipes. +func New(addr string, options ...Option) (*Client, error) { + o, err := resolveOptions(options) + if err != nil { + return nil, err + } + + addr = resolveAddr(addr) + w, writerType, err := createWriter(addr, o.writeTimeout, o.connectTimeout) + if err != nil { + return nil, err + } + + client, err := newWithWriter(w, o, writerType) + if err == nil { + client.options = append(client.options, options...) + client.addrOption = addr + } + return client, err +} + +type customWriter struct { + io.WriteCloser +} + +func (w *customWriter) GetTransportName() string { + return writerNameCustom +} + +// NewWithWriter creates a new Client with given writer. Writer is a +// io.WriteCloser +func NewWithWriter(w io.WriteCloser, options ...Option) (*Client, error) { + o, err := resolveOptions(options) + if err != nil { + return nil, err + } + return newWithWriter(&customWriter{w}, o, writerNameCustom) +} + +// CloneWithExtraOptions create a new Client with extra options +func CloneWithExtraOptions(c *Client, options ...Option) (*Client, error) { + if c == nil { + return nil, ErrNoClient + } + + if c.addrOption == "" { + return nil, fmt.Errorf("can't clone client with no addrOption") + } + opt := append(c.options, options...) + return New(c.addrOption, opt...) +} + +func newWithWriter(w Transport, o *Options, writerName string) (*Client, error) { + c := Client{ + namespace: o.namespace, + tags: o.tags, + telemetry: &statsdTelemetry{}, + errorOnBlockedChannel: o.channelModeErrorsWhenFull, + errorHandler: o.errorHandler, + } + + // Inject values of DD_* environment variables as global tags. + for _, mapping := range ddEnvTagsMapping { + if value := os.Getenv(mapping.envName); value != "" { + c.tags = append(c.tags, fmt.Sprintf("%s:%s", mapping.tagName, value)) + } + } + + initContainerID(o.containerID, isOriginDetectionEnabled(o), isHostCgroupNamespace()) + isUDS := writerName == writerNameUDS + + if o.maxBytesPerPayload == 0 { + if isUDS { + o.maxBytesPerPayload = DefaultMaxAgentPayloadSize + } else { + o.maxBytesPerPayload = OptimalUDPPayloadSize + } + } + if o.bufferPoolSize == 0 { + if isUDS { + o.bufferPoolSize = DefaultUDSBufferPoolSize + } else { + o.bufferPoolSize = DefaultUDPBufferPoolSize + } + } + if o.senderQueueSize == 0 { + if isUDS { + o.senderQueueSize = DefaultUDSBufferPoolSize + } else { + o.senderQueueSize = DefaultUDPBufferPoolSize + } + } + + bufferPool := newBufferPool(o.bufferPoolSize, o.maxBytesPerPayload, o.maxMessagesPerPayload) + c.sender = newSender(w, o.senderQueueSize, bufferPool, o.errorHandler) + c.aggregatorMode = o.receiveMode + + c.workersMode = o.receiveMode + // channelMode mode at the worker level is not enabled when + // ExtendedAggregation is since the user app will not directly + // use the worker (the aggregator sit between the app and the + // workers). + if o.extendedAggregation { + c.workersMode = mutexMode + } + + if o.aggregation || o.extendedAggregation || o.maxBufferedSamplesPerContext > 0 { + c.agg = newAggregator(&c, int64(o.maxBufferedSamplesPerContext)) + c.agg.start(o.aggregationFlushInterval) + + if o.extendedAggregation { + c.aggExtended = c.agg + + if c.aggregatorMode == channelMode { + c.agg.startReceivingMetric(o.channelModeBufferSize, o.workersCount) + } + } + } + + for i := 0; i < o.workersCount; i++ { + w := newWorker(bufferPool, c.sender) + c.workers = append(c.workers, w) + + if c.workersMode == channelMode { + w.startReceivingMetric(o.channelModeBufferSize) + } + } + + c.flushTime = o.bufferFlushInterval + c.stop = make(chan struct{}, 1) + + c.wg.Add(1) + go func() { + defer c.wg.Done() + c.watch() + }() + + if o.telemetry { + if o.telemetryAddr == "" { + c.telemetryClient = newTelemetryClient(&c, c.agg != nil) + } else { + var err error + c.telemetryClient, err = newTelemetryClientWithCustomAddr(&c, o.telemetryAddr, c.agg != nil, bufferPool, o.writeTimeout, o.connectTimeout) + if err != nil { + return nil, err + } + } + c.telemetryClient.run(&c.wg, c.stop) + } + + return &c, nil +} + +func (c *Client) watch() { + ticker := time.NewTicker(c.flushTime) + + for { + select { + case <-ticker.C: + for _, w := range c.workers { + w.flush() + } + case <-c.stop: + ticker.Stop() + return + } + } +} + +// Flush forces a flush of all the queued dogstatsd payloads This method is +// blocking and will not return until everything is sent through the network. +// In mutexMode, this will also block sampling new data to the client while the +// workers and sender are flushed. +func (c *Client) Flush() error { + if c == nil { + return ErrNoClient + } + if c.agg != nil { + c.agg.flush() + } + for _, w := range c.workers { + w.pause() + defer w.unpause() + w.flushUnsafe() + } + // Now that the worker are pause the sender can flush the queue between + // worker and senders + c.sender.flush() + return nil +} + +// IsClosed returns if the client has been closed. +func (c *Client) IsClosed() bool { + c.closerLock.Lock() + defer c.closerLock.Unlock() + return c.isClosed +} + +func (c *Client) flushTelemetryMetrics(t *Telemetry) { + t.TotalMetricsGauge = atomic.LoadUint64(&c.telemetry.totalMetricsGauge) + t.TotalMetricsCount = atomic.LoadUint64(&c.telemetry.totalMetricsCount) + t.TotalMetricsSet = atomic.LoadUint64(&c.telemetry.totalMetricsSet) + t.TotalMetricsHistogram = atomic.LoadUint64(&c.telemetry.totalMetricsHistogram) + t.TotalMetricsDistribution = atomic.LoadUint64(&c.telemetry.totalMetricsDistribution) + t.TotalMetricsTiming = atomic.LoadUint64(&c.telemetry.totalMetricsTiming) + t.TotalEvents = atomic.LoadUint64(&c.telemetry.totalEvents) + t.TotalServiceChecks = atomic.LoadUint64(&c.telemetry.totalServiceChecks) + t.TotalDroppedOnReceive = atomic.LoadUint64(&c.telemetry.totalDroppedOnReceive) +} + +// GetTelemetry return the telemetry metrics for the client since it started. +func (c *Client) GetTelemetry() Telemetry { + return c.telemetryClient.getTelemetry() +} + +// GetTransport return the name of the transport used. +func (c *Client) GetTransport() string { + if c.sender == nil { + return "" + } + return c.sender.getTransportName() +} + +type ErrorInputChannelFull struct { + Metric metric + ChannelSize int + Msg string +} + +func (e ErrorInputChannelFull) Error() string { + return e.Msg +} + +func (c *Client) send(m metric) error { + h := hashString32(m.name) + worker := c.workers[h%uint32(len(c.workers))] + + if c.workersMode == channelMode { + select { + case worker.inputMetrics <- m: + default: + atomic.AddUint64(&c.telemetry.totalDroppedOnReceive, 1) + err := &ErrorInputChannelFull{m, len(worker.inputMetrics), "Worker input channel full"} + if c.errorHandler != nil { + c.errorHandler(err) + } + if c.errorOnBlockedChannel { + return err + } + } + return nil + } + return worker.processMetric(m) +} + +// sendBlocking is used by the aggregator to inject aggregated metrics. +func (c *Client) sendBlocking(m metric) error { + m.globalTags = c.tags + m.namespace = c.namespace + + h := hashString32(m.name) + worker := c.workers[h%uint32(len(c.workers))] + return worker.processMetric(m) +} + +func (c *Client) sendToAggregator(mType metricType, name string, value float64, tags []string, rate float64, f bufferedMetricSampleFunc) error { + if c.aggregatorMode == channelMode { + m := metric{metricType: mType, name: name, fvalue: value, tags: tags, rate: rate} + select { + case c.aggExtended.inputMetrics <- m: + default: + atomic.AddUint64(&c.telemetry.totalDroppedOnReceive, 1) + err := &ErrorInputChannelFull{m, len(c.aggExtended.inputMetrics), "Aggregator input channel full"} + if c.errorHandler != nil { + c.errorHandler(err) + } + if c.errorOnBlockedChannel { + return err + } + } + return nil + } + return f(name, value, tags, rate) +} + +// Gauge measures the value of a metric at a particular time. +func (c *Client) Gauge(name string, value float64, tags []string, rate float64) error { + if c == nil { + return ErrNoClient + } + atomic.AddUint64(&c.telemetry.totalMetricsGauge, 1) + if c.agg != nil { + return c.agg.gauge(name, value, tags) + } + return c.send(metric{metricType: gauge, name: name, fvalue: value, tags: tags, rate: rate, globalTags: c.tags, namespace: c.namespace}) +} + +// GaugeWithTimestamp measures the value of a metric at a given time. +// BETA - Please contact our support team for more information to use this feature: https://www.datadoghq.com/support/ +// The value will bypass any aggregation on the client side and agent side, this is +// useful when sending points in the past. +// +// Minimum Datadog Agent version: 7.40.0 +func (c *Client) GaugeWithTimestamp(name string, value float64, tags []string, rate float64, timestamp time.Time) error { + if c == nil { + return ErrNoClient + } + + if timestamp.IsZero() || timestamp.Unix() <= noTimestamp { + return InvalidTimestamp + } + + atomic.AddUint64(&c.telemetry.totalMetricsGauge, 1) + return c.send(metric{metricType: gauge, name: name, fvalue: value, tags: tags, rate: rate, globalTags: c.tags, namespace: c.namespace, timestamp: timestamp.Unix()}) +} + +// Count tracks how many times something happened per second. +func (c *Client) Count(name string, value int64, tags []string, rate float64) error { + if c == nil { + return ErrNoClient + } + atomic.AddUint64(&c.telemetry.totalMetricsCount, 1) + if c.agg != nil { + return c.agg.count(name, value, tags) + } + return c.send(metric{metricType: count, name: name, ivalue: value, tags: tags, rate: rate, globalTags: c.tags, namespace: c.namespace}) +} + +// CountWithTimestamp tracks how many times something happened at the given second. +// BETA - Please contact our support team for more information to use this feature: https://www.datadoghq.com/support/ +// The value will bypass any aggregation on the client side and agent side, this is +// useful when sending points in the past. +// +// Minimum Datadog Agent version: 7.40.0 +func (c *Client) CountWithTimestamp(name string, value int64, tags []string, rate float64, timestamp time.Time) error { + if c == nil { + return ErrNoClient + } + + if timestamp.IsZero() || timestamp.Unix() <= noTimestamp { + return InvalidTimestamp + } + + atomic.AddUint64(&c.telemetry.totalMetricsCount, 1) + return c.send(metric{metricType: count, name: name, ivalue: value, tags: tags, rate: rate, globalTags: c.tags, namespace: c.namespace, timestamp: timestamp.Unix()}) +} + +// Histogram tracks the statistical distribution of a set of values on each host. +func (c *Client) Histogram(name string, value float64, tags []string, rate float64) error { + if c == nil { + return ErrNoClient + } + atomic.AddUint64(&c.telemetry.totalMetricsHistogram, 1) + if c.aggExtended != nil { + return c.sendToAggregator(histogram, name, value, tags, rate, c.aggExtended.histogram) + } + return c.send(metric{metricType: histogram, name: name, fvalue: value, tags: tags, rate: rate, globalTags: c.tags, namespace: c.namespace}) +} + +// Distribution tracks the statistical distribution of a set of values across your infrastructure. +func (c *Client) Distribution(name string, value float64, tags []string, rate float64) error { + if c == nil { + return ErrNoClient + } + atomic.AddUint64(&c.telemetry.totalMetricsDistribution, 1) + if c.aggExtended != nil { + return c.sendToAggregator(distribution, name, value, tags, rate, c.aggExtended.distribution) + } + return c.send(metric{metricType: distribution, name: name, fvalue: value, tags: tags, rate: rate, globalTags: c.tags, namespace: c.namespace}) +} + +// Decr is just Count of -1 +func (c *Client) Decr(name string, tags []string, rate float64) error { + return c.Count(name, -1, tags, rate) +} + +// Incr is just Count of 1 +func (c *Client) Incr(name string, tags []string, rate float64) error { + return c.Count(name, 1, tags, rate) +} + +// Set counts the number of unique elements in a group. +func (c *Client) Set(name string, value string, tags []string, rate float64) error { + if c == nil { + return ErrNoClient + } + atomic.AddUint64(&c.telemetry.totalMetricsSet, 1) + if c.agg != nil { + return c.agg.set(name, value, tags) + } + return c.send(metric{metricType: set, name: name, svalue: value, tags: tags, rate: rate, globalTags: c.tags, namespace: c.namespace}) +} + +// Timing sends timing information, it is an alias for TimeInMilliseconds +func (c *Client) Timing(name string, value time.Duration, tags []string, rate float64) error { + return c.TimeInMilliseconds(name, value.Seconds()*1000, tags, rate) +} + +// TimeInMilliseconds sends timing information in milliseconds. +// It is flushed by statsd with percentiles, mean and other info (https://github.com/etsy/statsd/blob/master/docs/metric_types.md#timing) +func (c *Client) TimeInMilliseconds(name string, value float64, tags []string, rate float64) error { + if c == nil { + return ErrNoClient + } + atomic.AddUint64(&c.telemetry.totalMetricsTiming, 1) + if c.aggExtended != nil { + return c.sendToAggregator(timing, name, value, tags, rate, c.aggExtended.timing) + } + return c.send(metric{metricType: timing, name: name, fvalue: value, tags: tags, rate: rate, globalTags: c.tags, namespace: c.namespace}) +} + +// Event sends the provided Event. +func (c *Client) Event(e *Event) error { + if c == nil { + return ErrNoClient + } + atomic.AddUint64(&c.telemetry.totalEvents, 1) + return c.send(metric{metricType: event, evalue: e, rate: 1, globalTags: c.tags, namespace: c.namespace}) +} + +// SimpleEvent sends an event with the provided title and text. +func (c *Client) SimpleEvent(title, text string) error { + e := NewEvent(title, text) + return c.Event(e) +} + +// ServiceCheck sends the provided ServiceCheck. +func (c *Client) ServiceCheck(sc *ServiceCheck) error { + if c == nil { + return ErrNoClient + } + atomic.AddUint64(&c.telemetry.totalServiceChecks, 1) + return c.send(metric{metricType: serviceCheck, scvalue: sc, rate: 1, globalTags: c.tags, namespace: c.namespace}) +} + +// SimpleServiceCheck sends an serviceCheck with the provided name and status. +func (c *Client) SimpleServiceCheck(name string, status ServiceCheckStatus) error { + sc := NewServiceCheck(name, status) + return c.ServiceCheck(sc) +} + +// Close the client connection. +func (c *Client) Close() error { + if c == nil { + return ErrNoClient + } + + // Acquire closer lock to ensure only one thread can close the stop channel + c.closerLock.Lock() + defer c.closerLock.Unlock() + + if c.isClosed { + return nil + } + + // Notify all other threads that they should stop + select { + case <-c.stop: + return nil + default: + } + close(c.stop) + + if c.workersMode == channelMode { + for _, w := range c.workers { + w.stopReceivingMetric() + } + } + + // flush the aggregator first + if c.agg != nil { + if c.aggExtended != nil && c.aggregatorMode == channelMode { + c.agg.stopReceivingMetric() + } + c.agg.stop() + } + + // Wait for the threads to stop + c.wg.Wait() + + c.Flush() + + c.isClosed = true + return c.sender.close() +} + +// isOriginDetectionEnabled returns whether the clients should fill the container field. +// +// Disable origin detection only in one of the following cases: +// - DD_ORIGIN_DETECTION_ENABLED is explicitly set to false +// - o.originDetection is explicitly set to false, which is true by default +func isOriginDetectionEnabled(o *Options) bool { + if !o.originDetection || o.containerID != "" { + return false + } + + envVarValue := os.Getenv(originDetectionEnabled) + if envVarValue == "" { + // DD_ORIGIN_DETECTION_ENABLED is not set + // default to true + return true + } + + enabled, err := strconv.ParseBool(envVarValue) + if err != nil { + // Error due to an unsupported DD_ORIGIN_DETECTION_ENABLED value + // default to true + return true + } + + return enabled +} diff --git a/vendor/github.com/DataDog/datadog-go/v5/statsd/statsd_direct.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/statsd_direct.go new file mode 100644 index 000000000..af66517cb --- /dev/null +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/statsd_direct.go @@ -0,0 +1,69 @@ +package statsd + +import ( + "io" + "strings" + "sync/atomic" +) + +type ClientDirectInterface interface { + DistributionSamples(name string, values []float64, tags []string, rate float64) error +} + +// ClientDirect is an *experimental* statsd client that gives direct access to some dogstatsd features. +// +// It is not recommended to use this client in production. This client might allow you to take advantage of +// new features in the agent before they are released, but it might also break your application. +type ClientDirect struct { + *Client +} + +// NewDirect returns a pointer to a new ClientDirect given an addr in the format "hostname:port" for UDP, +// "unix:///path/to/socket" for UDS or "\\.\pipe\path\to\pipe" for Windows Named Pipes. +func NewDirect(addr string, options ...Option) (*ClientDirect, error) { + client, err := New(addr, options...) + if err != nil { + return nil, err + } + return &ClientDirect{ + client, + }, nil +} + +func NewDirectWithWriter(writer io.WriteCloser, options ...Option) (*ClientDirect, error) { + client, err := NewWithWriter(writer, options...) + if err != nil { + return nil, err + } + return &ClientDirect{ + client, + }, nil +} + +// DistributionSamples is similar to Distribution, but it lets the client deals with the sampling. +// +// The provided `rate` is the sampling rate applied by the client and will *not* be used to apply further +// sampling. This is recommended in high performance cases were the overhead of the statsd library might be +// significant and the sampling is already done by the client. +// +// `WithMaxBufferedMetricsPerContext` is ignored when using this method. +func (c *ClientDirect) DistributionSamples(name string, values []float64, tags []string, rate float64) error { + if c == nil { + return ErrNoClient + } + atomic.AddUint64(&c.telemetry.totalMetricsDistribution, uint64(len(values))) + return c.send(metric{ + metricType: distributionAggregated, + name: name, + fvalues: values, + tags: tags, + stags: strings.Join(tags, tagSeparatorSymbol), + rate: rate, + globalTags: c.tags, + namespace: c.namespace, + }) +} + +// Validate that ClientDirect implements ClientDirectInterface and ClientInterface. +var _ ClientDirectInterface = (*ClientDirect)(nil) +var _ ClientInterface = (*ClientDirect)(nil) diff --git a/vendor/github.com/DataDog/datadog-go/v5/statsd/telemetry.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/telemetry.go new file mode 100644 index 000000000..feda764b5 --- /dev/null +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/telemetry.go @@ -0,0 +1,307 @@ +package statsd + +import ( + "fmt" + "sync" + "time" +) + +/* +telemetryInterval is the interval at which telemetry will be sent by the client. +*/ +const telemetryInterval = 10 * time.Second + +/* +clientTelemetryTag is a tag identifying this specific client. +*/ +var clientTelemetryTag = "client:go" + +/* +clientVersionTelemetryTag is a tag identifying this specific client version. +*/ +var clientVersionTelemetryTag = "client_version:5.4.0" + +// Telemetry represents internal metrics about the client behavior since it started. +type Telemetry struct { + // + // Those are produced by the 'Client' + // + + // TotalMetrics is the total number of metrics sent by the client before aggregation and sampling. + TotalMetrics uint64 + // TotalMetricsGauge is the total number of gauges sent by the client before aggregation and sampling. + TotalMetricsGauge uint64 + // TotalMetricsCount is the total number of counts sent by the client before aggregation and sampling. + TotalMetricsCount uint64 + // TotalMetricsHistogram is the total number of histograms sent by the client before aggregation and sampling. + TotalMetricsHistogram uint64 + // TotalMetricsDistribution is the total number of distributions sent by the client before aggregation and + // sampling. + TotalMetricsDistribution uint64 + // TotalMetricsSet is the total number of sets sent by the client before aggregation and sampling. + TotalMetricsSet uint64 + // TotalMetricsTiming is the total number of timings sent by the client before aggregation and sampling. + TotalMetricsTiming uint64 + // TotalEvents is the total number of events sent by the client before aggregation and sampling. + TotalEvents uint64 + // TotalServiceChecks is the total number of service_checks sent by the client before aggregation and sampling. + TotalServiceChecks uint64 + + // TotalDroppedOnReceive is the total number metrics/event/service_checks dropped when using ChannelMode (see + // WithChannelMode option). + TotalDroppedOnReceive uint64 + + // + // Those are produced by the 'sender' + // + + // TotalPayloadsSent is the total number of payload (packet on the network) succesfully sent by the client. When + // using UDP we don't know if packet dropped or not, so all packet are considered as succesfully sent. + TotalPayloadsSent uint64 + // TotalPayloadsDropped is the total number of payload dropped by the client. This includes all cause of dropped + // (TotalPayloadsDroppedQueueFull and TotalPayloadsDroppedWriter). When using UDP This won't includes the + // network dropped. + TotalPayloadsDropped uint64 + // TotalPayloadsDroppedWriter is the total number of payload dropped by the writer (when using UDS or named + // pipe) due to network timeout or error. + TotalPayloadsDroppedWriter uint64 + // TotalPayloadsDroppedQueueFull is the total number of payload dropped internally because the queue of payloads + // waiting to be sent on the wire is full. This means the client is generating more metrics than can be sent on + // the wire. If your app sends metrics in batch look at WithSenderQueueSize option to increase the queue size. + TotalPayloadsDroppedQueueFull uint64 + + // TotalBytesSent is the total number of bytes succesfully sent by the client. When using UDP we don't know if + // packet dropped or not, so all packet are considered as succesfully sent. + TotalBytesSent uint64 + // TotalBytesDropped is the total number of bytes dropped by the client. This includes all cause of dropped + // (TotalBytesDroppedQueueFull and TotalBytesDroppedWriter). When using UDP This + // won't includes the network dropped. + TotalBytesDropped uint64 + // TotalBytesDroppedWriter is the total number of bytes dropped by the writer (when using UDS or named pipe) due + // to network timeout or error. + TotalBytesDroppedWriter uint64 + // TotalBytesDroppedQueueFull is the total number of bytes dropped internally because the queue of payloads + // waiting to be sent on the wire is full. This means the client is generating more metrics than can be sent on + // the wire. If your app sends metrics in batch look at WithSenderQueueSize option to increase the queue size. + TotalBytesDroppedQueueFull uint64 + + // + // Those are produced by the 'aggregator' + // + + // AggregationNbContext is the total number of contexts flushed by the aggregator when either + // WithClientSideAggregation or WithExtendedClientSideAggregation options are enabled. + AggregationNbContext uint64 + // AggregationNbContextGauge is the total number of contexts for gauges flushed by the aggregator when either + // WithClientSideAggregation or WithExtendedClientSideAggregation options are enabled. + AggregationNbContextGauge uint64 + // AggregationNbContextCount is the total number of contexts for counts flushed by the aggregator when either + // WithClientSideAggregation or WithExtendedClientSideAggregation options are enabled. + AggregationNbContextCount uint64 + // AggregationNbContextSet is the total number of contexts for sets flushed by the aggregator when either + // WithClientSideAggregation or WithExtendedClientSideAggregation options are enabled. + AggregationNbContextSet uint64 + // AggregationNbContextHistogram is the total number of contexts for histograms flushed by the aggregator when either + // WithClientSideAggregation or WithExtendedClientSideAggregation options are enabled. + AggregationNbContextHistogram uint64 + // AggregationNbContextDistribution is the total number of contexts for distributions flushed by the aggregator when either + // WithClientSideAggregation or WithExtendedClientSideAggregation options are enabled. + AggregationNbContextDistribution uint64 + // AggregationNbContextTiming is the total number of contexts for timings flushed by the aggregator when either + // WithClientSideAggregation or WithExtendedClientSideAggregation options are enabled. + AggregationNbContextTiming uint64 +} + +type telemetryClient struct { + sync.RWMutex // used mostly to change the transport tag. + + c *Client + aggEnabled bool // is aggregation enabled and should we sent aggregation telemetry. + transport string + tags []string + tagsByType map[metricType][]string + transportTagKnown bool + sender *sender + worker *worker + lastSample Telemetry // The previous sample of telemetry sent +} + +func newTelemetryClient(c *Client, aggregationEnabled bool) *telemetryClient { + t := &telemetryClient{ + c: c, + aggEnabled: aggregationEnabled, + tags: []string{}, + tagsByType: map[metricType][]string{}, + } + + t.setTags() + return t +} + +func newTelemetryClientWithCustomAddr(c *Client, telemetryAddr string, aggregationEnabled bool, pool *bufferPool, + writeTimeout time.Duration, connectTimeout time.Duration, +) (*telemetryClient, error) { + telemetryAddr = resolveAddr(telemetryAddr) + telemetryWriter, _, err := createWriter(telemetryAddr, writeTimeout, connectTimeout) + if err != nil { + return nil, fmt.Errorf("Could not resolve telemetry address: %v", err) + } + + t := newTelemetryClient(c, aggregationEnabled) + + // Creating a custom sender/worker with 1 worker in mutex mode for the + // telemetry that share the same bufferPool. + // FIXME due to performance pitfall, we're always using UDP defaults + // even for UDS. + t.sender = newSender(telemetryWriter, DefaultUDPBufferPoolSize, pool, c.errorHandler) + t.worker = newWorker(pool, t.sender) + return t, nil +} + +func (t *telemetryClient) run(wg *sync.WaitGroup, stop chan struct{}) { + wg.Add(1) + go func() { + defer wg.Done() + ticker := time.NewTicker(telemetryInterval) + for { + select { + case <-ticker.C: + t.sendTelemetry() + case <-stop: + ticker.Stop() + if t.sender != nil { + t.sender.close() + } + return + } + } + }() +} + +func (t *telemetryClient) sendTelemetry() { + for _, m := range t.flush() { + if t.worker != nil { + t.worker.processMetric(m) + } else { + t.c.send(m) + } + } + + if t.worker != nil { + t.worker.flush() + } +} + +func (t *telemetryClient) getTelemetry() Telemetry { + if t == nil { + // telemetry was disabled through the WithoutTelemetry option + return Telemetry{} + } + + tlm := Telemetry{} + t.c.flushTelemetryMetrics(&tlm) + t.c.sender.flushTelemetryMetrics(&tlm) + t.c.agg.flushTelemetryMetrics(&tlm) + + tlm.TotalMetrics = tlm.TotalMetricsGauge + + tlm.TotalMetricsCount + + tlm.TotalMetricsSet + + tlm.TotalMetricsHistogram + + tlm.TotalMetricsDistribution + + tlm.TotalMetricsTiming + + tlm.TotalPayloadsDropped = tlm.TotalPayloadsDroppedQueueFull + tlm.TotalPayloadsDroppedWriter + tlm.TotalBytesDropped = tlm.TotalBytesDroppedQueueFull + tlm.TotalBytesDroppedWriter + + if t.aggEnabled { + tlm.AggregationNbContext = tlm.AggregationNbContextGauge + + tlm.AggregationNbContextCount + + tlm.AggregationNbContextSet + + tlm.AggregationNbContextHistogram + + tlm.AggregationNbContextDistribution + + tlm.AggregationNbContextTiming + } + return tlm +} + +// setTransportTag if it was never set and is now known. +func (t *telemetryClient) setTags() { + transport := t.c.GetTransport() + t.RLock() + // We need to refresh if we never set the tags or if the transport changed. + // For example when `unix://` is used we might return `uds` until we actually connect and detect that + // this is a UDS Stream socket and then return `uds-stream`. + needsRefresh := len(t.tags) == len(t.c.tags) || t.transport != transport + t.RUnlock() + + if !needsRefresh { + return + } + + t.Lock() + defer t.Unlock() + + t.transport = transport + t.tags = append(t.c.tags, clientTelemetryTag, clientVersionTelemetryTag) + if transport != "" { + t.tags = append(t.tags, "client_transport:"+transport) + } + t.tagsByType[gauge] = append(append([]string{}, t.tags...), "metrics_type:gauge") + t.tagsByType[count] = append(append([]string{}, t.tags...), "metrics_type:count") + t.tagsByType[set] = append(append([]string{}, t.tags...), "metrics_type:set") + t.tagsByType[timing] = append(append([]string{}, t.tags...), "metrics_type:timing") + t.tagsByType[histogram] = append(append([]string{}, t.tags...), "metrics_type:histogram") + t.tagsByType[distribution] = append(append([]string{}, t.tags...), "metrics_type:distribution") +} + +// flushTelemetry returns Telemetry metrics to be flushed. It's its own function to ease testing. +func (t *telemetryClient) flush() []metric { + m := []metric{} + + // same as Count but without global namespace + telemetryCount := func(name string, value int64, tags []string) { + m = append(m, metric{metricType: count, name: name, ivalue: value, tags: tags, rate: 1}) + } + + tlm := t.getTelemetry() + t.setTags() + + // We send the diff between now and the previous telemetry flush. This keep the same telemetry behavior from V4 + // so users dashboard's aren't broken when upgrading to V5. It also allow to graph on the same dashboard a mix + // of V4 and V5 apps. + telemetryCount("datadog.dogstatsd.client.metrics", int64(tlm.TotalMetrics-t.lastSample.TotalMetrics), t.tags) + telemetryCount("datadog.dogstatsd.client.metrics_by_type", int64(tlm.TotalMetricsGauge-t.lastSample.TotalMetricsGauge), t.tagsByType[gauge]) + telemetryCount("datadog.dogstatsd.client.metrics_by_type", int64(tlm.TotalMetricsCount-t.lastSample.TotalMetricsCount), t.tagsByType[count]) + telemetryCount("datadog.dogstatsd.client.metrics_by_type", int64(tlm.TotalMetricsHistogram-t.lastSample.TotalMetricsHistogram), t.tagsByType[histogram]) + telemetryCount("datadog.dogstatsd.client.metrics_by_type", int64(tlm.TotalMetricsDistribution-t.lastSample.TotalMetricsDistribution), t.tagsByType[distribution]) + telemetryCount("datadog.dogstatsd.client.metrics_by_type", int64(tlm.TotalMetricsSet-t.lastSample.TotalMetricsSet), t.tagsByType[set]) + telemetryCount("datadog.dogstatsd.client.metrics_by_type", int64(tlm.TotalMetricsTiming-t.lastSample.TotalMetricsTiming), t.tagsByType[timing]) + telemetryCount("datadog.dogstatsd.client.events", int64(tlm.TotalEvents-t.lastSample.TotalEvents), t.tags) + telemetryCount("datadog.dogstatsd.client.service_checks", int64(tlm.TotalServiceChecks-t.lastSample.TotalServiceChecks), t.tags) + + telemetryCount("datadog.dogstatsd.client.metric_dropped_on_receive", int64(tlm.TotalDroppedOnReceive-t.lastSample.TotalDroppedOnReceive), t.tags) + + telemetryCount("datadog.dogstatsd.client.packets_sent", int64(tlm.TotalPayloadsSent-t.lastSample.TotalPayloadsSent), t.tags) + telemetryCount("datadog.dogstatsd.client.packets_dropped", int64(tlm.TotalPayloadsDropped-t.lastSample.TotalPayloadsDropped), t.tags) + telemetryCount("datadog.dogstatsd.client.packets_dropped_queue", int64(tlm.TotalPayloadsDroppedQueueFull-t.lastSample.TotalPayloadsDroppedQueueFull), t.tags) + telemetryCount("datadog.dogstatsd.client.packets_dropped_writer", int64(tlm.TotalPayloadsDroppedWriter-t.lastSample.TotalPayloadsDroppedWriter), t.tags) + + telemetryCount("datadog.dogstatsd.client.bytes_dropped", int64(tlm.TotalBytesDropped-t.lastSample.TotalBytesDropped), t.tags) + telemetryCount("datadog.dogstatsd.client.bytes_sent", int64(tlm.TotalBytesSent-t.lastSample.TotalBytesSent), t.tags) + telemetryCount("datadog.dogstatsd.client.bytes_dropped_queue", int64(tlm.TotalBytesDroppedQueueFull-t.lastSample.TotalBytesDroppedQueueFull), t.tags) + telemetryCount("datadog.dogstatsd.client.bytes_dropped_writer", int64(tlm.TotalBytesDroppedWriter-t.lastSample.TotalBytesDroppedWriter), t.tags) + + if t.aggEnabled { + telemetryCount("datadog.dogstatsd.client.aggregated_context", int64(tlm.AggregationNbContext-t.lastSample.AggregationNbContext), t.tags) + telemetryCount("datadog.dogstatsd.client.aggregated_context_by_type", int64(tlm.AggregationNbContextGauge-t.lastSample.AggregationNbContextGauge), t.tagsByType[gauge]) + telemetryCount("datadog.dogstatsd.client.aggregated_context_by_type", int64(tlm.AggregationNbContextSet-t.lastSample.AggregationNbContextSet), t.tagsByType[set]) + telemetryCount("datadog.dogstatsd.client.aggregated_context_by_type", int64(tlm.AggregationNbContextCount-t.lastSample.AggregationNbContextCount), t.tagsByType[count]) + telemetryCount("datadog.dogstatsd.client.aggregated_context_by_type", int64(tlm.AggregationNbContextHistogram-t.lastSample.AggregationNbContextHistogram), t.tagsByType[histogram]) + telemetryCount("datadog.dogstatsd.client.aggregated_context_by_type", int64(tlm.AggregationNbContextDistribution-t.lastSample.AggregationNbContextDistribution), t.tagsByType[distribution]) + telemetryCount("datadog.dogstatsd.client.aggregated_context_by_type", int64(tlm.AggregationNbContextTiming-t.lastSample.AggregationNbContextTiming), t.tagsByType[timing]) + } + + t.lastSample = tlm + + return m +} diff --git a/vendor/github.com/DataDog/datadog-go/statsd/udp.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/udp.go similarity index 73% rename from vendor/github.com/DataDog/datadog-go/statsd/udp.go rename to vendor/github.com/DataDog/datadog-go/v5/statsd/udp.go index 8af522c5b..b90f75279 100644 --- a/vendor/github.com/DataDog/datadog-go/statsd/udp.go +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/udp.go @@ -1,7 +1,6 @@ package statsd import ( - "errors" "net" "time" ) @@ -12,7 +11,7 @@ type udpWriter struct { } // New returns a pointer to a new udpWriter given an addr in the format "hostname:port". -func newUDPWriter(addr string) (*udpWriter, error) { +func newUDPWriter(addr string, _ time.Duration) (*udpWriter, error) { udpAddr, err := net.ResolveUDPAddr("udp", addr) if err != nil { return nil, err @@ -25,11 +24,6 @@ func newUDPWriter(addr string) (*udpWriter, error) { return writer, nil } -// SetWriteTimeout is not needed for UDP, returns error -func (w *udpWriter) SetWriteTimeout(d time.Duration) error { - return errors.New("SetWriteTimeout: not supported for UDP connections") -} - // Write data to the UDP connection with no error handling func (w *udpWriter) Write(data []byte) (int, error) { return w.conn.Write(data) @@ -38,3 +32,8 @@ func (w *udpWriter) Write(data []byte) (int, error) { func (w *udpWriter) Close() error { return w.conn.Close() } + +// GetTransportName returns the transport used by the sender +func (w *udpWriter) GetTransportName() string { + return writerNameUDP +} diff --git a/vendor/github.com/DataDog/datadog-go/v5/statsd/uds.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/uds.go new file mode 100644 index 000000000..09518992a --- /dev/null +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/uds.go @@ -0,0 +1,167 @@ +//go:build !windows +// +build !windows + +package statsd + +import ( + "encoding/binary" + "net" + "strings" + "sync" + "time" +) + +// udsWriter is an internal class wrapping around management of UDS connection +type udsWriter struct { + // Address to send metrics to, needed to allow reconnection on error + addr string + // Transport used + transport string + // Established connection object, or nil if not connected yet + conn net.Conn + // write timeout + writeTimeout time.Duration + // connect timeout + connectTimeout time.Duration + sync.RWMutex // used to lock conn / writer can replace it +} + +// newUDSWriter returns a pointer to a new udsWriter given a socket file path as addr. +func newUDSWriter(addr string, writeTimeout time.Duration, connectTimeout time.Duration, transport string) (*udsWriter, error) { + // Defer connection to first Write + writer := &udsWriter{addr: addr, transport: transport, conn: nil, writeTimeout: writeTimeout, connectTimeout: connectTimeout} + return writer, nil +} + +// GetTransportName returns the transport used by the writer +func (w *udsWriter) GetTransportName() string { + w.RLock() + defer w.RUnlock() + + if w.transport == "unix" { + return writerNameUDSStream + } else { + return writerNameUDS + } +} + +func (w *udsWriter) shouldCloseConnection(err error, partialWrite bool) bool { + if err != nil && partialWrite { + // We can't recover from a partial write + return true + } + if err, isNetworkErr := err.(net.Error); err != nil && (!isNetworkErr || !err.Timeout()) { + // Statsd server disconnected, retry connecting at next packet + return true + } + return false +} + +// Write data to the UDS connection with write timeout and minimal error handling: +// create the connection if nil, and destroy it if the statsd server has disconnected +func (w *udsWriter) Write(data []byte) (int, error) { + var n int + partialWrite := false + conn, err := w.ensureConnection() + if err != nil { + return 0, err + } + stream := conn.LocalAddr().Network() == "unix" + + // When using streams the deadline will only make us drop the packet if we can't write it at all, + // once we've started writing we need to finish. + conn.SetWriteDeadline(time.Now().Add(w.writeTimeout)) + + // When using streams, we append the length of the packet to the data + if stream { + bs := []byte{0, 0, 0, 0} + binary.LittleEndian.PutUint32(bs, uint32(len(data))) + _, err = w.conn.Write(bs) + + partialWrite = true + + // W need to be able to finish to write partially written packets once we have started. + // But we will reset the connection if we can't write anything at all for a long time. + w.conn.SetWriteDeadline(time.Now().Add(w.connectTimeout)) + + // Continue writing only if we've written the length of the packet + if err == nil { + n, err = w.conn.Write(data) + if err == nil { + partialWrite = false + } + } + } else { + n, err = w.conn.Write(data) + } + + if w.shouldCloseConnection(err, partialWrite) { + w.unsetConnection() + } + return n, err +} + +func (w *udsWriter) Close() error { + if w.conn != nil { + return w.conn.Close() + } + return nil +} + +func (w *udsWriter) tryToDial(network string) (net.Conn, error) { + udsAddr, err := net.ResolveUnixAddr(network, w.addr) + if err != nil { + return nil, err + } + newConn, err := net.DialTimeout(udsAddr.Network(), udsAddr.String(), w.connectTimeout) + if err != nil { + return nil, err + } + return newConn, nil +} + +func (w *udsWriter) ensureConnection() (net.Conn, error) { + // Check if we've already got a socket we can use + w.RLock() + currentConn := w.conn + w.RUnlock() + + if currentConn != nil { + return currentConn, nil + } + + // Looks like we might need to connect - try again with write locking. + w.Lock() + defer w.Unlock() + if w.conn != nil { + return w.conn, nil + } + + var newConn net.Conn + var err error + + // Try to guess the transport if not specified. + if w.transport == "" { + newConn, err = w.tryToDial("unixgram") + // try to connect with unixgram failed, try again with unix streams. + if err != nil && strings.Contains(err.Error(), "protocol wrong type for socket") { + newConn, err = w.tryToDial("unix") + } + } else { + newConn, err = w.tryToDial(w.transport) + } + + if err != nil { + return nil, err + } + w.conn = newConn + w.transport = newConn.RemoteAddr().Network() + return newConn, nil +} + +func (w *udsWriter) unsetConnection() { + w.Lock() + defer w.Unlock() + _ = w.conn.Close() + w.conn = nil +} diff --git a/vendor/github.com/DataDog/datadog-go/v5/statsd/uds_windows.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/uds_windows.go new file mode 100644 index 000000000..909f5a0a0 --- /dev/null +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/uds_windows.go @@ -0,0 +1,15 @@ +//go:build windows +// +build windows + +package statsd + +import ( + "fmt" + "time" +) + +// newUDSWriter is disabled on Windows, SOCK_DGRAM are still unavailable but +// SOCK_STREAM should work once implemented in the agent (https://devblogs.microsoft.com/commandline/af_unix-comes-to-windows/) +func newUDSWriter(_ string, _ time.Duration, _ time.Duration, _ string) (Transport, error) { + return nil, fmt.Errorf("Unix socket is not available on Windows") +} diff --git a/vendor/github.com/DataDog/datadog-go/statsd/utils.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/utils.go similarity index 77% rename from vendor/github.com/DataDog/datadog-go/statsd/utils.go rename to vendor/github.com/DataDog/datadog-go/v5/statsd/utils.go index a2829d94f..8c3ac8426 100644 --- a/vendor/github.com/DataDog/datadog-go/statsd/utils.go +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/utils.go @@ -19,5 +19,14 @@ func shouldSample(rate float64, r *rand.Rand, lock *sync.Mutex) bool { } lock.Unlock() return true +} + +func copySlice(src []string) []string { + if src == nil { + return nil + } + c := make([]string, len(src)) + copy(c, src) + return c } diff --git a/vendor/github.com/DataDog/datadog-go/statsd/worker.go b/vendor/github.com/DataDog/datadog-go/v5/statsd/worker.go similarity index 79% rename from vendor/github.com/DataDog/datadog-go/statsd/worker.go rename to vendor/github.com/DataDog/datadog-go/v5/statsd/worker.go index e5a3bac56..19dccd339 100644 --- a/vendor/github.com/DataDog/datadog-go/statsd/worker.go +++ b/vendor/github.com/DataDog/datadog-go/v5/statsd/worker.go @@ -59,8 +59,11 @@ func (w *worker) pullMetric() { } func (w *worker) processMetric(m metric) error { - if !shouldSample(m.rate, w.random, &w.randomLock) { - return nil + // Aggregated metrics are already sampled. + if m.metricType != distributionAggregated && m.metricType != histogramAggregated && m.metricType != timingAggregated { + if !shouldSample(m.rate, w.random, &w.randomLock) { + return nil + } } w.Lock() var err error @@ -72,19 +75,24 @@ func (w *worker) processMetric(m metric) error { return err } -func (w *worker) writeAggregatedMetricUnsafe(m metric, metricSymbol []byte, precision int) error { +func (w *worker) writeAggregatedMetricUnsafe(m metric, metricSymbol []byte, precision int, rate float64) error { globalPos := 0 // first check how much data we can write to the buffer: // +3 + len(metricSymbol) because the message will include '||#' before the tags // +1 for the potential line break at the start of the metric - tagsSize := len(m.stags) + 4 + len(metricSymbol) + extraSize := len(m.stags) + 4 + len(metricSymbol) + if m.rate < 1 { + // +2 for "|@" + // + the maximum size of a rate (https://en.wikipedia.org/wiki/IEEE_754-1985) + extraSize += 2 + 18 + } for _, t := range m.globalTags { - tagsSize += len(t) + 1 + extraSize += len(t) + 1 } for { - pos, err := w.buffer.writeAggregated(metricSymbol, m.namespace, m.globalTags, m.name, m.fvalues[globalPos:], m.stags, tagsSize, precision) + pos, err := w.buffer.writeAggregated(metricSymbol, m.namespace, m.globalTags, m.name, m.fvalues[globalPos:], m.stags, extraSize, precision, rate) if err == errPartialWrite { // We successfully wrote part of the histogram metrics. // We flush the current buffer and finish the histogram @@ -100,9 +108,9 @@ func (w *worker) writeAggregatedMetricUnsafe(m metric, metricSymbol []byte, prec func (w *worker) writeMetricUnsafe(m metric) error { switch m.metricType { case gauge: - return w.buffer.writeGauge(m.namespace, m.globalTags, m.name, m.fvalue, m.tags, m.rate) + return w.buffer.writeGauge(m.namespace, m.globalTags, m.name, m.fvalue, m.tags, m.rate, m.timestamp) case count: - return w.buffer.writeCount(m.namespace, m.globalTags, m.name, m.ivalue, m.tags, m.rate) + return w.buffer.writeCount(m.namespace, m.globalTags, m.name, m.ivalue, m.tags, m.rate, m.timestamp) case histogram: return w.buffer.writeHistogram(m.namespace, m.globalTags, m.name, m.fvalue, m.tags, m.rate) case distribution: @@ -112,15 +120,15 @@ func (w *worker) writeMetricUnsafe(m metric) error { case timing: return w.buffer.writeTiming(m.namespace, m.globalTags, m.name, m.fvalue, m.tags, m.rate) case event: - return w.buffer.writeEvent(*m.evalue, m.globalTags) + return w.buffer.writeEvent(m.evalue, m.globalTags) case serviceCheck: - return w.buffer.writeServiceCheck(*m.scvalue, m.globalTags) + return w.buffer.writeServiceCheck(m.scvalue, m.globalTags) case histogramAggregated: - return w.writeAggregatedMetricUnsafe(m, histogramSymbol, -1) + return w.writeAggregatedMetricUnsafe(m, histogramSymbol, -1, m.rate) case distributionAggregated: - return w.writeAggregatedMetricUnsafe(m, distributionSymbol, -1) + return w.writeAggregatedMetricUnsafe(m, distributionSymbol, -1, m.rate) case timingAggregated: - return w.writeAggregatedMetricUnsafe(m, timingSymbol, 6) + return w.writeAggregatedMetricUnsafe(m, timingSymbol, 6, m.rate) default: return nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index 401659e82..d4431ad59 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -128,9 +128,9 @@ github.com/AzureAD/microsoft-authentication-library-for-go/apps/internal/options github.com/AzureAD/microsoft-authentication-library-for-go/apps/internal/shared github.com/AzureAD/microsoft-authentication-library-for-go/apps/internal/version github.com/AzureAD/microsoft-authentication-library-for-go/apps/public -# github.com/DataDog/datadog-go v4.8.3+incompatible -## explicit -github.com/DataDog/datadog-go/statsd +# github.com/DataDog/datadog-go/v5 v5.6.0 +## explicit; go 1.13 +github.com/DataDog/datadog-go/v5/statsd # github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 ## explicit; go 1.21 github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp