From b8b10f81cb613ce8e14348535335be50298ab22d Mon Sep 17 00:00:00 2001 From: InfiniteStash <117855276+InfiniteStash@users.noreply.github.com> Date: Thu, 2 Jan 2025 14:43:41 +0100 Subject: [PATCH] Add OTel tracing support (#774) --- go.mod | 25 +++++- go.sum | 84 +++++++++++++++---- main.go | 9 +- pkg/api/factory.go | 5 +- pkg/api/server.go | 15 +++- .../databasetest/database_test_utils.go | 3 +- pkg/database/postgres.go | 23 ++++- pkg/dataloader/loaders.go | 6 +- pkg/logger/otel.go | 56 +++++++++++++ pkg/manager/config/config.go | 16 ++++ pkg/manager/cron/cron.go | 22 +++-- pkg/sqlx/dbi.go | 6 +- pkg/sqlx/querybuilder_edit.go | 2 +- pkg/sqlx/querybuilder_invite_key.go | 2 +- pkg/sqlx/querybuilder_joins.go | 4 +- pkg/sqlx/querybuilder_performer.go | 2 +- pkg/sqlx/querybuilder_scene.go | 15 ++-- pkg/sqlx/querybuilder_sql.go | 6 +- pkg/sqlx/querybuilder_studio.go | 4 +- pkg/sqlx/querybuilder_tag.go | 2 +- pkg/sqlx/querybuilder_user.go | 2 +- pkg/sqlx/querybuilder_user_token.go | 2 +- pkg/sqlx/sql.go | 14 ++-- pkg/sqlx/transaction.go | 31 ++++--- 24 files changed, 283 insertions(+), 73 deletions(-) create mode 100644 pkg/logger/otel.go diff --git a/go.mod b/go.mod index d0c4917e5..d7c8fa214 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,8 @@ require ( github.com/lib/pq v1.10.9 github.com/minio/minio-go/v7 v7.0.80 github.com/pkg/errors v0.9.1 + github.com/ravilushqa/otelgqlgen v0.17.0 + github.com/riandyrn/otelchi v0.11.0 github.com/robfig/cron/v3 v3.0.1 github.com/rs/cors v1.11.1 github.com/sirupsen/logrus v1.9.3 @@ -28,9 +30,15 @@ require ( github.com/vektah/gqlparser/v2 v2.5.19 github.com/wneessen/go-mail v0.5.2 go.deanishe.net/favicon v0.1.0 + go.nhat.io/otelsql v0.14.0 + go.opentelemetry.io/otel v1.33.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 + go.opentelemetry.io/otel/sdk v1.33.0 + go.opentelemetry.io/otel/trace v1.33.0 golang.org/x/crypto v0.31.0 golang.org/x/image v0.22.0 - golang.org/x/net v0.31.0 + golang.org/x/net v0.32.0 golang.org/x/sync v0.10.0 gotest.tools/v3 v3.5.1 ) @@ -39,17 +47,22 @@ require ( github.com/PuerkitoBio/goquery v1.9.3 // indirect github.com/agnivade/levenshtein v1.1.1 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/friendsofgo/errors v0.9.2 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-ini/ini v1.67.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/securecookie v1.1.2 // indirect - github.com/gorilla/websocket v1.5.0 // indirect + github.com/gorilla/websocket v1.5.3 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect @@ -70,6 +83,10 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/urfave/cli/v2 v2.27.5 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib v1.29.0 // indirect + go.opentelemetry.io/otel/metric v1.33.0 // indirect + go.opentelemetry.io/proto/otlp v1.4.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect @@ -78,6 +95,10 @@ require ( golang.org/x/text v0.21.0 // indirect golang.org/x/tools v0.24.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/grpc v1.68.1 // indirect + google.golang.org/protobuf v1.35.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 81bffa6ef..c026989ef 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/99designs/gqlgen v0.17.56/go.mod h1:rmB6vLvtL8uf9F9w0/irJ5alBkD8DJvj3 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/PuerkitoBio/goquery v1.6.0/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= @@ -21,6 +23,10 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= +github.com/bool64/shared v0.1.5 h1:fp3eUhBsrSjNCQPcSdQqZxxh9bBwrYiZ+zOKFkM0/2E= +github.com/bool64/shared v0.1.5/go.mod h1:081yz68YC9jeFB3+Bbmno2RFWvGKv1lPKkMP6MHJlPs= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -57,6 +63,7 @@ github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -103,6 +110,8 @@ github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17w github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang-migrate/migrate/v4 v4.18.1 h1:JML/k+t4tpHCpQTCAD62Nu43NUFzHY4CV3uAuvHGC+Y= github.com/golang-migrate/migrate/v4 v4.18.1/go.mod h1:HAX6m3sQgcdO81tdjn5exv20+3Kb13cmGli1hrD6hks= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -116,8 +125,10 @@ github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kX github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo= github.com/gorilla/sessions v1.4.0 h1:kpIYOp/oi6MG/p5PgxApU8srsSw9tuFbt46Lt7auzqQ= github.com/gorilla/sessions v1.4.0/go.mod h1:FLWm50oby91+hl7p/wRxDth9bWSuk0qVL2emc7lT5ik= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c h1:fEE5/5VNnYUoBOj2I9TP8Jc+a7lge3QWn9DKE7NCwfc= github.com/h2non/go-is-svg v0.0.0-20160927212452-35e8c4b0612c/go.mod h1:ObS/W+h8RYb1Y7fYivughjxojTmIu5iAIjSrSLCLeqE= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -129,6 +140,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/iancoleman/orderedmap v0.3.0 h1:5cbR2grmZR/DiVt+VJopEhtVs9YGInGIxAoMJn+Ichc= +github.com/iancoleman/orderedmap v0.3.0/go.mod h1:XuLcCUkdL5owUCQeF2Ue9uuw1EptkJDkXXS7VoV7XGE= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -176,7 +189,6 @@ github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3 github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= @@ -192,13 +204,17 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/ravilushqa/otelgqlgen v0.17.0 h1:bLwQfKqtj9P24QpjM2sc1ipBm5Fqv2u7DKN5LIpj3g8= +github.com/ravilushqa/otelgqlgen v0.17.0/go.mod h1:orOIikuYsay1y3CmLgd5gsHcT9EsnXwNKmkAplzzYXQ= +github.com/riandyrn/otelchi v0.11.0 h1:x9MFoTgHcwCC2DdWkTEEZ2ZQFkbl6z7GXLQtTANN6Gk= +github.com/riandyrn/otelchi v0.11.0/go.mod h1:FlBYmG9fBQu0jFRvZZrATP4mDvLX2H5gwELfpZvNlxY= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= @@ -244,10 +260,13 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/swaggest/assertjson v1.9.0 h1:dKu0BfJkIxv/xe//mkCrK5yZbs79jL7OVf9Ija7o2xQ= +github.com/swaggest/assertjson v1.9.0/go.mod h1:b+ZKX2VRiUjxfUIal0HDN85W0nHPAYUbYH5WkkSsFsU= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= @@ -261,20 +280,46 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhe github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= +github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.deanishe.net/favicon v0.1.0 h1:Afy941gjRik+DjUUcYHUxcztFEeFse2ITBkMMOlgefM= go.deanishe.net/favicon v0.1.0/go.mod h1:vIKVI+lUh8k3UAzaN4gjC+cpyatLQWmx0hVX4vLE8jU= go.mongodb.org/mongo-driver v1.4.2/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= +go.nhat.io/otelsql v0.14.0 h1:Mz4xo+WVQLAOPZy6abxjVzZzNe8xoOUh/tOMJoxo3oo= +go.nhat.io/otelsql v0.14.0/go.mod h1:iO9KfDBZO2WI6O7n+ippHe5OHdXQ5iiA2aIa3Kzywo8= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib v1.29.0 h1:fLxD2N918DFRlES8q9iv2yE7iIFlaIMZ7ek0D6qJMqk= +go.opentelemetry.io/contrib v1.29.0/go.mod h1:Tmhw9grdWtmXy6DxZNpIAudzYJqLeEM2P6QTZQSRwU8= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= -go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= -go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= -go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= -go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= -go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= -go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= +go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= +go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.30.0 h1:IyFlqNsi8VT/nwYlLJfdM0y1gavxGpEvnf6FtVfZ6X4= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.30.0/go.mod h1:bxiX8eUeKoAEQmbq/ecUT8UqZwCjZW52yJrXJUSozsk= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.30.0 h1:kn1BudCgwtE7PxLqcZkErpD8GKqLZ6BSzeW9QihQJeM= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.30.0/go.mod h1:ljkUDtAMdleoi9tIG1R6dJUpVwDcYjw3J2Q6Q/SuiC0= +go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= +go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= +go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= +go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= +go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= +go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= +go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= +go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= +go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= +go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -316,8 +361,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= -golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= +golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -392,10 +437,19 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= +google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= +google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= +google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= +google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/main.go b/main.go index ca3070d34..dcb98d531 100644 --- a/main.go +++ b/main.go @@ -2,11 +2,13 @@ package main import ( + "context" "embed" "github.com/stashapp/stash-box/pkg/api" "github.com/stashapp/stash-box/pkg/database" "github.com/stashapp/stash-box/pkg/image" + "github.com/stashapp/stash-box/pkg/logger" "github.com/stashapp/stash-box/pkg/manager" "github.com/stashapp/stash-box/pkg/manager/config" "github.com/stashapp/stash-box/pkg/manager/cron" @@ -19,12 +21,17 @@ var ui embed.FS func main() { manager.Initialize() + + cleanup := logger.InitTracer() + //nolint:errcheck + defer cleanup(context.Background()) + api.InitializeSession() const databaseProvider = "postgres" db := database.Initialize(databaseProvider, config.GetDatabasePath()) txnMgr := sqlx.NewTxnMgr(db) - user.CreateSystemUsers(txnMgr.Repo()) + user.CreateSystemUsers(txnMgr.Repo(context.Background())) api.Start(txnMgr, ui) cron.Init(txnMgr) diff --git a/pkg/api/factory.go b/pkg/api/factory.go index 003dd76cc..604bbeaa1 100644 --- a/pkg/api/factory.go +++ b/pkg/api/factory.go @@ -10,14 +10,15 @@ import ( type RepoProvider interface { // IMPORTANT: the returned Repo object MUST NOT be shared between goroutines. // that is: call Repo for each new request/goroutine - Repo() models.Repo + Repo(ctx context.Context) models.Repo } // creates a new Repo (with its own transaction boundary) for each incoming request func repoMiddleware(provider RepoProvider) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - r = r.WithContext(context.WithValue(r.Context(), ContextRepo, provider.Repo())) + ctx := r.Context() + r = r.WithContext(context.WithValue(ctx, ContextRepo, provider.Repo(ctx))) next.ServeHTTP(w, r) }) diff --git a/pkg/api/server.go b/pkg/api/server.go index 8f3bbcf98..bf63453e2 100644 --- a/pkg/api/server.go +++ b/pkg/api/server.go @@ -14,13 +14,19 @@ import ( "strings" "github.com/klauspost/compress/flate" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "github.com/ravilushqa/otelgqlgen" + + "github.com/99designs/gqlgen/graphql" gqlHandler "github.com/99designs/gqlgen/graphql/handler" gqlExtension "github.com/99designs/gqlgen/graphql/handler/extension" gqlTransport "github.com/99designs/gqlgen/graphql/handler/transport" gqlPlayground "github.com/99designs/gqlgen/graphql/playground" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" + "github.com/riandyrn/otelchi" "github.com/rs/cors" "github.com/stashapp/stash-box/pkg/dataloader" "github.com/stashapp/stash-box/pkg/logger" @@ -93,6 +99,11 @@ func authenticateHandler() func(http.Handler) http.Handler { ctx = context.WithValue(ctx, user.ContextUser, u) ctx = context.WithValue(ctx, user.ContextRoles, roles) + span := trace.SpanFromContext(ctx) + if span.SpanContext().IsValid() && u != nil { + span.SetAttributes(attribute.String("user.id", u.ID.String())) + } + r = r.WithContext(ctx) next.ServeHTTP(w, r) @@ -110,6 +121,7 @@ func redirect(w http.ResponseWriter, req *http.Request) { func Start(rfp RepoProvider, ui embed.FS) { r := chi.NewRouter() + r.Use(otelchi.Middleware("", otelchi.WithChiRoutes(r))) var corsConfig *cors.Cors if config.GetIsProduction() { @@ -154,8 +166,9 @@ func Start(rfp RepoProvider, ui embed.FS) { gqlSrv.AddTransport(gqlTransport.POST{}) gqlSrv.AddTransport(gqlTransport.MultipartForm{}) gqlSrv.Use(gqlExtension.Introspection{}) + gqlSrv.Use(otelgqlgen.Middleware(otelgqlgen.WithCreateSpanFromFields(func(fieldCtx *graphql.FieldContext) bool { return fieldCtx.IsResolver }))) - r.Handle("/graphql", dataloader.Middleware(rfp.Repo())(gqlSrv)) + r.Handle("/graphql", dataloader.Middleware(getRepo)(gqlSrv)) if !config.GetIsProduction() { r.Handle("/playground", gqlPlayground.Handler("GraphQL playground", "/graphql")) diff --git a/pkg/database/databasetest/database_test_utils.go b/pkg/database/databasetest/database_test_utils.go index cd9e89a39..5d7230e3c 100644 --- a/pkg/database/databasetest/database_test_utils.go +++ b/pkg/database/databasetest/database_test_utils.go @@ -1,6 +1,7 @@ package databasetest import ( + "context" "database/sql" "errors" "fmt" @@ -59,7 +60,7 @@ func initPostgres(connString string) func() { db = database.Initialize(databaseType, connString) txnMgr := sqlxx.NewTxnMgr(db) - repo = txnMgr.Repo() + repo = txnMgr.Repo(context.TODO()) return teardownPostgres } diff --git a/pkg/database/postgres.go b/pkg/database/postgres.go index 244e362b1..bdf384eb5 100644 --- a/pkg/database/postgres.go +++ b/pkg/database/postgres.go @@ -1,6 +1,7 @@ package database import ( + "database/sql" "embed" "fmt" "time" @@ -9,11 +10,13 @@ import ( "github.com/jmoiron/sqlx" "github.com/stashapp/stash-box/pkg/logger" "github.com/stashapp/stash-box/pkg/manager/config" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" // Driver used here only _ "github.com/golang-migrate/migrate/v4/database/postgres" "github.com/golang-migrate/migrate/v4/source/iofs" _ "github.com/lib/pq" + "go.nhat.io/otelsql" ) const postgresDriver = "postgres" @@ -30,7 +33,25 @@ type PostgresProvider struct{} func (p *PostgresProvider) Open(databasePath string) *sqlx.DB { p.runMigrations(databasePath) - conn, err := sqlx.Open(postgresDriver, "postgres://"+databasePath) + driverName, err := otelsql.Register("postgres", + otelsql.TraceQueryWithoutArgs(), + otelsql.WithSystem(semconv.DBSystemPostgreSQL), + ) + + if err != nil { + logger.Fatalf("db.Open(): %q\n", err) + } + + db, err := sql.Open(driverName, "postgres://"+databasePath) + if err != nil { + logger.Fatalf("db.Open(): %q\n", err) + } + + if err := otelsql.RecordStats(db); err != nil { + logger.Fatalf("db.Open(): %q\n", err) + } + + conn := sqlx.NewDb(db, "postgres") conn.SetMaxOpenConns(config.GetMaxOpenConns()) conn.SetMaxIdleConns(config.GetMaxIdleConns()) conn.SetConnMaxLifetime(time.Duration(config.GetConnMaxLifetime()) * time.Minute) diff --git a/pkg/dataloader/loaders.go b/pkg/dataloader/loaders.go index e3b00f974..401796c4b 100644 --- a/pkg/dataloader/loaders.go +++ b/pkg/dataloader/loaders.go @@ -46,10 +46,12 @@ type Loaders struct { EditCommentByID EditCommentLoader } -func Middleware(fac models.Repo) func(next http.Handler) http.Handler { +func Middleware(getRepo func(ctx context.Context) models.Repo) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - ctx := context.WithValue(r.Context(), loadersKey, GetLoaders(r.Context(), fac)) + ctx := r.Context() + fac := getRepo(ctx) + ctx = context.WithValue(ctx, loadersKey, GetLoaders(r.Context(), fac)) r = r.WithContext(ctx) next.ServeHTTP(w, r) }) diff --git a/pkg/logger/otel.go b/pkg/logger/otel.go new file mode 100644 index 000000000..0663ef3b4 --- /dev/null +++ b/pkg/logger/otel.go @@ -0,0 +1,56 @@ +package logger + +import ( + "context" + "log" + + "github.com/stashapp/stash-box/pkg/manager/config" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +func InitTracer() func(context.Context) error { + otelConfig := config.GetOTelConfig() + if otelConfig == nil { + return nil + } + + exporter, err := otlptrace.New( + context.Background(), + otlptracegrpc.NewClient( + otlptracegrpc.WithInsecure(), + otlptracegrpc.WithEndpoint(otelConfig.Endpoint), + ), + ) + + if err != nil { + log.Fatal(err) + } + + resources, err := resource.New( + context.Background(), + resource.WithAttributes( + attribute.String("service.name", config.GetTitle()), + attribute.String("library.language", "go"), + ), + ) + if err != nil { + log.Print("Could not set resources: ", err) + } + + otel.SetTracerProvider( + sdktrace.NewTracerProvider( + sdktrace.WithSampler(sdktrace.TraceIDRatioBased(otelConfig.TraceRatio)), + sdktrace.WithBatcher(exporter), + sdktrace.WithResource(resources), + ), + ) + + logger.Infof("otel initialized with collector: %s, ratio: %f", otelConfig.Endpoint, otelConfig.TraceRatio) + + return exporter.Shutdown +} diff --git a/pkg/manager/config/config.go b/pkg/manager/config/config.go index 0216b83f2..2168b421b 100644 --- a/pkg/manager/config/config.go +++ b/pkg/manager/config/config.go @@ -25,6 +25,11 @@ type PostgresConfig struct { ConnMaxLifetime int `mapstructure:"conn_max_lifetime"` } +type OTelConfig struct { + Endpoint string `mapstructure:"endpoint"` + TraceRatio float64 `mapstructure:"trace_ratio"` +} + type config struct { Host string `mapstructure:"host"` Port int `mapstructure:"port"` @@ -91,6 +96,10 @@ type config struct { PostgresConfig `mapstructure:",squash"` } + OTel struct { + OTelConfig `mapstructure:",squash"` + } + PHashDistance int `mapstructure:"phash_distance"` Title string `mapstructure:"title"` @@ -246,6 +255,13 @@ func GetS3Config() *S3Config { return &C.S3.S3Config } +func GetOTelConfig() *OTelConfig { + if C.OTel.Endpoint != "" { + return &C.OTel.OTelConfig + } + return nil +} + // ValidateImageLocation returns an error is image_location is not set. func ValidateImageLocation() error { if C.ImageLocation == "" { diff --git a/pkg/manager/cron/cron.go b/pkg/manager/cron/cron.go index 86c110e76..2763fce72 100644 --- a/pkg/manager/cron/cron.go +++ b/pkg/manager/cron/cron.go @@ -1,6 +1,8 @@ package cron import ( + "context" + "github.com/robfig/cron/v3" "golang.org/x/sync/semaphore" @@ -26,7 +28,10 @@ func (c Cron) processEdits() { } defer sem.Release(1) - edits, err := c.rfp.Repo().Edit().FindCompletedEdits(config.GetVotingPeriod(), config.GetMinDestructiveVotingPeriod(), config.GetVoteApplicationThreshold()) + ctx := context.Background() + repo := c.rfp.Repo(ctx) + + edits, err := repo.Edit().FindCompletedEdits(config.GetVotingPeriod(), config.GetMinDestructiveVotingPeriod(), config.GetVoteApplicationThreshold()) if err != nil { logger.Errorf("Edit cronjob failed to fetch completed edits: %s", err.Error()) return @@ -34,7 +39,7 @@ func (c Cron) processEdits() { logger.Debugf("Edit cronjob running for %d edits", len(edits)) for _, e := range edits { - if err := c.rfp.Repo().WithTxn(func() error { + if err := repo.WithTxn(func() error { voteThreshold := 0 if e.IsDestructive() { // Require at least +1 votes to pass destructive edits @@ -43,9 +48,9 @@ func (c Cron) processEdits() { var err error if e.VoteCount >= voteThreshold { - _, err = edit.ApplyEdit(c.rfp.Repo(), e.ID, false) + _, err = edit.ApplyEdit(repo, e.ID, false) } else { - _, err = edit.CloseEdit(c.rfp.Repo(), e.ID, models.VoteStatusEnumRejected) + _, err = edit.CloseEdit(repo, e.ID, models.VoteStatusEnumRejected) } return err }); err != nil { @@ -55,7 +60,8 @@ func (c Cron) processEdits() { } func (c Cron) cleanDrafts() { - fac := c.rfp.Repo() + ctx := context.Background() + fac := c.rfp.Repo(ctx) err := fac.WithTxn(func() error { drafts, err := fac.Draft().FindExpired(config.GetDraftTimeLimit()) if err != nil { @@ -75,7 +81,8 @@ func (c Cron) cleanDrafts() { } func (c Cron) cleanTokens() { - fac := c.rfp.Repo() + ctx := context.Background() + fac := c.rfp.Repo(ctx) err := fac.WithTxn(func() error { return fac.UserToken().DestroyExpired() }) @@ -86,7 +93,8 @@ func (c Cron) cleanTokens() { } func (c Cron) cleanInvites() { - fac := c.rfp.Repo() + ctx := context.Background() + fac := c.rfp.Repo(ctx) err := fac.WithTxn(func() error { return fac.Invite().DestroyExpired() }) diff --git a/pkg/sqlx/dbi.go b/pkg/sqlx/dbi.go index 820edf7a2..49da1256a 100644 --- a/pkg/sqlx/dbi.go +++ b/pkg/sqlx/dbi.go @@ -106,7 +106,7 @@ func selectStatement(t table) string { func (q dbi) queryx(query string, args ...interface{}) (*sqlx.Rows, error) { query = q.db().Rebind(query) - return q.db().Queryx(query, args...) + return q.db().QueryxContext(q.txn.ctx, query, args...) } func (q dbi) queryFunc(query string, args []interface{}, f func(rows *sqlx.Rows) error) error { @@ -271,7 +271,7 @@ func (q dbi) Count(query queryBuilder) (int, error) { rawQuery := query.buildCountQuery() rawQuery = q.db().Rebind(rawQuery) - err = q.db().Get(&result, rawQuery, query.args...) + err = q.db().GetContext(q.txn.ctx, &result, rawQuery, query.args...) if err != nil && errors.Is(err, sql.ErrNoRows) { // TODO - log error instead of returning SQL @@ -309,6 +309,6 @@ func (q dbi) QueryOnly(query queryBuilder, output Models) error { func (q dbi) DeleteQuery(query queryBuilder) error { ensureTx(q.txn) queryStr := q.db().Rebind(query.buildQuery()) - _, err := q.db().Exec(queryStr, query.args...) + _, err := q.db().ExecContext(q.txn.ctx, queryStr, query.args...) return err } diff --git a/pkg/sqlx/querybuilder_edit.go b/pkg/sqlx/querybuilder_edit.go index 00cee5ba8..a2f6a7bab 100644 --- a/pkg/sqlx/querybuilder_edit.go +++ b/pkg/sqlx/querybuilder_edit.go @@ -197,7 +197,7 @@ func (qb *editQueryBuilder) FindSceneID(id uuid.UUID) (*uuid.UUID, error) { // } func (qb *editQueryBuilder) Count() (int, error) { - return runCountQuery(qb.dbi.db(), buildCountQuery("SELECT edits.id FROM edits"), nil) + return runCountQuery(qb.dbi, buildCountQuery("SELECT edits.id FROM edits"), nil) } func (qb *editQueryBuilder) buildQuery(filter models.EditQueryInput, userID uuid.UUID) (*queryBuilder, error) { diff --git a/pkg/sqlx/querybuilder_invite_key.go b/pkg/sqlx/querybuilder_invite_key.go index 58ceaa4de..a0d863e64 100644 --- a/pkg/sqlx/querybuilder_invite_key.go +++ b/pkg/sqlx/querybuilder_invite_key.go @@ -133,7 +133,7 @@ func (qb *inviteKeyQueryBuilder) FindActiveKeysForUser(userID uuid.UUID, expireT } func (qb *inviteKeyQueryBuilder) Count() (int, error) { - return runCountQuery(qb.dbi.db(), buildCountQuery("SELECT invite_keys.id FROM invite_keys"), nil) + return runCountQuery(qb.dbi, buildCountQuery("SELECT invite_keys.id FROM invite_keys"), nil) } func (qb *inviteKeyQueryBuilder) KeyUsed(id uuid.UUID) (*int, error) { diff --git a/pkg/sqlx/querybuilder_joins.go b/pkg/sqlx/querybuilder_joins.go index 6acb0622d..9a5615de3 100644 --- a/pkg/sqlx/querybuilder_joins.go +++ b/pkg/sqlx/querybuilder_joins.go @@ -146,7 +146,7 @@ func (qb *joinsQueryBuilder) IsPerformerFavorite(favorite models.PerformerFavori AND user_id = $2 ` args := []interface{}{favorite.PerformerID, favorite.UserID} - res, err := runCountQuery(qb.dbi.db(), query, args) + res, err := runCountQuery(qb.dbi, query, args) return res > 0, err } @@ -157,7 +157,7 @@ func (qb *joinsQueryBuilder) IsStudioFavorite(favorite models.StudioFavorite) (b AND user_id = $2 ` args := []interface{}{favorite.StudioID, favorite.UserID} - res, err := runCountQuery(qb.dbi.db(), query, args) + res, err := runCountQuery(qb.dbi, query, args) return res > 0, err } diff --git a/pkg/sqlx/querybuilder_performer.go b/pkg/sqlx/querybuilder_performer.go index 3cd3a2b57..a7ca73592 100644 --- a/pkg/sqlx/querybuilder_performer.go +++ b/pkg/sqlx/querybuilder_performer.go @@ -186,7 +186,7 @@ func (qb *performerQueryBuilder) FindByName(name string) (models.Performers, err } func (qb *performerQueryBuilder) Count() (int, error) { - return runCountQuery(qb.dbi.db(), buildCountQuery("SELECT performers.id FROM performers"), nil) + return runCountQuery(qb.dbi, buildCountQuery("SELECT performers.id FROM performers"), nil) } func (qb *performerQueryBuilder) buildQuery(filter models.PerformerQueryInput, userID uuid.UUID) *queryBuilder { diff --git a/pkg/sqlx/querybuilder_scene.go b/pkg/sqlx/querybuilder_scene.go index 8f65ba688..7f159a454 100644 --- a/pkg/sqlx/querybuilder_scene.go +++ b/pkg/sqlx/querybuilder_scene.go @@ -116,8 +116,7 @@ func (qb *sceneQueryBuilder) UpdateFingerprints(sceneID uuid.UUID, updatedJoins func (qb *sceneQueryBuilder) DestroyFingerprints(sceneID uuid.UUID, toDestroy models.SceneFingerprints) error { for _, fp := range toDestroy { - fmt.Println(fp) - res, err := qb.dbi.db().Exec(` + res, err := qb.dbi.db().ExecContext(qb.dbi.txn.ctx, ` DELETE FROM scene_fingerprints SFP USING fingerprints FP WHERE SFP.fingerprint_id = FP.id @@ -321,7 +320,7 @@ func (qb *sceneQueryBuilder) FindIdsBySceneFingerprints(fingerprints []*models.F query = qb.dbi.db().Rebind(query) output := models.SceneFingerprints{} - if err := qb.dbi.db().Select(&output, query, args...); err != nil { + if err := qb.dbi.db().SelectContext(qb.dbi.txn.ctx, &output, query, args...); err != nil { return nil, err } @@ -335,7 +334,7 @@ func (qb *sceneQueryBuilder) FindIdsBySceneFingerprints(fingerprints []*models.F } func (qb *sceneQueryBuilder) Count() (int, error) { - return runCountQuery(qb.dbi.db(), buildCountQuery("SELECT scenes.id FROM scenes"), nil) + return runCountQuery(qb.dbi, buildCountQuery("SELECT scenes.id FROM scenes"), nil) } func (qb *sceneQueryBuilder) buildQuery(filter models.SceneQueryInput, userID uuid.UUID, isCount bool) (*queryBuilder, error) { @@ -607,7 +606,7 @@ func fingerprintGroupToFingerprint(fpg sceneFingerprintGroup) *models.Fingerprin func (qb *sceneQueryBuilder) GetFingerprints(id uuid.UUID) (models.SceneFingerprints, error) { fingerprints := models.SceneFingerprints{} - err := qb.dbi.db().Select(&fingerprints, ` + err := qb.dbi.db().SelectContext(qb.dbi.txn.ctx, &fingerprints, ` SELECT SFP.scene_id, SFP.user_id, SFP.duration, SFP.created_at, FP.hash, FP.algorithm FROM scene_fingerprints SFP JOIN fingerprints FP ON SFP.fingerprint_id = FP.id @@ -821,7 +820,7 @@ func (qb *sceneQueryBuilder) SearchScenes(term string, limit int) ([]*models.Sce func (qb *sceneQueryBuilder) CountByPerformer(id uuid.UUID) (int, error) { var args []interface{} args = append(args, id) - return runCountQuery(qb.dbi.db(), buildCountQuery("SELECT scene_id FROM scene_performers WHERE performer_id = ?"), args) + return runCountQuery(qb.dbi, buildCountQuery("SELECT scene_id FROM scene_performers WHERE performer_id = ?"), args) } func (qb *sceneQueryBuilder) SoftDelete(scene models.Scene) (*models.Scene, error) { @@ -1072,14 +1071,14 @@ func (qb *sceneQueryBuilder) getOrCreateFingerprintID(hash string, algorithm str func (qb *sceneQueryBuilder) getFingerprintID(hash string, algorithm string) (int, error) { var id int - err := qb.dbi.db().Get(&id, "SELECT id FROM fingerprints WHERE hash = $1 AND algorithm = $2", hash, algorithm) + err := qb.dbi.db().GetContext(qb.dbi.txn.ctx, &id, "SELECT id FROM fingerprints WHERE hash = $1 AND algorithm = $2", hash, algorithm) return id, err } func (qb *sceneQueryBuilder) createFingerprint(hash string, algorithm string) (int, error) { var id int - err := qb.dbi.db().Get(&id, "INSERT INTO fingerprints (hash, algorithm) VALUES ($1, $2) RETURNING id", hash, algorithm) + err := qb.dbi.db().GetContext(qb.dbi.txn.ctx, &id, "INSERT INTO fingerprints (hash, algorithm) VALUES ($1, $2) RETURNING id", hash, algorithm) return id, err } diff --git a/pkg/sqlx/querybuilder_sql.go b/pkg/sqlx/querybuilder_sql.go index b53114136..42417198b 100644 --- a/pkg/sqlx/querybuilder_sql.go +++ b/pkg/sqlx/querybuilder_sql.go @@ -128,14 +128,14 @@ func getInBinding(length int) string { return "(" + bindings + ")" } -func runCountQuery(db db, query string, args []interface{}) (int, error) { +func runCountQuery(dbi *dbi, query string, args []interface{}) (int, error) { // Perform query and fetch result result := struct { Count int `db:"count"` }{0} - query = db.Rebind(query) - if err := db.Get(&result, query, args...); err != nil && !errors.Is(err, sql.ErrNoRows) { + query = dbi.db().Rebind(query) + if err := dbi.db().GetContext(dbi.txn.ctx, &result, query, args...); err != nil && !errors.Is(err, sql.ErrNoRows) { return 0, err } diff --git a/pkg/sqlx/querybuilder_studio.go b/pkg/sqlx/querybuilder_studio.go index eeacce95a..2e12c0bdc 100644 --- a/pkg/sqlx/querybuilder_studio.go +++ b/pkg/sqlx/querybuilder_studio.go @@ -159,7 +159,7 @@ func (qb *studioQueryBuilder) FindByParentID(id uuid.UUID) (models.Studios, erro } func (qb *studioQueryBuilder) Count() (int, error) { - return runCountQuery(qb.dbi.db(), buildCountQuery("SELECT studios.id FROM studios"), nil) + return runCountQuery(qb.dbi, buildCountQuery("SELECT studios.id FROM studios"), nil) } func (qb *studioQueryBuilder) Query(filter models.StudioQueryInput, userID uuid.UUID) (models.Studios, int, error) { @@ -311,7 +311,7 @@ func (qb *studioQueryBuilder) CountByPerformer(performerID uuid.UUID) ([]*models GROUP BY studio_id ) C ON S.id = C.studio_id` query = qb.dbi.db().Rebind(query) - if err := qb.dbi.db().Select(&results, query, performerID); err != nil && !errors.Is(err, sql.ErrNoRows) { + if err := qb.dbi.db().SelectContext(qb.dbi.txn.ctx, &results, query, performerID); err != nil && !errors.Is(err, sql.ErrNoRows) { return nil, err } diff --git a/pkg/sqlx/querybuilder_tag.go b/pkg/sqlx/querybuilder_tag.go index 3f949e8ee..97d971e45 100644 --- a/pkg/sqlx/querybuilder_tag.go +++ b/pkg/sqlx/querybuilder_tag.go @@ -222,7 +222,7 @@ func (qb *tagQueryBuilder) FindWithRedirect(id uuid.UUID) (*models.Tag, error) { } func (qb *tagQueryBuilder) Count() (int, error) { - return runCountQuery(qb.dbi.db(), buildCountQuery("SELECT tags.id FROM tags"), nil) + return runCountQuery(qb.dbi, buildCountQuery("SELECT tags.id FROM tags"), nil) } func (qb *tagQueryBuilder) Query(filter models.TagQueryInput) ([]*models.Tag, int, error) { diff --git a/pkg/sqlx/querybuilder_user.go b/pkg/sqlx/querybuilder_user.go index d64a64a55..844860d48 100644 --- a/pkg/sqlx/querybuilder_user.go +++ b/pkg/sqlx/querybuilder_user.go @@ -93,7 +93,7 @@ func (qb *userQueryBuilder) FindByEmail(email string) (*models.User, error) { } func (qb *userQueryBuilder) Count() (int, error) { - return runCountQuery(qb.dbi.db(), buildCountQuery("SELECT users.id FROM users"), nil) + return runCountQuery(qb.dbi, buildCountQuery("SELECT users.id FROM users"), nil) } func (qb *userQueryBuilder) Query(filter models.UserQueryInput) (models.Users, int, error) { diff --git a/pkg/sqlx/querybuilder_user_token.go b/pkg/sqlx/querybuilder_user_token.go index 5e3973be2..8608bddbf 100644 --- a/pkg/sqlx/querybuilder_user_token.go +++ b/pkg/sqlx/querybuilder_user_token.go @@ -69,5 +69,5 @@ func (qb *userTokenQueryBuilder) FindByInviteKey(key uuid.UUID) ([]*models.UserT } func (qb *userTokenQueryBuilder) Count() (int, error) { - return runCountQuery(qb.dbi.db(), buildCountQuery("SELECT "+userTokenTable+".id FROM "+userTokenTable), nil) + return runCountQuery(qb.dbi, buildCountQuery("SELECT "+userTokenTable+".id FROM "+userTokenTable), nil) } diff --git a/pkg/sqlx/sql.go b/pkg/sqlx/sql.go index 45bc97933..3ecb6289c 100644 --- a/pkg/sqlx/sql.go +++ b/pkg/sqlx/sql.go @@ -139,7 +139,7 @@ func ensureTx(txn *txnState) { func getByID(txn *txnState, t string, id uuid.UUID, object interface{}) error { query := txn.DB().Rebind(`SELECT * FROM ` + t + ` WHERE id = ? LIMIT 1`) - return txn.DB().Get(object, query, id) + return txn.DB().GetContext(txn.ctx, object, query, id) } func insertObject(txn *txnState, t string, object interface{}, conflictHandling *string) error { @@ -151,7 +151,8 @@ func insertObject(txn *txnState, t string, object interface{}, conflictHandling conflictClause = *conflictHandling } - _, err := txn.DB().NamedExec( + _, err := txn.DB().NamedExecContext( + txn.ctx, `INSERT INTO `+t+` (`+fields+`) VALUES (`+values+`) `+conflictClause+` @@ -164,7 +165,8 @@ func insertObject(txn *txnState, t string, object interface{}, conflictHandling func updateObjectByID(txn *txnState, t string, object interface{}, updateEmptyValues bool) error { ensureTx(txn) - _, err := txn.DB().NamedExec( + _, err := txn.DB().NamedExecContext( + txn.ctx, `UPDATE `+t+` SET `+sqlGenKeys(object, updateEmptyValues)+` WHERE `+t+`.id = :id`, object, ) @@ -176,7 +178,7 @@ func executeDeleteQuery(tableName string, id uuid.UUID, txn *txnState) error { ensureTx(txn) idColumnName := getColumn(tableName, "id") query := txn.DB().Rebind(`DELETE FROM ` + tableName + ` WHERE ` + idColumnName + ` = ?`) - _, err := txn.DB().Exec(query, id) + _, err := txn.DB().ExecContext(txn.ctx, query, id) return err } @@ -184,14 +186,14 @@ func softDeleteObjectByID(txn *txnState, t string, id uuid.UUID) error { ensureTx(txn) idColumnName := getColumn(t, "id") query := txn.DB().Rebind(`UPDATE ` + t + ` SET deleted=TRUE WHERE ` + idColumnName + ` = ?`) - _, err := txn.DB().Exec(query, id) + _, err := txn.DB().ExecContext(txn.ctx, query, id) return err } func deleteObjectsByColumn(txn *txnState, t string, column string, value interface{}) error { ensureTx(txn) query := txn.DB().Rebind(`DELETE FROM ` + t + ` WHERE ` + column + ` = ?`) - _, err := txn.DB().Exec(query, value) + _, err := txn.DB().ExecContext(txn.ctx, query, value) return err } diff --git a/pkg/sqlx/transaction.go b/pkg/sqlx/transaction.go index ffa0a9f46..5b2e1273d 100644 --- a/pkg/sqlx/transaction.go +++ b/pkg/sqlx/transaction.go @@ -1,6 +1,7 @@ package sqlx import ( + "context" "database/sql" "fmt" @@ -12,17 +13,23 @@ import ( // db is intended as an interface to both sqlx.db and sqlx.Tx, dependent // on transaction state. Add sqlx.* methods as needed. type db interface { - NamedExec(query string, arg interface{}) (sql.Result, error) - Exec(query string, args ...interface{}) (sql.Result, error) + // NamedExec(query string, arg interface{}) (sql.Result, error) + NamedExecContext(ctx context.Context, query string, arg interface{}) (sql.Result, error) + // Exec(query string, args ...interface{}) (sql.Result, error) + ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) Rebind(query string) string - Get(dest interface{}, query string, args ...interface{}) error - Select(dest interface{}, query string, args ...interface{}) error - Queryx(query string, args ...interface{}) (*sqlx.Rows, error) + // Get(dest interface{}, query string, args ...interface{}) error + GetContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error + // Select(dest interface{}, query string, args ...interface{}) error + SelectContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error + // Queryx(query string, args ...interface{}) (*sqlx.Rows, error) + QueryxContext(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error) } type txnState struct { rootDB *sqlx.DB tx *sqlx.Tx + ctx context.Context } func (m *txnState) WithTxn(fn func() error) (err error) { @@ -31,7 +38,7 @@ func (m *txnState) WithTxn(fn func() error) (err error) { return } - tx, err := m.rootDB.Beginx() + tx, err := m.rootDB.BeginTxx(m.ctx, nil) if err != nil { return } @@ -67,7 +74,7 @@ func (m *txnState) ResetTxn() error { return err } - tx, err := m.rootDB.Beginx() + tx, err := m.rootDB.BeginTxx(m.ctx, nil) if err != nil { return err } @@ -92,17 +99,19 @@ type TxnMgr struct { db *sqlx.DB } -func (m *TxnMgr) New() txn.State { +func (m *TxnMgr) New(ctx context.Context) txn.State { return &txnState{ - rootDB: m.db, + m.db, + nil, + ctx, } } // Repo creates a new TxnState object and initialises the Repo // with it. -func (m *TxnMgr) Repo() models.Repo { +func (m *TxnMgr) Repo(ctx context.Context) models.Repo { return &repo{ - txnState: m.New().(*txnState), + txnState: m.New(ctx).(*txnState), } }