diff --git a/.gitignore b/.gitignore index 6cfeaa413..c2eff4529 100644 --- a/.gitignore +++ b/.gitignore @@ -182,3 +182,4 @@ dump.rdb *:Zone.Identifier node_modules/ +.env* diff --git a/benchmarks/django-workers/bench6.sh b/benchmarks/django-workers/bench6.sh new file mode 100755 index 000000000..04a99233e --- /dev/null +++ b/benchmarks/django-workers/bench6.sh @@ -0,0 +1,150 @@ +# trio is not supported by django yet and should break gevent + +FILE="./threads-py313.md" +CONNECTIONS=2000 +THREADS=20 +PORT=8000 +HOST="http://localhost:$PORT" +TIMEOUT=10 +SLEEP_TIME=3 + + +# it support wsgi and asgi +function bench { + HOST="http://localhost:8000" + + echo "" >> "$FILE" + echo "### JSON performance" >> "$FILE" + echo "#### Sync" >> "$FILE" + echo "" >> "$FILE" + echo "\`\`\`bash" >> "$FILE" + wrk -t "$THREADS" -c "$CONNECTIONS" -d10s "$HOST/myapp/sync/json" >> "$FILE" + echo "\`\`\`" >> "$FILE" + echo "" >> "$FILE" + echo "#### Async" >> "$FILE" + echo "" >> "$FILE" + echo "\`\`\`bash" >> "$FILE" + wrk -t "$THREADS" -c "$CONNECTIONS" -d10s "$HOST/myapp/async/json" >> "$FILE" + echo "\`\`\`" >> "$FILE" + echo "" >> "$FILE" + + echo "### Queries returned as JSON" >> "$FILE" + echo "#### Sync" >> "$FILE" + echo "" >> "$FILE" + echo "\`\`\`bash" >> "$FILE" + wrk -t "$THREADS" -c "$CONNECTIONS" -d10s "$HOST/myapp/sync/json_query" >> "$FILE" + echo "\`\`\`" >> "$FILE" + echo "" >> "$FILE" + echo "#### Async" >> "$FILE" + echo "" >> "$FILE" + echo "\`\`\`bash" >> "$FILE" + wrk -t "$THREADS" -c "$CONNECTIONS" -d10s "$HOST/myapp/async/json_query" >> "$FILE" + echo "\`\`\`" >> "$FILE" + echo "" >> "$FILE" + + echo "### Queries returned as HTML" >> "$FILE" + echo "#### Sync" >> "$FILE" + echo "" >> "$FILE" + echo "\`\`\`bash" >> "$FILE" + wrk -t "$THREADS" -c "$CONNECTIONS" -d10s "$HOST/myapp/sync/template_query" >> "$FILE" + echo "\`\`\`" >> "$FILE" + echo "" >> "$FILE" + echo "#### Async" >> "$FILE" + echo "" >> "$FILE" + echo "\`\`\`bash" >> "$FILE" + wrk -t "$THREADS" -c "$CONNECTIONS" -d10s "$HOST/myapp/async/template_query" >> "$FILE" + echo "\`\`\`" >> "$FILE" + echo "" >> "$FILE" + + echo "### Simulate a request 1s inside the server, then return a JSON" >> "$FILE" + echo "#### Sync" >> "$FILE" + echo "" >> "$FILE" + echo "\`\`\`bash" >> "$FILE" + wrk -t "$THREADS" -c "$CONNECTIONS" -d10s "$HOST/myapp/sync/gateway_1s" >> "$FILE" + echo "\`\`\`" >> "$FILE" + echo "" >> "$FILE" + echo "#### Async" >> "$FILE" + echo "" >> "$FILE" + echo "\`\`\`bash" >> "$FILE" + wrk -t "$THREADS" -c "$CONNECTIONS" -d10s "$HOST/myapp/async/gateway_1s" >> "$FILE" + echo "\`\`\`" >> "$FILE" + echo "" >> "$FILE" + + sleep $SLEEP_TIME + + echo "### Simulate a request 3s inside the server, then return a JSON" >> "$FILE" + echo "#### Sync" >> "$FILE" + echo "" >> "$FILE" + echo "\`\`\`bash" >> "$FILE" + wrk -t "$THREADS" -c "$CONNECTIONS" -d10s "$HOST/myapp/sync/gateway_3s" >> "$FILE" + echo "\`\`\`" >> "$FILE" + echo "" >> "$FILE" + echo "#### Async" >> "$FILE" + echo "" >> "$FILE" + echo "\`\`\`bash" >> "$FILE" + wrk -t "$THREADS" -c "$CONNECTIONS" -d10s "$HOST/myapp/async/gateway_3s" >> "$FILE" + echo "\`\`\`" >> "$FILE" + echo "" >> "$FILE" + + sleep $SLEEP_TIME + + echo "### Simulate a request 10s inside the server, then return a JSON" >> "$FILE" + echo "#### Sync" >> "$FILE" + echo "" >> "$FILE" + echo "\`\`\`bash" >> "$FILE" + wrk -t "$THREADS" -c "$CONNECTIONS" -d10s "$HOST/myapp/sync/gateway_10s" >> "$FILE" + echo "\`\`\`" >> "$FILE" + echo "" >> "$FILE" + echo "#### Async" >> "$FILE" + echo "" >> "$FILE" + echo "\`\`\`bash" >> "$FILE" + wrk -t "$THREADS" -c "$CONNECTIONS" -d10s "$HOST/myapp/async/gateway_10s" >> "$FILE" + echo "\`\`\`" >> "$FILE" + echo "" >> "$FILE" + + sleep $SLEEP_TIME + + echo "### Brotli" >> "$FILE" + echo "#### Sync" >> "$FILE" + echo "" >> "$FILE" + echo "\`\`\`bash" >> "$FILE" + wrk -t "$THREADS" -c "$CONNECTIONS" -d10s "$HOST/myapp/sync/brotli" >> "$FILE" + echo "\`\`\`" >> "$FILE" + echo "" >> "$FILE" + echo "#### Async" >> "$FILE" + echo "" >> "$FILE" + echo "\`\`\`bash" >> "$FILE" + wrk -t "$THREADS" -c "$CONNECTIONS" -d10s "$HOST/myapp/async/brotli" >> "$FILE" + echo "\`\`\`" >> "$FILE" + echo "" >> "$FILE" +} + + +echo "# Django Workers" > $FILE + +sudo fuser -k $PORT/tcp +PYTHON_GIL=0 python -Xgil=0 -m gunicorn mysite.asgi --timeout $TIMEOUT --threads $THREADS --worker-class uvicorn.workers.UvicornWorker & echo "starting server..." +sleep $SLEEP_TIME +echo "## ASGI Gunicorn Uvicorn, with threads -Xgil=0" >> $FILE +bench + +sudo fuser -k $PORT/tcp +PYTHON_GIL=1 python -Xgil=1 -m gunicorn mysite.asgi --timeout $TIMEOUT --threads $THREADS --worker-class uvicorn.workers.UvicornWorker & echo "starting server..." +sleep $SLEEP_TIME +echo "## ASGI Gunicorn Uvicorn, with threads -Xgil=1" >> $FILE +bench + +sudo fuser -k $PORT/tcp +PYTHON_GIL=0 python -Xgil=0 -m gunicorn mysite.asgi --timeout $TIMEOUT --workers $THREADS --worker-class uvicorn.workers.UvicornWorker & echo "starting server..." +sleep $SLEEP_TIME +echo "## ASGI Gunicorn Uvicorn, with workers -Xgil=0" >> $FILE +bench + +sudo fuser -k $PORT/tcp +PYTHON_GIL=1 python -Xgil=1 -m gunicorn mysite.asgi --timeout $TIMEOUT --workers $THREADS --worker-class uvicorn.workers.UvicornWorker & echo "starting server..." +sleep $SLEEP_TIME +echo "## ASGI Gunicorn Uvicorn, with workers -Xgil=1" >> $FILE +bench + + +sudo fuser -k $PORT/tcp diff --git a/benchmarks/django-workers/threads-py313.md b/benchmarks/django-workers/threads-py313.md new file mode 100644 index 000000000..defae88d4 --- /dev/null +++ b/benchmarks/django-workers/threads-py313.md @@ -0,0 +1,809 @@ +# Django Workers +## ASGI Gunicorn Uvicorn, with threads -Xgil=0 + +### JSON performance +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/json + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 136.35 242.30 0.99k 86.49% + 2000 requests in 10.09s, 550.78KB read + Socket errors: connect 0, read 0, write 0, timeout 2000 +Requests/sec: 198.31 +Transfer/sec: 54.61KB +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/json + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 179.31 276.02 0.91k 83.33% + 2000 requests in 10.05s, 550.78KB read + Socket errors: connect 0, read 0, write 0, timeout 2000 +Requests/sec: 199.02 +Transfer/sec: 54.81KB +``` + +### Queries returned as JSON +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/json_query + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.08s, 0.00B read +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/json_query + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.09s, 0.00B read + Socket errors: connect 0, read 2000, write 0, timeout 0 +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +### Queries returned as HTML +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/template_query + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.09s, 0.00B read + Socket errors: connect 0, read 2000, write 0, timeout 0 +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/template_query + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.09s, 0.00B read +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +### Simulate a request 1s inside the server, then return a JSON +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/gateway_1s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.09s, 0.00B read + Socket errors: connect 0, read 2000, write 0, timeout 0 +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/gateway_1s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 153.47 254.75 0.87k 79.41% + 2000 requests in 10.07s, 523.44KB read + Socket errors: connect 0, read 0, write 0, timeout 2000 +Requests/sec: 198.62 +Transfer/sec: 51.98KB +``` + +### Simulate a request 3s inside the server, then return a JSON +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/gateway_3s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 175.09 239.35 0.96k 84.06% + 1715 requests in 10.09s, 448.85KB read + Socket errors: connect 0, read 0, write 0, timeout 1715 +Requests/sec: 169.98 +Transfer/sec: 44.49KB +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/gateway_3s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.08s, 0.00B read + Socket errors: connect 0, read 2000, write 0, timeout 0 +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +### Simulate a request 10s inside the server, then return a JSON +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/gateway_10s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.10s, 0.00B read +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/gateway_10s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.10s, 0.00B read +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +### Brotli +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/brotli + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.10 0.32 1.00 90.00% + 10 requests in 10.06s, 65.45KB read + Socket errors: connect 0, read 0, write 0, timeout 10 +Requests/sec: 0.99 +Transfer/sec: 6.50KB +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/brotli + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.05s, 0.00B read +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +## ASGI Gunicorn Uvicorn, with threads -Xgil=1 + +### JSON performance +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/json + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 186.82 273.64 830.00 78.75% + 2000 requests in 10.07s, 550.78KB read + Socket errors: connect 0, read 0, write 0, timeout 2000 +Requests/sec: 198.58 +Transfer/sec: 54.69KB +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/json + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.10s, 0.00B read +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +### Queries returned as JSON +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/json_query + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.07s, 0.00B read + Socket errors: connect 0, read 2000, write 0, timeout 0 +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/json_query + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.09s, 0.00B read + Socket errors: connect 0, read 2000, write 0, timeout 0 +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +### Queries returned as HTML +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/template_query + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.09s, 0.00B read + Socket errors: connect 0, read 2000, write 0, timeout 0 +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/template_query + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.05s, 0.00B read + Socket errors: connect 0, read 2000, write 0, timeout 0 +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +### Simulate a request 1s inside the server, then return a JSON +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/gateway_1s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.10s, 0.00B read + Socket errors: connect 0, read 2000, write 0, timeout 0 +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/gateway_1s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.05s, 0.00B read +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +### Simulate a request 3s inside the server, then return a JSON +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/gateway_3s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.06s, 0.00B read +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/gateway_3s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.08s, 0.00B read + Socket errors: connect 0, read 2000, write 0, timeout 0 +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +### Simulate a request 10s inside the server, then return a JSON +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/gateway_10s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.07s, 0.00B read +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/gateway_10s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.03s, 0.00B read +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +### Brotli +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/brotli + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.11s, 0.00B read +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/brotli + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 -nan% + 0 requests in 10.09s, 0.00B read + Socket errors: connect 0, read 2000, write 0, timeout 0 +Requests/sec: 0.00 +Transfer/sec: 0.00B +``` + +## ASGI Gunicorn Uvicorn, with workers -Xgil=0 + +### JSON performance +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/json + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 178.74ms 339.44ms 2.00s 93.94% + Req/Sec 383.85 379.53 1.18k 70.97% + 28846 requests in 10.10s, 7.76MB read + Socket errors: connect 0, read 0, write 0, timeout 2256 +Requests/sec: 2856.05 +Transfer/sec: 786.55KB +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/json + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 698.98ms 384.09ms 1.98s 69.18% + Req/Sec 148.62 121.28 1.67k 73.84% + 27942 requests in 10.06s, 7.51MB read + Socket errors: connect 0, read 0, write 0, timeout 112 +Requests/sec: 2776.16 +Transfer/sec: 764.55KB +``` + +### Queries returned as JSON +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/json_query + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 879.42ms 533.47ms 2.00s 54.36% + Req/Sec 107.06 93.71 600.00 78.11% + 20292 requests in 10.08s, 4.90MB read + Socket errors: connect 0, read 0, write 0, timeout 948 +Requests/sec: 2013.20 +Transfer/sec: 497.43KB +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/json_query + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 957.97ms 309.21ms 1.85s 71.55% + Req/Sec 113.84 124.82 660.00 86.43% + 16727 requests in 10.10s, 4.04MB read + Socket errors: connect 0, read 0, write 0, timeout 1428 +Requests/sec: 1656.08 +Transfer/sec: 409.19KB +``` + +### Queries returned as HTML +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/template_query + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.13s 314.86ms 1.96s 64.61% + Req/Sec 87.98 88.27 730.00 86.57% + 14841 requests in 10.10s, 8.14MB read + Socket errors: connect 0, read 0, write 0, timeout 1182 +Requests/sec: 1470.13 +Transfer/sec: 825.54KB +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/template_query + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 946.53ms 420.16ms 2.00s 78.18% + Req/Sec 100.77 112.74 727.00 86.11% + 14171 requests in 10.08s, 7.77MB read + Socket errors: connect 0, read 0, write 0, timeout 3069 +Requests/sec: 1405.21 +Transfer/sec: 789.06KB +``` + +### Simulate a request 1s inside the server, then return a JSON +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/gateway_1s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.45s 262.79ms 2.00s 59.58% + Req/Sec 97.63 113.74 800.00 86.95% + 11344 requests in 10.10s, 2.90MB read + Socket errors: connect 0, read 0, write 0, timeout 2651 +Requests/sec: 1123.38 +Transfer/sec: 294.01KB +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/gateway_1s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.42s 244.45ms 2.00s 68.50% + Req/Sec 92.13 104.20 0.91k 87.93% + 12180 requests in 10.10s, 3.11MB read + Socket errors: connect 0, read 0, write 0, timeout 1337 +Requests/sec: 1206.22 +Transfer/sec: 315.69KB +``` + +### Simulate a request 3s inside the server, then return a JSON +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/gateway_3s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 106.23 141.99 0.86k 87.46% + 4721 requests in 10.06s, 1.21MB read + Socket errors: connect 0, read 0, write 0, timeout 4721 +Requests/sec: 469.39 +Transfer/sec: 122.87KB +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/gateway_3s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.35s 212.12ms 1.94s 70.02% + Req/Sec 85.03 86.76 737.00 91.33% + 13657 requests in 10.10s, 3.49MB read + Socket errors: connect 0, read 0, write 0, timeout 176 +Requests/sec: 1352.33 +Transfer/sec: 353.93KB +``` + +### Simulate a request 10s inside the server, then return a JSON +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/gateway_10s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 145.50 282.27 717.00 83.33% + 157 requests in 10.05s, 41.09KB read + Socket errors: connect 0, read 0, write 0, timeout 157 +Requests/sec: 15.63 +Transfer/sec: 4.09KB +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/gateway_10s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 100.00% + 5 requests in 10.10s, 1.31KB read + Socket errors: connect 0, read 0, write 0, timeout 5 +Requests/sec: 0.50 +Transfer/sec: 132.68B +``` + +### Brotli +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/brotli + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.43s 409.09ms 2.00s 68.74% + Req/Sec 24.95 27.19 210.00 89.87% + 3057 requests in 10.10s, 19.54MB read + Socket errors: connect 0, read 172, write 0, timeout 2190 +Requests/sec: 302.78 +Transfer/sec: 1.94MB +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/brotli + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.36s 405.31ms 1.95s 61.58% + Req/Sec 31.40 104.70 0.92k 97.06% + 1287 requests in 10.09s, 8.23MB read + Socket errors: connect 0, read 1142, write 0, timeout 920 +Requests/sec: 127.49 +Transfer/sec: 834.41KB +``` + +## ASGI Gunicorn Uvicorn, with workers -Xgil=1 + +### JSON performance +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/json + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 552.16ms 366.20ms 2.00s 80.77% + Req/Sec 113.62 102.55 740.00 76.80% + 18140 requests in 10.10s, 4.88MB read + Socket errors: connect 0, read 0, write 0, timeout 4102 +Requests/sec: 1796.89 +Transfer/sec: 494.87KB +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/json + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 806.98ms 418.05ms 2.00s 62.49% + Req/Sec 102.03 94.98 580.00 78.75% + 15685 requests in 10.09s, 4.22MB read + Socket errors: connect 0, read 0, write 0, timeout 1028 +Requests/sec: 1554.58 +Transfer/sec: 428.19KB +``` + +### Queries returned as JSON +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/json_query + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.09s 604.32ms 2.00s 56.31% + Req/Sec 63.52 59.24 450.00 82.75% + 9382 requests in 10.06s, 2.26MB read + Socket errors: connect 0, read 0, write 0, timeout 2097 +Requests/sec: 932.46 +Transfer/sec: 230.38KB +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/json_query + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.25s 471.66ms 2.00s 66.91% + Req/Sec 65.14 62.76 464.00 83.21% + 10158 requests in 10.04s, 2.45MB read + Socket errors: connect 0, read 0, write 0, timeout 3876 +Requests/sec: 1011.50 +Transfer/sec: 249.96KB +``` + +### Queries returned as HTML +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/template_query + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.33s 483.35ms 2.00s 66.95% + Req/Sec 41.79 37.81 525.00 86.85% + 6981 requests in 10.10s, 3.83MB read + Socket errors: connect 0, read 0, write 0, timeout 2806 +Requests/sec: 691.23 +Transfer/sec: 388.17KB +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/template_query + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.36s 403.77ms 2.00s 65.89% + Req/Sec 55.69 61.10 383.00 86.28% + 7921 requests in 10.06s, 4.34MB read + Socket errors: connect 0, read 0, write 0, timeout 3978 +Requests/sec: 787.03 +Transfer/sec: 441.94KB +``` + +### Simulate a request 1s inside the server, then return a JSON +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/gateway_1s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.49s 245.81ms 2.00s 66.50% + Req/Sec 66.44 74.91 490.00 89.22% + 9192 requests in 10.07s, 2.35MB read + Socket errors: connect 0, read 0, write 0, timeout 3380 +Requests/sec: 912.86 +Transfer/sec: 238.94KB +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/gateway_1s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.44s 269.68ms 2.00s 60.46% + Req/Sec 67.22 70.76 590.00 86.94% + 10597 requests in 10.08s, 2.71MB read + Socket errors: connect 0, read 0, write 0, timeout 2774 +Requests/sec: 1051.56 +Transfer/sec: 275.21KB +``` + +### Simulate a request 3s inside the server, then return a JSON +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/gateway_3s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 65.69 78.46 505.00 88.74% + 4381 requests in 10.05s, 1.12MB read + Socket errors: connect 0, read 0, write 0, timeout 4381 +Requests/sec: 436.10 +Transfer/sec: 114.14KB +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/gateway_3s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.48s 236.21ms 2.00s 63.70% + Req/Sec 62.13 70.03 530.00 88.32% + 9284 requests in 10.07s, 2.37MB read + Socket errors: connect 0, read 0, write 0, timeout 3064 +Requests/sec: 922.30 +Transfer/sec: 241.41KB +``` + +### Simulate a request 10s inside the server, then return a JSON +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/gateway_10s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 2.00 0.82 3.00 50.00% + 94 requests in 10.10s, 25.09KB read + Socket errors: connect 0, read 0, write 0, timeout 94 +Requests/sec: 9.30 +Transfer/sec: 2.48KB +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/gateway_10s + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 0.00us 0.00us 0.00us -nan% + Req/Sec 0.00 0.00 0.00 100.00% + 3 requests in 10.08s, 804.00B read + Socket errors: connect 0, read 0, write 0, timeout 3 +Requests/sec: 0.30 +Transfer/sec: 79.74B +``` + +### Brotli +#### Sync + +```bash +Running 10s test @ http://localhost:8000/myapp/sync/brotli + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 1.17s 540.89ms 2.00s 57.58% + Req/Sec 22.56 30.03 363.00 90.41% + 2575 requests in 10.10s, 16.46MB read + Socket errors: connect 0, read 0, write 0, timeout 1948 +Requests/sec: 254.92 +Transfer/sec: 1.63MB +``` + +#### Async + +```bash +Running 10s test @ http://localhost:8000/myapp/async/brotli + 20 threads and 2000 connections + Thread Stats Avg Stdev Max +/- Stdev + Latency 975.61ms 236.78ms 1.56s 72.91% + Req/Sec 45.34 69.98 434.00 91.47% + 1384 requests in 10.09s, 8.85MB read + Socket errors: connect 0, read 1144, write 0, timeout 775 +Requests/sec: 137.11 +Transfer/sec: 0.88MB +``` diff --git a/benchmarks/http/bench.sh b/benchmarks/http/bench.sh index b862c00f5..0d69ee352 100755 --- a/benchmarks/http/bench.sh +++ b/benchmarks/http/bench.sh @@ -5,12 +5,12 @@ echo "Async..." echo "Async" > $file echo "" >> $file -python async.py >> $file +PYTHON_GIL=0 python async.py -Xgil=0 >> $file echo "" >> $file echo "Sync..." echo "Sync" >> $file -python sync.py >> $file +PYTHON_GIL=0 python sync.py -Xgil=0 >> $file echo "Done" diff --git a/benchmarks/http/bench2.sh b/benchmarks/http/bench2.sh index 0ed2e974c..3b2d8d4d2 100755 --- a/benchmarks/http/bench2.sh +++ b/benchmarks/http/bench2.sh @@ -5,12 +5,12 @@ echo "Async..." echo "Async" > $file echo "" >> $file -python async2.py >> $file +PYTHON_GIL=0 python async2.py -Xgil=0 >> $file echo "" >> $file echo "Sync..." echo "Sync" >> $file -python sync2.py >> $file +PYTHON_GIL=0 python sync2.py -Xgil=0 >> $file echo "Done" diff --git a/benchmarks/threads/main.py b/benchmarks/threads/main.py new file mode 100644 index 000000000..aaa347b57 --- /dev/null +++ b/benchmarks/threads/main.py @@ -0,0 +1,47 @@ +import sys +import threading +import time + +# Shared resource +shared_resource = 0 + +# Lock for synchronization +# lock = threading.Lock() + + +# Define a function that will be executed by each thread +def worker(): + global shared_resource + accum = 0 + for n in range(10000000): # Perform some computation + # with lock: + accum += n + shared_resource += accum + + +def run_threads(num_threads): + start_time = time.time() + + # Create a list to store references to the threads + threads = [] + + # Create and start the specified number of threads + for _ in range(num_threads): + thread = threading.Thread(target=worker) + thread.start() + threads.append(thread) + + # Wait for all threads to complete + for thread in threads: + thread.join() + + end_time = time.time() + elapsed_time = end_time - start_time + print(f"Ran {num_threads} threads cooperatively in {elapsed_time:.2f} seconds") + + +if __name__ == '__main__': + print(f"gil_enabled={sys._is_gil_enabled()}") + # Run the benchmark with different numbers of threads + for num_threads in [1, 2, 4, 8, 20]: + run_threads(num_threads) diff --git a/breathecode/mentorship/models.py b/breathecode/mentorship/models.py index 41c532011..030f1da21 100644 --- a/breathecode/mentorship/models.py +++ b/breathecode/mentorship/models.py @@ -12,73 +12,127 @@ from breathecode.notify.models import SlackChannel from breathecode.utils.validators.language import validate_language_code -# settings customizable for each academy -# class AcademySettings(models.Model): -# is_video_streaming_active = models.BooleanField(default=False) -# academy = models.OneToOneField(Academy, on_delete=models.CASCADE) -# @staticmethod -# def get(pk): -# settings = AcademySettings.objects.filter(academy__id=pk).first() -# # lets create the settings if they dont exist for this academy -# if settings is None: -# settings = AcademySettings.objects.create(academy=pk) -# return settings -# def warnings(self): -# # return a dictionary with a list of the fields and warning messages related to them -# # for example: { "is_video_streaming_active": "Please settup a video streaming" } -# return {} -# def errors(self): -# # return a dictionary with a list of the fields and errors messages related to them -# return {} - -DRAFT = 'DRAFT' -ACTIVE = 'ACTIVE' -UNLISTED = 'UNLISTED' -INNACTIVE = 'INNACTIVE' -MENTORSHIP_STATUS = ( - (DRAFT, 'Draft'), - (ACTIVE, 'Active'), - (UNLISTED, 'Unlisted'), - (INNACTIVE, 'Innactive'), -) + +class VideoProvider(models.TextChoices): + DAILY = ('DAILY', 'Daily') + GOOGLE_MEET = ('GOOGLE_MEET', 'Google Meet') + + +MENTORSHIP_SETTINGS = { + 'duration': timedelta(hours=1), + 'max_duration': timedelta(hours=2), + 'missed_meeting_duration': timedelta(minutes=10), + 'language': 'en', + 'allow_mentee_to_extend': True, + 'allow_mentors_to_extend': True, +} + + +class AcademyMentorshipSettings(models.Model): + VideoProvider = VideoProvider + + academy = models.OneToOneField(Academy, on_delete=models.CASCADE) + duration = models.DurationField(default=MENTORSHIP_SETTINGS['duration'], + help_text='Default duration for mentorship sessions of this service') + + max_duration = models.DurationField( + default=MENTORSHIP_SETTINGS['max_duration'], + help_text='Maximum allowed duration or extra time, make it 0 for unlimited meetings') + + missed_meeting_duration = models.DurationField( + default=MENTORSHIP_SETTINGS['missed_meeting_duration'], + help_text='Duration that will be paid when the mentee doesn\'t come to the session') + + language = models.CharField(max_length=5, + default=MENTORSHIP_SETTINGS['language'], + validators=[validate_language_code], + help_text='ISO 639-1 language code + ISO 3166-1 alpha-2 country code, e.g. en-US') + + allow_mentee_to_extend = models.BooleanField(default=MENTORSHIP_SETTINGS['allow_mentee_to_extend'], + help_text='If true, mentees will be able to extend mentorship session') + allow_mentors_to_extend = models.BooleanField( + default=MENTORSHIP_SETTINGS['allow_mentors_to_extend'], + help_text='If true, mentors will be able to extend mentorship session') + + video_provider = models.CharField(max_length=15, choices=VideoProvider, default=VideoProvider.GOOGLE_MEET) + + created_at = models.DateTimeField(auto_now_add=True, editable=False) + updated_at = models.DateTimeField(auto_now=True, editable=False) + + def __str__(self): + return self.academy.name + + def clean(self) -> None: + return super().clean() + + def save(self, **kwargs) -> None: + return super().save(**kwargs) class MentorshipService(models.Model): + VideoProvider = VideoProvider + + class Status(models.TextChoices): + DRAFT = ('DRAFT', 'Draft') + ACTIVE = ('ACTIVE', 'Active') + UNLISTED = ('UNLISTED', 'Unlisted') + INNACTIVE = ('INNACTIVE', 'Innactive') + slug = models.SlugField(max_length=150, unique=True) name = models.CharField(max_length=150) logo_url = models.CharField(max_length=150, default=None, blank=True, null=True) description = models.TextField(max_length=500, default=None, blank=True, null=True) - duration = models.DurationField(default=timedelta(hours=1), - help_text='Default duration for mentorship sessions of this service') + duration = models.DurationField(blank=True, help_text='Default duration for mentorship sessions of this service') max_duration = models.DurationField( - default=timedelta(hours=2), - help_text='Maximum allowed duration or extra time, make it 0 for unlimited meetings') + blank=True, help_text='Maximum allowed duration or extra time, make it 0 for unlimited meetings') missed_meeting_duration = models.DurationField( - default=timedelta(minutes=10), - help_text='Duration that will be paid when the mentee doesn\'t come to the session') + blank=True, help_text='Duration that will be paid when the mentee doesn\'t come to the session') - status = models.CharField(max_length=15, choices=MENTORSHIP_STATUS, default=DRAFT) + status = models.CharField(max_length=15, choices=Status, default=Status.DRAFT) language = models.CharField(max_length=5, - default='en', + blank=True, validators=[validate_language_code], help_text='ISO 639-1 language code + ISO 3166-1 alpha-2 country code, e.g. en-US') - allow_mentee_to_extend = models.BooleanField(default=True, + allow_mentee_to_extend = models.BooleanField(blank=True, help_text='If true, mentees will be able to extend mentorship session') allow_mentors_to_extend = models.BooleanField( - default=True, help_text='If true, mentors will be able to extend mentorship session') + blank=True, help_text='If true, mentors will be able to extend mentorship session') academy = models.ForeignKey(Academy, on_delete=models.CASCADE) + video_provider = models.CharField(max_length=15, choices=VideoProvider, blank=True) created_at = models.DateTimeField(auto_now_add=True, editable=False) updated_at = models.DateTimeField(auto_now=True, editable=False) def __str__(self): - return f'{self.name} ({self.id})' + return f'{self.name} ({self.slug})' + + def clean(self) -> None: + fetched = False + academy_settings = None + for field, value in MENTORSHIP_SETTINGS.items(): + current = getattr(self, field) + if current is None: + if fetched is False: + fetched = True + academy_settings = AcademyMentorshipSettings.objects.filter(academy=self.academy).first() + + if academy_settings: + academy_value = getattr(academy_settings, field) + setattr(self, field, academy_value) + + else: + setattr(self, field, value) + + return super().clean() + + def save(self, **kwargs) -> None: + return super().save(**kwargs) class SupportChannel(models.Model): @@ -91,13 +145,11 @@ class SupportChannel(models.Model): updated_at = models.DateTimeField(auto_now=True, editable=False) -INVITED = 'INVITED' -MENTOR_STATUS = ( - (INVITED, 'Invited'), - (ACTIVE, 'Active'), - (UNLISTED, 'Unlisted'), - (INNACTIVE, 'Innactive'), -) +class MentorStatus(models.TextChoices): + INVITED = ('INVITED', 'Invited') + ACTIVE = ('ACTIVE', 'Active') + UNLISTED = ('UNLISTED', 'Unlisted') + INNACTIVE = ('INNACTIVE', 'Innactive') class SupportAgent(models.Model): @@ -109,9 +161,9 @@ class SupportAgent(models.Model): unique=True, help_text='Used for inviting the user to become a support agent') status = models.CharField(max_length=15, - choices=MENTOR_STATUS, - default=INVITED, - help_text=f'Options are: {", ".join([key for key,label in MENTOR_STATUS])}') + choices=MentorStatus, + default=MentorStatus.INVITED, + help_text=f'Options are: {", ".join([key for key,label in MentorStatus.choices])}') email = models.CharField(blank=True, max_length=150, @@ -176,9 +228,9 @@ class MentorProfile(models.Model): help_text='What syllabis is this mentor going to be menting to?') status = models.CharField(max_length=15, - choices=MENTOR_STATUS, - default=INVITED, - help_text=f'Options are: {", ".join([key for key,label in MENTOR_STATUS])}') + choices=MentorStatus, + default=MentorStatus.INVITED, + help_text=f'Options are: {", ".join([key for key,label in MentorStatus.choices])}') email = models.CharField(blank=True, max_length=150, diff --git a/breathecode/mentorship/tasks.py b/breathecode/mentorship/tasks.py index de13eb5e8..ab7e235ae 100644 --- a/breathecode/mentorship/tasks.py +++ b/breathecode/mentorship/tasks.py @@ -1,18 +1,25 @@ import logging import os -from typing import Any +from datetime import datetime, timedelta +from typing import Any, Optional +import pytz import requests from celery import shared_task +from google.apps.meet_v2.types import Space, SpaceConfig from task_manager.core.exceptions import AbortTask from task_manager.django.decorators import task import breathecode.activity.tasks as tasks_activity +import breathecode.notify.actions as notify_actions +from breathecode.authenticate.actions import get_user_settings from breathecode.services.calendly import Calendly from breathecode.services.calendly.actions import invitee_created +from breathecode.services.google_meet.google_meet import GoogleMeet from breathecode.utils.decorators import TaskPriority +from breathecode.utils.i18n import translation -from .models import CalendlyOrganization, CalendlyWebhook, MentorProfile +from .models import CalendlyOrganization, CalendlyWebhook, MentorProfile, MentorshipSession logger = logging.getLogger(__name__) @@ -100,3 +107,172 @@ def check_mentorship_profile(mentor_id: int, **_: Any): mentor.availability_report = status mentor.save() + + +def localize_date(dt: datetime, timezone: str = 'UTC'): + # Localize the datetime object to the specified timezone + local_tz = pytz.timezone(timezone) + localized_dt = dt.astimezone(local_tz) + + # Format the localized datetime object to a string + localized_string = localized_dt.strftime('%Y-%m-%d %I:%M:%S %p') + + return localized_string + + +@task(bind=False, priority=TaskPriority.STUDENT.value) +def cancellate_conference_on_google_meet(session_id: int, **_: Any): + """Cancellate conference on google meet for a mentorship session.""" + + def get_translations(lang: str) -> dict[str, str]: + return { + 'organizers': translation(lang, en='Organizers', es='Organizadores'), + 'invitees': translation(lang, en='Invitees', es='Invitados'), + 'date': translation(lang, en='Date', es='Fecha'), + 'duration': translation(lang, en='Duration', es='Duración'), + 'location': translation(lang, en='Location', es='Ubicación'), + 'enter': translation(lang, en='Enter', es='Entrar'), + 'cancel': translation(lang, en='Cancel', es='Cancelar'), + 'details': translation(lang, en='Details', es='Detalles'), + 'qaa': translation(lang, en='Questions and Answers', es='Preguntas y Respuestas'), + } + + session = MentorshipSession.objects.filter(id=session_id, service__video_provider='GOOGLE_MEET').first() + + if session is None: + raise AbortTask(f'Mentorship session {session_id} not found') + + if session.mentee is None: + raise AbortTask(f'This session doesn\'t have a mentee') + + if not session.service: + raise AbortTask(f'Mentorship session doesn\'t have a service associated with it') + + mentor = session.mentor + mentee = session.mentee + + meet = GoogleMeet() + title = (f'{session.service.name} {session.id} | ' + f'{mentor.user.first_name} {mentor.user.last_name} | ' + f'{mentee.first_name} {mentee.last_name}') + + meet.end_active_conference(name=title) + + +@task(bind=False, priority=TaskPriority.STUDENT.value) +def create_room_on_google_meet(session_id: int, **_: Any): + """Create a room on google meet for a mentorship session.""" + + def get_translations(lang: str) -> dict[str, str]: + return { + 'organizers': translation(lang, en='Organizers', es='Organizadores'), + 'invitees': translation(lang, en='Invitees', es='Invitados'), + 'date': translation(lang, en='Date', es='Fecha'), + 'duration': translation(lang, en='Duration', es='Duración'), + 'location': translation(lang, en='Location', es='Ubicación'), + 'enter': translation(lang, en='Enter', es='Entrar'), + 'cancel': translation(lang, en='Cancel', es='Cancelar'), + 'details': translation(lang, en='Details', es='Detalles'), + 'qaa': translation(lang, en='Questions and Answers', es='Preguntas y Respuestas'), + } + + session = MentorshipSession.objects.filter(id=session_id, service__video_provider='GOOGLE_MEET').first() + + if session is None: + raise AbortTask(f'Mentorship session {session_id} not found') + + if session.mentee is None: + raise AbortTask(f'This session doesn\'t have a mentee') + + if not session.service: + raise AbortTask(f'Mentorship session doesn\'t have a service associated with it') + + if session.starts_at is None: + raise AbortTask(f'Mentorship session {session_id} doesn\'t have a start date') + + if session.ends_at is None: + raise AbortTask(f'Mentorship session {session_id} doesn\'t have an end date') + + mentor = session.mentor + mentee = session.mentee + + meet = GoogleMeet() + title = (f'{session.service.name} {session.id} | ' + f'{mentor.user.first_name} {mentor.user.last_name} | ' + f'{mentee.first_name} {mentee.last_name}') + s = Space( + name=title, + config=SpaceConfig(access_type=SpaceConfig.AccessType.OPEN), + ) + space = meet.create_space(space=s) + + mentor_lang = get_user_settings(mentor.user.id).lang + mentee_lang = get_user_settings(mentee.id).lang + + answers = session.questions_and_answers or [] + api_url = os.getenv('API_URL', '') + if api_url.endswith('/'): + api_url = api_url[:-1] + + academy = None + if session.mentor.academy and session.mentor.academy.timezone: + academy = session.mentor.academy + + if academy is None and session.service.academy and session.service.academy.timezone: + academy = session.service.academy + + timezone = academy.timezone if academy is not None else 'UTC' + + try: + local_tz = pytz.timezone(timezone) + + except pytz.exceptions.UnknownTimeZoneError: + local_tz = pytz.timezone('UTC') + + localized_dt = session.starts_at.astimezone(local_tz) + + # Format the localized datetime object to a string + localized_string = localized_dt.strftime('%Y-%m-%d %I:%M:%S %p') + + data = { + 'title': session.service.name, + 'description': session.service.description, + 'meet': { + 'url': space.meeting_uri, + 'cancellation_url': api_url + f'/mentor/session/{session.id}/cancel', + 'date': session.starts_at.isoformat(), + 'preformatted_date': localized_string, + 'duration': session.ends_at - session.starts_at, + }, + 'organizers': [ + { + 'name': f'{mentor.user.first_name} {mentor.user.last_name}', + }, + ], + 'invitees': [ + { + 'name': f'{mentee.first_name} {mentee.last_name}', + 'email': mentee.email, + }, + ], + 'answers': answers, + } + + if mentor_lang == mentee_lang: + emails = [mentor.user.email, mentee.email] + + data['translations'] = get_translations(mentor_lang) + notify_actions.send_email_message('meet_notification', emails, data) + + else: + emails = [mentor.user.email] + notify_actions.send_email_message('meet_notification', emails, { + **data, + 'translations': get_translations(mentor_lang), + }) + + emails = [mentee.email] + notify_actions.send_email_message('meet_notification', emails, { + **data, + 'translations': get_translations(mentee_lang), + }) diff --git a/breathecode/mentorship/templates/meet_notification.html b/breathecode/mentorship/templates/meet_notification.html new file mode 100644 index 000000000..1cb9fffc1 --- /dev/null +++ b/breathecode/mentorship/templates/meet_notification.html @@ -0,0 +1,181 @@ + + + +
+ + +{{ description }}
+ +{{ form.answer }}+
{{ description }}
+ +Internal meeting notes
", +# "status": "active", +# "start_time": "2019-08-24T14:15:22.123456Z", +# "end_time": "2019-08-24T14:15:22.123456Z", +# "event_type": "https://api.calendly.com/event_types/GBGBDCAADAEDCRZ2", +# "location": { +# "type": "physical", +# "location": "string", +# "additional_info": "string" +# }, +# "invitees_counter": { +# "total": 0, +# "active": 0, +# "limit": 0 +# }, +# "created_at": "2019-01-02T03:04:05.678123Z", +# "updated_at": "2019-01-02T03:04:05.678123Z", +# "event_memberships": [ +# { +# "user": "https://api.calendly.com/users/GBGBDCAADAEDCRZ2", +# "user_email": "user@example.com", +# "user_name": "John Smith" +# } +# ], +# "event_guests": [ +# { +# "email": "user@example.com", +# "created_at": "2019-08-24T14:15:22.123456Z", +# "updated_at": "2019-08-24T14:15:22.123456Z" +# } +# ] +# } +# } + def invitee_created(client, webhook, payload: dict): # lazyload to fix circular import - from breathecode.mentorship.models import MentorshipService, MentorProfile, MentorshipSession + from breathecode.mentorship.models import MentorProfile, MentorshipService, MentorshipSession + from breathecode.mentorship.tasks import create_room_on_google_meet + # from breathecode.events.actions import update_or_create_event # payload = payload['resource'] @@ -80,4 +166,7 @@ def invitee_created(client, webhook, payload: dict): session.calendly_uuid = event_uuid session.save() + if session.service and session.service.video_provider == MentorshipService.VideoProvider.GOOGLE_MEET: + create_room_on_google_meet.delay(session.id) + return session diff --git a/breathecode/services/google_meet/__init__.py b/breathecode/services/google_meet/__init__.py new file mode 100644 index 000000000..1919cdab5 --- /dev/null +++ b/breathecode/services/google_meet/__init__.py @@ -0,0 +1 @@ +from .google_meet import * # noqa: F401 diff --git a/breathecode/services/google_meet/google_meet.py b/breathecode/services/google_meet/google_meet.py index 2b0d1909f..9fe838196 100644 --- a/breathecode/services/google_meet/google_meet.py +++ b/breathecode/services/google_meet/google_meet.py @@ -1,7 +1,82 @@ -from typing import Optional +from typing import Optional, TypedDict, Unpack +import google.apps.meet_v2.services.conference_records_service.pagers as pagers from asgiref.sync import async_to_sync from google.apps import meet_v2 +from google.apps.meet_v2.types import Space +from google.protobuf.field_mask_pb2 import FieldMask +from google.protobuf.timestamp_pb2 import Timestamp + +__all__ = ['GoogleMeet'] + + +class CreateSpaceRequest(TypedDict): + space: Space + + +class EndActiveConferenceRequest(TypedDict): + name: str + + +class GetConferenceRecordRequest(TypedDict): + name: str + + +class GetParticipantRequest(TypedDict): + name: str + + +class GetParticipantSessionRequest(TypedDict): + name: str + + +class GetRecordingRequest(TypedDict): + name: str + + +class GetSpaceRequest(TypedDict): + name: str + + +class UpdateSpaceRequest(TypedDict): + space: Space + update_mask: FieldMask + + +class GetTranscriptRequest(TypedDict): + name: str + + +class ListConferenceRecordsRequest(TypedDict): + page_size: int + page_token: str + filter: str # in EBNF format, space.meeting_code, space.name, start_time and end_time + + +class ListRecordingsRequest(TypedDict): + parent: str + page_size: int + page_token: str + + +class ListParticipantSessionsRequest(TypedDict): + parent: str + page_size: int + page_token: str + filter: str # in EBNF format, start_time and end_time + + +class ListTranscriptsRequest(TypedDict): + parent: str + page_size: int + page_token: str + + +class ListParticipantsRequest(TypedDict): + parent: str + page_size: int + page_token: str + filter: str # in EBNF format, start_time and end_time class GoogleMeet: @@ -9,9 +84,12 @@ class GoogleMeet: _conference_records_service_client: Optional[meet_v2.ConferenceRecordsServiceAsyncClient] def __init__(self): + from breathecode.setup import resolve_gcloud_credentials + + resolve_gcloud_credentials() + self._spaces_service_client = None self._conference_records_service_client = None - pass async def spaces_service_client(self): if self._spaces_service_client is None: @@ -25,31 +103,28 @@ async def conference_records_service_client(self): return self._conference_records_service_client - async def acreate_meeting(self, **kwargs): + async def acreate_space(self, **kwargs: Unpack[CreateSpaceRequest]) -> meet_v2.Space: # Create a client client = await self.spaces_service_client() # Initialize request argument(s) - request = meet_v2.CreateSpaceRequest() + request = meet_v2.CreateSpaceRequest(**kwargs) # Make the request - response = await client.create_space(request=request) - - # Handle the response - print(response) + return await client.create_space(request=request) @async_to_sync - async def create_meeting(self): - return await self.acreate_meeting() + async def create_space(self, **kwargs: Unpack[CreateSpaceRequest]) -> meet_v2.Space: + return await self.acreate_space(**kwargs) - async def aget_meeting(self): + async def aget_space(self, **kwargs: Unpack[GetSpaceRequest]) -> meet_v2.Space: # Create a client client = await self.spaces_service_client() # Initialize request argument(s) - request = meet_v2.GetSpaceRequest() + request = meet_v2.GetSpaceRequest(**kwargs) # Make the request response = await client.get_space(request=request) @@ -58,15 +133,15 @@ async def aget_meeting(self): print(response) @async_to_sync - async def get_meeting(self): - return await self.aget_meeting() + async def get_space(self, **kwargs: Unpack[GetSpaceRequest]) -> meet_v2.Space: + return await self.aget_space(**kwargs) - async def aupdate_space(self): + async def aupdate_space(self, **kwargs: Unpack[UpdateSpaceRequest]) -> meet_v2.Space: # Create a client client = await self.spaces_service_client() # Initialize request argument(s) - request = meet_v2.UpdateSpaceRequest() + request = meet_v2.UpdateSpaceRequest(**kwargs) # Make the request response = await client.update_space(request=request) @@ -75,117 +150,90 @@ async def aupdate_space(self): print(response) @async_to_sync - async def update_space(self): - return await self.aupdate_space() + async def update_space(self, **kwargs: Unpack[UpdateSpaceRequest]) -> meet_v2.Space: + return await self.aupdate_space(**kwargs) - async def aend_meeting(self, name: str): + async def aend_active_conference(self, **kwargs: Unpack[EndActiveConferenceRequest]) -> None: # Create a client client = await self.spaces_service_client() # Initialize request argument(s) - request = meet_v2.EndActiveConferenceRequest(name=name) + request = meet_v2.EndActiveConferenceRequest(**kwargs) # Make the request - await client.end_active_conference(request=request) + return await client.end_active_conference(request=request) @async_to_sync - async def end_meeting(self, name: str): - return await self.aend_meeting(name) + async def end_active_conference(self, **kwargs: Unpack[EndActiveConferenceRequest]) -> None: + return await self.aend_active_conference(**kwargs) - async def alist_participants(self, parent: str): + async def alist_participants(self, **kwargs: Unpack[ListParticipantsRequest]) -> pagers.ListParticipantsAsyncPager: # Create a client client = await self.conference_records_service_client() # Initialize request argument(s) - request = meet_v2.ListParticipantsRequest(parent=parent) + request = meet_v2.ListParticipantsRequest(**kwargs) # Make the request - page_result = client.list_participants(request=request) - - # Handle the response - async for response in page_result: - print(response) + return await client.list_participants(request=request) - @async_to_sync - async def list_participants(self, parent: str): - return await self.alist_participants(parent) - - async def aget_participant(self, name: str): + async def aget_participant(self, **kwargs: Unpack[GetParticipantRequest]) -> meet_v2.Participant: # Create a client client = await self.conference_records_service_client() # Initialize request argument(s) - request = meet_v2.GetParticipantRequest(name=name) + request = meet_v2.GetParticipantRequest(**kwargs) # Make the request - response = await client.get_participant(request=request) - - # Handle the response - print(response) + return await client.get_participant(request=request) @async_to_sync - async def get_participant(self, name: str): - return await self.aget_participant(name) + async def get_participant(self, **kwargs: Unpack[GetParticipantRequest]) -> meet_v2.Participant: + return await self.aget_participant(**kwargs) - async def alist_participant_sessions(self, parent: str): + async def alist_participant_sessions( + self, **kwargs: Unpack[ListParticipantSessionsRequest]) -> pagers.ListParticipantSessionsAsyncPager: # Create a client client = await self.conference_records_service_client() # Initialize request argument(s) - request = meet_v2.ListParticipantSessionsRequest(parent=parent) + request = meet_v2.ListParticipantSessionsRequest(**kwargs) # Make the request - page_result = client.list_participant_sessions(request=request) - - # Handle the response - async for response in page_result: - print(response) - - @async_to_sync - async def list_participant_sessions(self, parent: str): - return await self.alist_participant_sessions(parent) + return await client.list_participant_sessions(request=request) - async def aget_participant_session(self, name: str): + async def aget_participant_session(self, + **kwargs: Unpack[GetParticipantSessionRequest]) -> meet_v2.ParticipantSession: # Create a client client = await self.conference_records_service_client() # Initialize request argument(s) - request = meet_v2.GetParticipantSessionRequest(name=name) + request = meet_v2.GetParticipantSessionRequest(**kwargs) # Make the request - response = await client.get_participant_session(request=request) - - # Handle the response - print(response) + return await client.get_participant_session(request=request) @async_to_sync - async def get_participant_session(self, name: str): - return await self.aget_participant_session(name) + async def get_participant_session(self, + **kwargs: Unpack[GetParticipantSessionRequest]) -> meet_v2.ParticipantSession: + return await self.aget_participant_session(**kwargs) - async def alist_recordings(self, parent: str): + async def alist_recordings(self, **kwargs: Unpack[ListRecordingsRequest]) -> pagers.ListRecordingsAsyncPager: # Create a client client = await self.conference_records_service_client() # Initialize request argument(s) - request = meet_v2.ListRecordingsRequest(parent=parent) + request = meet_v2.ListRecordingsRequest(**kwargs) # Make the request - page_result = client.list_recordings(request=request) + return await client.list_recordings(request=request) - # Handle the response - async for response in page_result: - print(response) - - @async_to_sync - async def list_recordings(self, parent: str): - return await self.alist_recordings(parent) - - async def aget_recording(self, name: str): + async def aget_recording(self, **kwargs: Unpack[GetRecordingRequest]) -> meet_v2.Recording: # Create a client client = await self.conference_records_service_client() # Initialize request argument(s) - request = meet_v2.GetRecordingRequest(name=name) + request = meet_v2.GetRecordingRequest(**kwargs) # Make the request response = await client.get_recording(request=request) @@ -194,33 +242,25 @@ async def aget_recording(self, name: str): print(response) @async_to_sync - async def get_recording(self, name: str): - return await self.aget_recording(name) + async def get_recording(self, **kwargs: Unpack[GetRecordingRequest]) -> meet_v2.Recording: + return await self.aget_recording(**kwargs) - async def alist_transcripts(self, parent: str): + async def alist_transcripts(self, **kwargs: Unpack[ListTranscriptsRequest]) -> pagers.ListTranscriptsAsyncPager: # Create a client client = await self.conference_records_service_client() # Initialize request argument(s) - request = meet_v2.ListTranscriptsRequest(parent=parent) + request = meet_v2.ListTranscriptsRequest(**kwargs) # Make the request - page_result = client.list_transcripts(request=request) - - # Handle the response - async for response in page_result: - print(response) - - @async_to_sync - async def list_transcripts(self, parent: str): - return await self.alist_transcripts(parent) + return await client.list_transcripts(request=request) - async def aget_transcript(self, name: str): + async def aget_transcript(self, **kwargs: Unpack[GetTranscriptRequest]) -> meet_v2.Transcript: # Create a client client = await self.conference_records_service_client() # Initialize request argument(s) - request = meet_v2.GetTranscriptRequest(name=name) + request = meet_v2.GetTranscriptRequest(**kwargs) # Make the request response = await client.get_transcript(request=request) @@ -229,40 +269,30 @@ async def aget_transcript(self, name: str): print(response) @async_to_sync - async def get_transcript(self, name: str): - return await self.aget_transcript(name) + async def get_transcript(self, **kwargs: Unpack[GetTranscriptRequest]) -> meet_v2.Transcript: + return await self.aget_transcript(**kwargs) - async def alist_conference_records(self): + async def alist_conference_records( + self, **kwargs: Unpack[ListConferenceRecordsRequest]) -> pagers.ListConferenceRecordsAsyncPager: # Create a client client = await self.conference_records_service_client() # Initialize request argument(s) - request = meet_v2.ListConferenceRecordsRequest() + request = meet_v2.ListConferenceRecordsRequest(**kwargs) # Make the request - page_result = client.list_conference_records(request=request) + return await client.list_conference_records(request=request) - # Handle the response - async for response in page_result: - print(response) - - @async_to_sync - async def list_conference_records(self): - return await self.alist_conference_records() - - async def aget_conference_record(self, name: str): + async def aget_conference_record(self, **kwargs: Unpack[GetConferenceRecordRequest]) -> meet_v2.ConferenceRecord: # Create a client client = await self.conference_records_service_client() # Initialize request argument(s) - request = meet_v2.GetConferenceRecordRequest(name=name) + request = meet_v2.GetConferenceRecordRequest(**kwargs) # Make the request - response = await client.get_conference_record(request=request) - - # Handle the response - print(response) + return await client.get_conference_record(request=request) @async_to_sync - async def get_conference_record(self, name: str): - return await self.aget_conference_record(name) + async def get_conference_record(self, **kwargs: Unpack[GetConferenceRecordRequest]) -> meet_v2.ConferenceRecord: + return await self.aget_conference_record(**kwargs) diff --git a/breathecode/tests/mixins/generate_models_mixin/utils/create_models.py b/breathecode/tests/mixins/generate_models_mixin/utils/create_models.py index 2b286e4a0..210d466cf 100644 --- a/breathecode/tests/mixins/generate_models_mixin/utils/create_models.py +++ b/breathecode/tests/mixins/generate_models_mixin/utils/create_models.py @@ -1,9 +1,10 @@ import logging from typing import Any -from breathecode.tests.mixins.generate_models_mixin.exceptions import BadArgument from mixer.backend.django import mixer +from breathecode.tests.mixins.generate_models_mixin.exceptions import BadArgument + from .argument_parser import argument_parser __all__ = ['create_models'] @@ -44,8 +45,8 @@ def debug_mixer(attr, path, **kwargs): def create_models(attr, path, **kwargs): - # does not remove this line are use very often - # debug_mixer(attr, path, **kwargs) + # does not remove this line it's used very often + debug_mixer(attr, path, **kwargs) result = [ cycle(how_many).blend(path, **{ diff --git a/capyc/django/pytest/fixtures/database.py b/capyc/django/pytest/fixtures/database.py index 8da56a78e..bb691aef1 100644 --- a/capyc/django/pytest/fixtures/database.py +++ b/capyc/django/pytest/fixtures/database.py @@ -3,6 +3,7 @@ import functools import random import re +from copy import copy from typing import Any, Generator, final import pytest @@ -352,6 +353,12 @@ def create(cls, **models): pending = {} + keys = [*models.keys()] + + for key in keys: + if models[key] is None or models[key] == 0: + del models[key] + # get descriptors for model_alias, _value in models.items(): try: