db.model schema - string column without size to default to length 255

.github/workflows/pullreqeust_tests.yaml

@@ -19,9 +19,16 @@ jobs:
           poetry --version
           # Check if poetry lock file is current with pyproject.toml
           poetry check --lock
-          poetry install --with=dev
+          poetry install --with=dev --no-ansi
+
+      - name: Run db/test with MySQL docker
+        run: |
+          sudo apt update
+          sudo apt install -y mysql-client
+          cd arxiv && poetry run pytest auth/legacy/tests --db=mysql
+
       - name: Run arxiv-base tests with coverage
-        run: poetry run pytest --cov=arxiv.base fourohfour --cov-fail-under=67 arxiv/base fourohfour
+        run: poetry run pytest --cov=arxiv.base fourohfour --cov-fail-under=75 arxiv/base fourohfour
       # - name: Check Types
       # TODO The types are in bad shape and need to be fixed
       #   run: poetry run mypy --exclude "test*" -p arxiv
@@ -31,16 +38,17 @@ jobs:
           sudo add-apt-repository -y ppa:mozillateam/ppa
           sudo apt update
           sudo apt install -y firefox-esr
-          wget https://github.com/mozilla/geckodriver/releases/latest/download/geckodriver-v0.35.0-linux64.tar.gz                                                                                                                        
-          tar -xvzf geckodriver-v0.35.0-linux64.tar.gz                                                                                                                                                                                     
-          sudo mv geckodriver /usr/local/bin/                                                                                                                                                                                              
+          wget https://github.com/mozilla/geckodriver/releases/latest/download/geckodriver-v0.35.0-linux64.tar.gz
+          tar -xvzf geckodriver-v0.35.0-linux64.tar.gz
+          sudo mv geckodriver /usr/local/bin/
           sudo chmod +x /usr/local/bin/geckodriver
 
       - name: Run other tests
         # These tests are split out because their coverage is low
-        run: poetry run pytest --cov=arxiv --cov-fail-under=25 arxiv
+        run: poetry run pytest --cov=arxiv --cov-fail-under=40 arxiv --ignore=arxiv/base --ignore=arxiv/auth/legacy --db=sqlite
       #- name: Check Doc Style
       #  run: poetry run pydocstyle --convention=numpy --add-ignore=D401 arxiv
 
       - name: Run App Tests
         run: poetry run python tests/run_app_tests.py
+

.gitignore

@@ -131,3 +131,7 @@ fastly_hourly_stats.ini
 
 foo.json
 test.db-journal
+
+arxiv/db/.autogen_models.py
+/arxiv/db/autogen_models.py
+/arxiv/db/near_models.py

Makefile

@@ -0,0 +1,41 @@
+PROD_DB_PROXY_PORT := 2021
+
+.PHONY: prod-proxy test
+
+default: venv/bin/poetry /usr/bin/firefox-esr /usr/local/bin/geckodriver
+
+venv/bin/poetry: venv/bin/pip
+	. venv/bin/activate && pip install poetry
+
+venv/bin/pip: venv
+	. venv/bin/activate && pip install --upgrade pip
+
+venv:
+	python3.11 -m venv venv
+
+/usr/bin/firefox-esr:
+	sudo add-apt-repository -y ppa:mozillateam/ppa
+	sudo apt update
+	sudo apt install -y firefox-esr
+
+/usr/local/bin/geckodriver:
+	wget https://github.com/mozilla/geckodriver/releases/latest/download/geckodriver-v0.35.0-linux64.tar.gz
+	tar -xvzf geckodriver-v0.35.0-linux64.tar.gz
+	sudo mv geckodriver /usr/local/bin/
+	sudo chmod +x /usr/local/bin/geckodriver
+
+~/.arxiv:
+	mkdir ~/.arxiv
+
+ ~/.arxiv/arxiv-db-prod-readonly:  ~/.arxiv
+	op item get as76tjqjnr5wduypygusvfyy34 --fields username,password --reveal | python3 -c "import sys;import urllib.parse; unpw=sys.stdin.read().split(','); sys.stdout.write('mysql://%s:%[email protected]:2021/arXiv' % (unpw[0].strip(),urllib.parse.quote(unpw[1].strip(),safe='*')))" > $@
+
+
+prod-proxy:
+	/usr/local/bin/cloud-sql-proxy --address 0.0.0.0 --port ${PROD_DB_PROXY_PORT} arxiv-production:us-central1:arxiv-production-rep9 > /dev/null 2>&1 &
+
+test: venv/bin/poetry
+	venv/bin/poetry run pytest --cov=arxiv.base fourohfour --cov-fail-under=67 arxiv/base fourohfour
+	TEST_ARXIV_DB_URI=mysql://testuser:[email protected]:13306/testdb venv/bin/poetry run pytest --cov=arxiv --cov-fail-under=25 arxiv
+	TEST_ARXIV_DB_URI=mysql://testuser:[email protected]:13306/testdb venv/bin/poetry run python tests/run_app_tests.py
+

arxiv/auth/auth/tests/test_decorators.py

@@ -10,6 +10,7 @@
 from .. import scopes, decorators
 from ... import domain
 
+
 EASTERN = timezone('US/Eastern')
 """    @mock.patch(f'{decorators.__name__}.request')"""
 

arxiv/auth/legacy/tests/test_accounts.py

@@ -31,7 +31,7 @@ def get_user(session, user_id):
 def test_with_nonexistant_user_w_app(app):
     """There is no user with the passed username."""
     with app.app_context():
-            assert not accounts.does_username_exist('baruser')
+        assert not accounts.does_username_exist('baruser')
 
 def test_with_nonexistant_user_wo_app(db_configed):
     """There is no user with the passed username."""

arxiv/auth/legacy/tests/test_bootstrap.py

@@ -1,21 +1,17 @@
 """Test the legacy integration with synthetic data."""
 
-import os
 import shutil
-import sys
 import tempfile
 from typing import Tuple
 from unittest import TestCase
 from flask import Flask
-import locale
+import pytest
 
-from typing import List
 import random
-from datetime import datetime
-from pytz import timezone, UTC
+from pytz import timezone
 from mimesis import Person, Internet, Datetime, locales
 
-from sqlalchemy import select, func
+from sqlalchemy import select
 
 import arxiv.db
 from arxiv.db import transaction
@@ -65,6 +61,8 @@ def setUp(self):
         arxiv.db.models.configure_db_engine(engine,None)
         util.create_all(engine)
 
+        visited = set()
+
         with self.app.app_context():
             with transaction() as session:
                 edc = session.execute(select(models.Endorsement)).all()
@@ -80,21 +78,22 @@ def setUp(self):
                     papers_to_endorse=3
                 ))
 
-            for category in definitions.CATEGORIES_ACTIVE.keys():
-                if '.' in category:
-                    archive, subject_class = category.split('.', 1)
-                else:
-                    archive, subject_class = category, ''
-
-                with transaction() as session:
-                    #print(f"arch: {archive} sc: {subject_class}")
-                    session.add(models.Category(
-                        archive=archive,
-                        subject_class=subject_class,
-                        definitive=1,
-                        active=1,
-                        endorsement_domain='test_domain_bootstrap'
-                    ))
+            # categories are loaded already
+            # for category in definitions.CATEGORIES_ACTIVE.keys():
+            #     if '.' in category:
+            #         archive, subject_class = category.split('.', 1)
+            #     else:
+            #         archive, subject_class = category, ''
+            #
+            #     with transaction() as session:
+            #         #print(f"arch: {archive} sc: {subject_class}")
+            #         session.add(models.Category(
+            #             archive=archive,
+            #             subject_class=subject_class,
+            #             definitive=1,
+            #             active=1,
+            #             endorsement_domain='test_domain_bootstrap'
+            #         ))
 
             COUNT = 100
 
@@ -192,6 +191,19 @@ def setUp(self):
                         issued_when = util.epoch(
                             Datetime(locale).datetime().replace(tzinfo=EASTERN)
                         )
+
+                        # These are the constraints on the table.
+                        key = {
+                            "endorsee": db_user.email,
+                            "endorser_id": endorser_id,
+                            "archive": archive,
+                            "subject_class": subject_class,
+                        }
+                        key_str = repr(key)
+                        if key_str in visited:
+                            # print(f"{key_str} appeared already. This would violate the table constraint.")
+                            continue
+                        visited.add(key_str)
                         session.add(models.Endorsement(
                             endorsee=db_user,
                             endorser_id=endorser_id,
@@ -235,6 +247,8 @@ def setUp(self):
     def tearDown(self):
         shutil.rmtree(self.db_path)
 
+    @pytest.mark.skipif(lambda request: request.config.getoption("--db") == "mysql",
+                            reason="is set up for sqlite3 only")
     def test_authenticate_and_use_session(self):
         """Attempt to authenticate users and create/load auth sessions."""
         with self.app.app_context():