diff --git a/src/stubber/publish/package.py b/src/stubber/publish/package.py index 07de7437..24add579 100644 --- a/src/stubber/publish/package.py +++ b/src/stubber/publish/package.py @@ -11,7 +11,6 @@ from mpflash.logger import log from mpflash.versions import clean_version - from stubber.publish.defaults import GENERIC, GENERIC_L, default_board from stubber.publish.enums import StubSource from stubber.publish.stubpackage import StubPackage, StubSources @@ -50,7 +49,14 @@ def get_package( mpy_version=version, ): # create package from the information retrieved from the database - return StubPackage(pkg_name, port, board=board, version=version, json_data=package_info) + p_db = StubPackage( + pkg_name, + port, + board=board, + version=version, + json_data=package_info, + ) + return p_db log.debug(f"No package found for {pkg_name} in database, creating new package") return create_package( @@ -71,7 +77,7 @@ def get_package_info( ) -> Union[Dict, None]: """ get a package's record from the json db if it can be found - matches om the package name and version + matches on the package name and version pkg_name: package name (micropython-esp32-stubs) mpy_version: micropython/firmware version (1.18) """ @@ -86,15 +92,14 @@ def get_package_info( """, (pkg_name, f"{mpy_version}%"), ) - packages = cursor.fetchall() + packages = [dict(row) for row in cursor.fetchall()] - if len(packages) > 0: - pkg_from_db = dict(packages[-1]) + if packages: + pkg_from_db = packages[0] log.debug(f"Found latest {pkg_name} == {pkg_from_db['pkg_version']}") return pkg_from_db - else: - return None + return None def create_package( diff --git a/src/stubber/publish/publish.py b/src/stubber/publish/publish.py index 98bb0ebf..3658525d 100644 --- a/src/stubber/publish/publish.py +++ b/src/stubber/publish/publish.py @@ -74,7 +74,7 @@ def publish_multiple( worklist = build_worklist(family, versions, ports, boards) if len(worklist) == 0: - log.error("Could not find any packages than can be published.") + log.error("Could not find any packages that can be published.") return results for todo in worklist: @@ -113,7 +113,7 @@ def build_worklist( worklist = filter_list(worklist, ports, boards) for b in boards: - if b == "auto": + if b in ["auto", "all", "*"]: continue if not any(i for i in worklist if i["board"].lower() == b.lower()): log.warning(f"Could not find any package candidate for board {b}") diff --git a/src/stubber/publish/stubpackage.py b/src/stubber/publish/stubpackage.py index 48614e31..d06d1064 100644 --- a/src/stubber/publish/stubpackage.py +++ b/src/stubber/publish/stubpackage.py @@ -82,6 +82,7 @@ def __str__(self) -> str: return f"{self.package_name}=={self.mpy_version}" def __repr__(self) -> str: + # todo: use pkg_version to allow for comparing different versions of the same package return f"{self.package_name}=={self.mpy_version}" def __eq__(self, o: object) -> bool: @@ -388,10 +389,12 @@ def write_package_json(self) -> None: json.dump(self.to_dict(), f, indent=4) def to_dict(self) -> dict: - """return the package as a dict to store in the jsondb + """return the package as a Dict[str,str] to store in the db need to simplify some of the Objects to allow serialization to json - - the paths to posix paths + - stub_sources + List[Tuple[StubSource, Path]] -> List[List[str, str]] # convert + the paths to posix paths - the version (semver) to a string - toml file to list of lines @@ -402,7 +405,9 @@ def to_dict(self) -> dict: "publish": self._publish, "pkg_version": str(self.pkg_version), "path": self.package_path.name, # only store the folder name , as it is relative to the publish folder - "stub_sources": [(name, Path(path).as_posix()) for (name, path) in self.stub_sources], + "stub_sources": json.dumps( + [(name, Path(path).as_posix()) for (name, path) in self.stub_sources] + ), "description": self.description, "hash": self.hash, "stub_hash": self.stub_hash, @@ -432,7 +437,7 @@ def from_dict(self, json_data: Dict) -> None: # set pkg version after creating the toml file self.pkg_version = json_data["pkg_version"] self.stub_sources = [] - for name, path in json_data["stub_sources"]: + for name, path in json.loads(json_data["stub_sources"]): if path.startswith("stubs/"): path = path.replace("stubs/", "") self.stub_sources.append((name, Path(path))) @@ -538,13 +543,13 @@ def create_readme(self) -> None: with open(self.package_path / "README.md", "w") as f: f.write(f"# {self.package_name}\n\n") f.write(TEMPLATE_README) - f.write(f"Included stubs:\n") + f.write("Included stubs:\n") for name, folder in self.stub_sources: f.write(f"* {name} from `stubs/{Path(folder).as_posix()}`\n") - f.write(f"\n\n") - f.write(f"origin | Family | Port | Board | Version\n") - f.write(f"-------|--------|------|-------|--------\n") + f.write("\n\n") + f.write("origin | Family | Port | Board | Version\n") + f.write("-------|--------|------|-------|--------\n") try: f.write( f"Firmware | {firmware_stubs['firmware']['family']} | {firmware_stubs['firmware']['port']} | {firmware_stubs['firmware']['machine']} | {clean_version(firmware_stubs['firmware']['version'])} \n" @@ -788,6 +793,14 @@ def __init__( STUB_PATH - root-relative path to the folder where the stubs are stored ('./stubs'). """ + super().__init__( + package_name=package_name, + mpy_version=clean_version(version, drop_v=True), # Initial version + port=port, + board=board, + description=description, + stubs=stubs or [], + ) self.port = port self.board = board if json_data is not None: @@ -796,14 +809,14 @@ def __init__( # store essentials self.package_name = package_name self.description = description - self.mpy_version = clean_version(version, drop_v=True) # Initial version + # self.mpy_version = clean_version(version, drop_v=True) # Initial version self.create_update_pyproject_toml() - self.stub_sources: StubSources = [] - # save the stub sources - if stubs: - self.stub_sources = stubs + # self.stub_sources: StubSources = [] + # # save the stub sources + # if stubs: + # self.stub_sources = stubs self.status: Status = Status( { @@ -814,14 +827,6 @@ def __init__( "path": self.package_path.as_posix(), } ) - super().__init__( - package_name=package_name, - mpy_version=self.mpy_version, - port=port, - board=board, - description=description, - stubs=self.stub_sources, - ) def update_sources(self) -> StubSources: """ @@ -875,7 +880,7 @@ def update_distribution(self, production: bool) -> bool: log.debug( f"{self.package_name}: skipping as one or more source stub folders are missing" ) - self.status["error"] = "Skipped, stub folder(s) missing" + self.status["error"] = "Skipped, No stubs found." shutil.rmtree(self.package_path.as_posix()) self._publish = False # type: ignore return False @@ -1052,7 +1057,7 @@ def publish_distribution( d["mpy_version"], d["pkg_version"], d["publish"], - json.dumps(d["stub_sources"]), + d["stub_sources"], d["path"], d["hash"], d["stub_hash"], diff --git a/tests/publish/conftest.py b/tests/publish/conftest.py index c228cfa4..713dbe8c 100644 --- a/tests/publish/conftest.py +++ b/tests/publish/conftest.py @@ -40,18 +40,18 @@ def fake_package(request, mocker: MockerFixture, tmp_path: Path, pytestconfig: p @pytest.fixture -def temp_db_conn( +def test_db_conn( pytestconfig: pytest.Config, tmp_path: Path, ): """""" - db_src = pytestconfig.rootpath / "tests/publish/data/all_packages_test.db" + db_src = pytestconfig.rootpath / "tests/publish/data/test_packages.db" db_path = tmp_path / "all_packages_test.db" # copy file to temp location shutil.copy(db_src, db_path) db_conn = sqlite3.connect(db_path) - db_conn.row_factory = sqlite3.Row # return rows as dicts + db_conn.row_factory = sqlite3.Row # return rows as dicts yield db_conn try: db_conn.close() diff --git a/tests/publish/data/all_packages_test.db b/tests/publish/data/all_packages_test.db deleted file mode 100644 index 8b1807d1..00000000 Binary files a/tests/publish/data/all_packages_test.db and /dev/null differ diff --git a/tests/publish/data/test_packages.db b/tests/publish/data/test_packages.db new file mode 100644 index 00000000..53c62a1c Binary files /dev/null and b/tests/publish/data/test_packages.db differ diff --git a/tests/publish/test_get_package.py b/tests/publish/test_get_package.py new file mode 100644 index 00000000..18c8e975 --- /dev/null +++ b/tests/publish/test_get_package.py @@ -0,0 +1,122 @@ +import sqlite3 +import tempfile +from pathlib import Path + +import pytest +from stubber.publish.package import get_package, get_package_info +from stubber.publish.stubpackage import StubPackage + + +@pytest.mark.parametrize( + "mpy_version, pkg_name", + [ + ("1.18", "micropython-esp32-stubs"), + ("1.20.0", "micropython-rp2-pico-stubs"), + ("1.22.1", "micropython-rp2-rpi_pico-stubs"), + # TODO: Fix naming of these packages + ("1.23.0", "micropython-esp32-esp32_generic_c3-stubs"), + ("1.23.0", "micropython-esp32-esp32_generic-stubs"), + ], +) +# @pytest.mark.parametrize("pkg_name", ["micropython-esp32-stubs"]) +def test_get_package_info(test_db_conn, pkg_name, mpy_version): + + pub_path = Path(tempfile.gettempdir()) + + package_info = get_package_info( + test_db_conn, pub_path, pkg_name=pkg_name, mpy_version=mpy_version + ) + assert package_info is not None + assert package_info["name"] == pkg_name + assert package_info["mpy_version"] == mpy_version + assert package_info["pkg_version"].startswith(mpy_version) + assert package_info["pkg_version"] > mpy_version + # Hashes + assert package_info["stub_hash"], "stub only hash not found" + assert package_info["hash"], "Hash not found" + # Stub sources - string + assert package_info["stub_sources"], "Stub sources not found" + assert isinstance(package_info["stub_sources"], str), "Stub sources is not a string" + + +def test_get_package_info_no_match(test_db_conn, tmp_path): + pub_path = tmp_path + pkg_name = "micropython-esp32-stubs" + mpy_version = "1.13.1" + + package_info = get_package_info( + test_db_conn, pub_path, pkg_name=pkg_name, mpy_version=mpy_version + ) + assert package_info is None + + def test_get_package_info_invalid_version(test_db_conn): + pub_path = Path(tempfile.gettempdir()) + pkg_name = "micropython-esp32-stubs" + mpy_version = "invalid_version" + + with pytest.raises(ValueError): + get_package_info(test_db_conn, pub_path, pkg_name=pkg_name, mpy_version=mpy_version) + + +# def test_get_package_info_empty_package_name(test_db_conn): +# pub_path = Path(tempfile.gettempdir()) +# pkg_name = "" +# mpy_version = "1.18" + +# with pytest.raises(ValueError): +# get_package_info(test_db_conn, pub_path, pkg_name=pkg_name, mpy_version=mpy_version) + + +@pytest.mark.parametrize( + "mpy_version, port, board", + [ + ("1.18", "esp32", "generic"), + ("1.20.0", "rp2", "pico"), + ("1.22.1", "rp2", "rpi_pico"), + # ("1.23.0", "esp32", "generic_c3"), + # ("1.23.0", "esp32", "generic"), + ], +) +def test_get_package(test_db_conn, mpy_version, port, board): + package = get_package(test_db_conn, port=port, board=board, version=mpy_version) + assert package is not None + assert package.port == port + assert package.board == board or board == "generic" + assert package.mpy_version == mpy_version + assert package.hash, "Hash not found" + assert package.stub_hash, "Stub only hash not found" + + +def test_qandd(): + pkg_name = "micropython-esp32-stubs" + port = "esp32" + board = "generic" + version = "1.18" + package_info = { + "id": 614587032550805777, + "name": "micropython-esp32-stubs", + "description": "MicroPython stubs", + "mpy_version": "1.18", + "pkg_version": "1.18.post3", + "publish": 1, + "stub_sources": '[["Merged stubs", "micropython-v1_18-esp32-merged"], ["Frozen stubs", "micropython-v1_18-frozen/esp32/GENERIC"], ["Core Stubs", "micropython-core"]]', + "path": "micropython-v1_18-esp32-stubs", + "hash": "ea79cc2c8b929dd33ac489fcd882b0272d75b83d", + "stub_hash": "50ef8cbd42e9599a892217ce1ef16cbed98e2466 ", + "port": "esp32", + "board": "", + "variant": "", + } + + p_db = StubPackage( + pkg_name, + port, + board=board, + version=version, + json_data=package_info, + ) + + assert p_db is not None + assert p_db.port == port + assert p_db.hash, "Hash not found" + assert p_db.stub_hash, "Stub only hash not found" diff --git a/tests/publish/test_package.py b/tests/publish/test_package.py index d6b2160d..788ede45 100644 --- a/tests/publish/test_package.py +++ b/tests/publish/test_package.py @@ -103,11 +103,11 @@ def test_create_package( "publish": True, "pkg_version": "1.18.post6", "path": "foo-v1_18-bar-stubs", - "stub_sources": [ + "stub_sources": """[ ["MCU stubs", "micropython-v1_17-stm32"], ["Frozen stubs", "micropython-v1_17-frozen/stm32/GENERIC"], - ["Core Stubs", "cpython_core-pycopy"], - ], + ["Core Stubs", "cpython_core-pycopy"] + ]""", "description": "foo bar stubs", "hash": "b09f9c819c9e98cbd9dfbc8158079146587e2d66", "stub_hash": "", @@ -121,11 +121,11 @@ def test_create_package( "publish": True, "pkg_version": "1.18.post6", "path": "foo-v1_18-bar-stubs", - "stub_sources": [ + "stub_sources": """[ ["MCU stubs", "stubs/micropython-v1_17-stm32"], ["Frozen stubs", "stubs/micropython-v1_17-frozen/stm32/GENERIC"], - ["Core Stubs", "stubs/cpython_core-pycopy"], - ], + ["Core Stubs", "stubs/cpython_core-pycopy"] + ]""", "description": "foo bar stubs", "hash": "b09f9c819c9e98cbd9dfbc8158079146587e2d66", "stub_hash": "", @@ -139,9 +139,9 @@ def test_create_package( "publish": True, "pkg_version": "1.18.post6", "path": "publish/foo-v1_18-bar-stubs", - "stub_sources": [ - ["MCU stubs", "micropython-v1_17-stm32"], - ], + "stub_sources": """[ + ["MCU stubs", "micropython-v1_17-stm32"] + ]""", "description": "foo bar stubs", "hash": "b09f9c819c9e98cbd9dfbc8158079146587e2d66", "stub_hash": "1234567890", @@ -172,7 +172,6 @@ def test_package_from_json(tmp_path, pytestconfig, mocker: MockerFixture, json): test_build=False, ) - def run_common_package_tests( package: StubPackage, pkg_name, publish_path: Path, stub_path: Path, test_build=True ): diff --git a/tests/publish/test_publish.py b/tests/publish/test_publish.py index 1ec71f64..2ffa2a55 100644 --- a/tests/publish/test_publish.py +++ b/tests/publish/test_publish.py @@ -126,7 +126,7 @@ def test_publish_package( tmp_path: Path, pytestconfig: pytest.Config, fake_package: StubPackage, - temp_db_conn: sqlite3.Connection, + test_db_conn: sqlite3.Connection, ): pkg = fake_package @@ -138,7 +138,9 @@ def test_publish_package( pkg._publish = True # type: ignore # FIXME : dependency to access to test.pypi.org - result = pkg.publish_distribution_ifchanged(production=False, force=False, db_conn=temp_db_conn) + result = pkg.publish_distribution_ifchanged( + production=False, force=False, db_conn=test_db_conn + ) assert result, "should be ok" @@ -156,7 +158,7 @@ def test_publish_package( assert m_publish.called, "should call poetry publish" # check is the hashes are added to the database - cursor = temp_db_conn.cursor() + cursor = test_db_conn.cursor() cursor.execute( "SELECT * FROM packages where name = ? AND mpy_version = ? ORDER by pkg_version DESC", (pkg.package_name, pkg.mpy_version),