From c3221457fdbaa156281b0a1d631aefbe2a25d843 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 29 Aug 2023 15:54:44 +0200 Subject: [PATCH 01/76] Add option to pass driver instead of database_url - WIP --- neomodel/config.py | 6 + neomodel/scripts/neomodel_install_labels.py | 10 +- neomodel/scripts/neomodel_remove_labels.py | 12 +- neomodel/util.py | 122 ++++++++++++-------- test/test_connection.py | 15 +-- test/test_database_management.py | 8 +- test/test_transactions.py | 21 ++-- 7 files changed, 115 insertions(+), 79 deletions(-) diff --git a/neomodel/config.py b/neomodel/config.py index 1f6df10b..5c882b1f 100644 --- a/neomodel/config.py +++ b/neomodel/config.py @@ -16,3 +16,9 @@ RESOLVER = None TRUSTED_CERTIFICATES = neo4j.TrustSystemCAs() USER_AGENT = f"neomodel/v{__version__}" + +DRIVER = neo4j.GraphDatabase().driver( + "bolt://localhost:7687", auth=("neo4j", "foobarbaz") +) +# TODO : Try passing a different database name +# DATABASE_NAME = "testdatabase" diff --git a/neomodel/scripts/neomodel_install_labels.py b/neomodel/scripts/neomodel_install_labels.py index 444838b2..8bd5119f 100755 --- a/neomodel/scripts/neomodel_install_labels.py +++ b/neomodel/scripts/neomodel_install_labels.py @@ -27,8 +27,8 @@ from __future__ import print_function import sys -from argparse import ArgumentParser, RawDescriptionHelpFormatter import textwrap +from argparse import ArgumentParser, RawDescriptionHelpFormatter from importlib import import_module from os import environ, path @@ -70,14 +70,16 @@ def load_python_module_or_file(name): def main(): parser = ArgumentParser( formatter_class=RawDescriptionHelpFormatter, - description=textwrap.dedent(""" + description=textwrap.dedent( + """ Setup indexes and constraints on labels in Neo4j for your neomodel schema. If a connection URL is not specified, the tool will look up the environment variable NEO4J_BOLT_URL. If that environment variable is not set, the tool will attempt to connect to the default URL bolt://neo4j:neo4j@localhost:7687 """ - )) + ), + ) parser.add_argument( "apps", @@ -107,7 +109,7 @@ def main(): # Connect after to override any code in the module that may set the connection print(f"Connecting to {bolt_url}") - db.set_connection(bolt_url) + db.set_connection(url=bolt_url) install_all_labels() diff --git a/neomodel/scripts/neomodel_remove_labels.py b/neomodel/scripts/neomodel_remove_labels.py index 58a57cdd..1ad6cc34 100755 --- a/neomodel/scripts/neomodel_remove_labels.py +++ b/neomodel/scripts/neomodel_remove_labels.py @@ -23,8 +23,8 @@ """ from __future__ import print_function -from argparse import ArgumentParser, RawDescriptionHelpFormatter import textwrap +from argparse import ArgumentParser, RawDescriptionHelpFormatter from os import environ from .. import db, remove_all_labels @@ -32,15 +32,17 @@ def main(): parser = ArgumentParser( - formatter_class=RawDescriptionHelpFormatter, - description=textwrap.dedent(""" + formatter_class=RawDescriptionHelpFormatter, + description=textwrap.dedent( + """ Drop all indexes and constraints on labels from schema in Neo4j database. If a connection URL is not specified, the tool will look up the environment variable NEO4J_BOLT_URL. If that environment variable is not set, the tool will attempt to connect to the default URL bolt://neo4j:neo4j@localhost:7687 """ - )) + ), + ) parser.add_argument( "--db", @@ -59,7 +61,7 @@ def main(): # Connect after to override any code in the module that may set the connection print(f"Connecting to {bolt_url}") - db.set_connection(bolt_url) + db.set_connection(url=bolt_url) remove_all_labels() diff --git a/neomodel/util.py b/neomodel/util.py index c03a1ce2..ccceb608 100644 --- a/neomodel/util.py +++ b/neomodel/util.py @@ -7,7 +7,7 @@ from typing import Optional, Sequence from urllib.parse import quote, unquote, urlparse -from neo4j import DEFAULT_DATABASE, GraphDatabase, basic_auth +from neo4j import DEFAULT_DATABASE, Driver, GraphDatabase, basic_auth from neo4j.api import Bookmarks from neo4j.exceptions import ClientError, ServiceUnavailable, SessionExpired from neo4j.graph import Node, Path, Relationship @@ -33,8 +33,11 @@ def wrapper(self, *args, **kwargs): else: _db = self - if not _db.url: - _db.set_connection(config.DATABASE_URL) + if not _db.driver: + if config.DRIVER: + _db.set_connection(driver=config.DRIVER) + elif config.DATABASE_URL: + _db.set_connection(url=config.DATABASE_URL) return func(self, *args, **kwargs) @@ -78,65 +81,85 @@ def __init__(self): self._database_edition = None self.impersonated_user = None - def set_connection(self, url): + def set_connection(self, url: str = None, driver: Driver = None): """ Sets the connection URL to the address a Neo4j server is set up at """ - p_start = url.replace(":", "", 1).find(":") + 2 - p_end = url.rfind("@") - password = url[p_start:p_end] - url = url.replace(password, quote(password)) - parsed_url = urlparse(url) - - valid_schemas = [ - "bolt", - "bolt+s", - "bolt+ssc", - "bolt+routing", - "neo4j", - "neo4j+s", - "neo4j+ssc", - ] - - if parsed_url.netloc.find("@") > -1 and parsed_url.scheme in valid_schemas: - credentials, hostname = parsed_url.netloc.rsplit("@", 1) - username, password = credentials.split(":") - password = unquote(password) - database_name = parsed_url.path.strip("/") - else: - raise ValueError( - f"Expecting url format: bolt://user:password@localhost:7687 got {url}" + if driver: + self.driver = driver + if hasattr(config, "DATABASE_NAME"): + self._database_name = config.DATABASE_NAME + elif url: + p_start = url.replace(":", "", 1).find(":") + 2 + p_end = url.rfind("@") + password = url[p_start:p_end] + url = url.replace(password, quote(password)) + parsed_url = urlparse(url) + + valid_schemas = [ + "bolt", + "bolt+s", + "bolt+ssc", + "bolt+routing", + "neo4j", + "neo4j+s", + "neo4j+ssc", + ] + + if parsed_url.netloc.find("@") > -1 and parsed_url.scheme in valid_schemas: + credentials, hostname = parsed_url.netloc.rsplit("@", 1) + username, password = credentials.split(":") + password = unquote(password) + database_name = parsed_url.path.strip("/") + else: + raise ValueError( + f"Expecting url format: bolt://user:password@localhost:7687 got {url}" + ) + + options = { + "auth": basic_auth(username, password), + "connection_acquisition_timeout": config.CONNECTION_ACQUISITION_TIMEOUT, + "connection_timeout": config.CONNECTION_TIMEOUT, + "keep_alive": config.KEEP_ALIVE, + "max_connection_lifetime": config.MAX_CONNECTION_LIFETIME, + "max_connection_pool_size": config.MAX_CONNECTION_POOL_SIZE, + "max_transaction_retry_time": config.MAX_TRANSACTION_RETRY_TIME, + "resolver": config.RESOLVER, + "user_agent": config.USER_AGENT, + } + + if "+s" not in parsed_url.scheme: + options["encrypted"] = config.ENCRYPTED + options["trusted_certificates"] = config.TRUSTED_CERTIFICATES + + self.driver = GraphDatabase.driver( + parsed_url.scheme + "://" + hostname, **options + ) + self.url = url + self._database_name = ( + DEFAULT_DATABASE if database_name == "" else database_name ) - options = { - "auth": basic_auth(username, password), - "connection_acquisition_timeout": config.CONNECTION_ACQUISITION_TIMEOUT, - "connection_timeout": config.CONNECTION_TIMEOUT, - "keep_alive": config.KEEP_ALIVE, - "max_connection_lifetime": config.MAX_CONNECTION_LIFETIME, - "max_connection_pool_size": config.MAX_CONNECTION_POOL_SIZE, - "max_transaction_retry_time": config.MAX_TRANSACTION_RETRY_TIME, - "resolver": config.RESOLVER, - "user_agent": config.USER_AGENT, - } - - if "+s" not in parsed_url.scheme: - options["encrypted"] = config.ENCRYPTED - options["trusted_certificates"] = config.TRUSTED_CERTIFICATES - - self.driver = GraphDatabase.driver( - parsed_url.scheme + "://" + hostname, **options - ) - self.url = url self._pid = os.getpid() self._active_transaction = None - self._database_name = DEFAULT_DATABASE if database_name == "" else database_name # Getting the information about the database version requires a connection to the database self._database_version = None self._database_edition = None self._update_database_version() + def close_connection(self): + """ + Closes the currently open driver. + The driver should always be called at the end of the application's lifecyle. + If you pass your own driver to neomodel, you can also close it yourself without this method. + """ + self._database_version = None + self._database_edition = None + self._database_name = None + self.driver.close() + self.driver = None + @property def database_version(self): if self._database_version is None: @@ -420,6 +443,7 @@ def _run_cypher_query( raise exc_info[1].with_traceback(exc_info[2]) except SessionExpired: if retry_on_session_expire: + # TODO : What about if config passes driver instead of url ? self.set_connection(self.url) return self.cypher_query( query=query, diff --git a/test/test_connection.py b/test/test_connection.py index 702a4122..7a199139 100644 --- a/test/test_connection.py +++ b/test/test_connection.py @@ -5,16 +5,14 @@ from neomodel import config, db -INITIAL_URL = db.url - @pytest.fixture(autouse=True) def setup_teardown(): yield # Teardown actions after tests have run # Reconnect to initial URL for potential subsequent tests - db.driver.close() - db.set_connection(INITIAL_URL) + db.close_connection() + db.set_connection(url=config.DATABASE_URL) @pytest.fixture(autouse=True, scope="session") @@ -27,7 +25,7 @@ def neo4j_logging(): def test_connect_to_aura(protocol): cypher_return = "hello world" default_cypher_query = f"RETURN '{cypher_return}'" - db.driver.close() + db.close_connection() _set_connection(protocol=protocol) result, _ = db.cypher_query(default_cypher_query) @@ -41,17 +39,16 @@ def _set_connection(protocol): AURA_TEST_DB_PASSWORD = os.environ["AURA_TEST_DB_PASSWORD"] AURA_TEST_DB_HOSTNAME = os.environ["AURA_TEST_DB_HOSTNAME"] - config.DATABASE_URL = f"{protocol}://{AURA_TEST_DB_USER}:{AURA_TEST_DB_PASSWORD}@{AURA_TEST_DB_HOSTNAME}" - db.set_connection(config.DATABASE_URL) + database_url = f"{protocol}://{AURA_TEST_DB_USER}:{AURA_TEST_DB_PASSWORD}@{AURA_TEST_DB_HOSTNAME}" + db.set_connection(url=database_url) @pytest.mark.parametrize( "url", ["bolt://user:password", "http://user:password@localhost:7687"] ) def test_wrong_url_format(url): - prev_url = db.url with pytest.raises( ValueError, match=rf"Expecting url format: bolt://user:password@localhost:7687 got {url}", ): - db.set_connection(url) + db.set_connection(url=url) diff --git a/test/test_database_management.py b/test/test_database_management.py index 4a09c3e3..5cc92c70 100644 --- a/test/test_database_management.py +++ b/test/test_database_management.py @@ -61,11 +61,11 @@ def test_change_password(): util.change_neo4j_password(db, "neo4j", new_password) - db.set_connection(new_url) + db.set_connection(url=new_url) with pytest.raises(AuthError): - db.set_connection(prev_url) + db.set_connection(url=prev_url) - db.set_connection(new_url) + db.set_connection(url=new_url) util.change_neo4j_password(db, "neo4j", prev_password) - db.set_connection(prev_url) + db.set_connection(url=prev_url) diff --git a/test/test_transactions.py b/test/test_transactions.py index 35e7c01f..62d13a43 100644 --- a/test/test_transactions.py +++ b/test/test_transactions.py @@ -3,7 +3,14 @@ from neo4j.exceptions import ClientError, TransactionError from pytest import raises -from neomodel import StringProperty, StructuredNode, UniqueProperty, db, install_labels +from neomodel import ( + StringProperty, + StructuredNode, + UniqueProperty, + config, + db, + install_labels, +) class APerson(StructuredNode): @@ -80,9 +87,9 @@ def test_read_transaction(): people = APerson.nodes.all() assert people - with pytest.raises(TransactionError): + with raises(TransactionError): with db.read_transaction: - with pytest.raises(ClientError) as e: + with raises(ClientError) as e: APerson(name="Gina").save() assert e.value.code == "Neo.ClientError.Statement.AccessMode" @@ -97,7 +104,7 @@ def test_write_transaction(): def double_transaction(): db.begin() - with pytest.raises(SystemError, match=r"Transaction in progress"): + with raises(SystemError, match=r"Transaction in progress"): db.begin() db.rollback() @@ -105,13 +112,11 @@ def double_transaction(): def test_set_connection_works(): assert APerson(name="New guy 1").save() - from socket import gaierror - old_url = db.url with raises(ValueError): - db.set_connection("bolt://user:password@6.6.6.6.6.6.6.6:7687") + db.set_connection(url="bolt://user:password@6.6.6.6.6.6.6.6:7687") APerson(name="New guy 2").save() - db.set_connection(old_url) + db.set_connection(url=config.DATABASE_URL) # set connection back assert APerson(name="New guy 3").save() From 45177eb7c8155d04444ffca4326ede14279c60c9 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 5 Sep 2023 15:42:30 +0200 Subject: [PATCH 02/76] Explictly close the driver in all tests --- test/conftest.py | 5 +++++ test/test_database_management.py | 6 ++++++ test/test_transactions.py | 3 +++ 3 files changed, 14 insertions(+) diff --git a/test/conftest.py b/test/conftest.py index 1cf682df..1be37a5d 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -82,6 +82,11 @@ def pytest_sessionstart(session): db.cypher_query("GRANT IMPERSONATE (troygreene) ON DBMS TO admin") +@pytest.hookimpl +def pytest_unconfigure(config): + db.close_connection() + + def version_to_dec(a_version_string): """ Converts a version string to a number to allow for quick checks on the versions of specific components. diff --git a/test/test_database_management.py b/test/test_database_management.py index 5cc92c70..2a2ece34 100644 --- a/test/test_database_management.py +++ b/test/test_database_management.py @@ -60,12 +60,18 @@ def test_change_password(): new_url = f"bolt://neo4j:{new_password}@localhost:7687" util.change_neo4j_password(db, "neo4j", new_password) + db.close_connection() db.set_connection(url=new_url) + db.close_connection() with pytest.raises(AuthError): db.set_connection(url=prev_url) + db.close_connection() + db.set_connection(url=new_url) util.change_neo4j_password(db, "neo4j", prev_password) + db.close_connection() + db.set_connection(url=prev_url) diff --git a/test/test_transactions.py b/test/test_transactions.py index 62d13a43..9cbfb8f3 100644 --- a/test/test_transactions.py +++ b/test/test_transactions.py @@ -112,10 +112,13 @@ def double_transaction(): def test_set_connection_works(): assert APerson(name="New guy 1").save() + db.close_connection() with raises(ValueError): db.set_connection(url="bolt://user:password@6.6.6.6.6.6.6.6:7687") APerson(name="New guy 2").save() + + db.close_connection() db.set_connection(url=config.DATABASE_URL) # set connection back assert APerson(name="New guy 3").save() From fc8aad27c0f9ae58a8c42473b617e3c66f116d27 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 5 Sep 2023 16:36:59 +0200 Subject: [PATCH 03/76] Add explicit connection tests --- neomodel/config.py | 11 ++++++--- neomodel/util.py | 2 +- test/conftest.py | 10 +++++---- test/test_connection.py | 43 ++++++++++++++++++++++++++---------- test/test_multiprocessing.py | 2 +- test/test_transactions.py | 14 ------------ 6 files changed, 47 insertions(+), 35 deletions(-) diff --git a/neomodel/config.py b/neomodel/config.py index 5c882b1f..2cc5539d 100644 --- a/neomodel/config.py +++ b/neomodel/config.py @@ -3,6 +3,9 @@ from ._version import __version__ AUTO_INSTALL_LABELS = False + +# Use this to connect with automatically created driver +# The following options are the default ones that will be used as driver config DATABASE_URL = "bolt://neo4j:foobarbaz@localhost:7687" FORCE_TIMEZONE = False @@ -17,8 +20,10 @@ TRUSTED_CERTIFICATES = neo4j.TrustSystemCAs() USER_AGENT = f"neomodel/v{__version__}" -DRIVER = neo4j.GraphDatabase().driver( - "bolt://localhost:7687", auth=("neo4j", "foobarbaz") -) +# Use this to connect with your self-managed driver instead +# DRIVER = neo4j.GraphDatabase().driver( +# "bolt://localhost:7687", auth=("neo4j", "foobarbaz") +# ) + # TODO : Try passing a different database name # DATABASE_NAME = "testdatabase" diff --git a/neomodel/util.py b/neomodel/util.py index ccceb608..c32945aa 100644 --- a/neomodel/util.py +++ b/neomodel/util.py @@ -34,7 +34,7 @@ def wrapper(self, *args, **kwargs): _db = self if not _db.driver: - if config.DRIVER: + if hasattr(config, "DRIVER") and config.DRIVER: _db.set_connection(driver=config.DRIVER) elif config.DATABASE_URL: _db.set_connection(url=config.DATABASE_URL) diff --git a/test/conftest.py b/test/conftest.py index 1be37a5d..c5ef2737 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -4,10 +4,12 @@ import warnings import pytest -from neo4j.exceptions import ClientError as CypherError -from neobolt.exceptions import ClientError -from neomodel import change_neo4j_password, clear_neo4j_database, config, db +from neomodel import clear_neo4j_database, config, db + +NEO4J_URL = os.environ.get("NEO4J_URL", "bolt://localhost:7687") +NEO4J_USERNAME = os.environ.get("NEO4J_USERNAME", "neo4j") +NEO4J_PASSWORD = os.environ.get("NEO4J_PASSWORD", "foobarbaz") def pytest_addoption(parser): @@ -83,7 +85,7 @@ def pytest_sessionstart(session): @pytest.hookimpl -def pytest_unconfigure(config): +def pytest_unconfigure(): db.close_connection() diff --git a/test/test_connection.py b/test/test_connection.py index 7a199139..ccac8739 100644 --- a/test/test_connection.py +++ b/test/test_connection.py @@ -1,9 +1,12 @@ import os import pytest +from neo4j import GraphDatabase from neo4j.debug import watch -from neomodel import config, db +from neomodel import StringProperty, StructuredNode, config, db + +from .conftest import NEO4J_PASSWORD, NEO4J_URL, NEO4J_USERNAME @pytest.fixture(autouse=True) @@ -21,6 +24,33 @@ def neo4j_logging(): yield +class Pastry(StructuredNode): + name = StringProperty(unique_index=True) + + +def test_set_connection_driver_works(): + # Verify that current connection is up + assert Pastry(name="Chocolatine").save() + db.close_connection() + + # Test connection using a driver + db.set_connection( + driver=GraphDatabase().driver(NEO4J_URL, auth=(NEO4J_USERNAME, NEO4J_PASSWORD)) + ) + assert Pastry(name="Croissant").save() + + +@pytest.mark.parametrize( + "url", ["bolt://user:password", "http://user:password@localhost:7687"] +) +def test_wrong_url_format(url): + with pytest.raises( + ValueError, + match=rf"Expecting url format: bolt://user:password@localhost:7687 got {url}", + ): + db.set_connection(url=url) + + @pytest.mark.parametrize("protocol", ["neo4j+s", "neo4j+ssc", "bolt+s", "bolt+ssc"]) def test_connect_to_aura(protocol): cypher_return = "hello world" @@ -41,14 +71,3 @@ def _set_connection(protocol): database_url = f"{protocol}://{AURA_TEST_DB_USER}:{AURA_TEST_DB_PASSWORD}@{AURA_TEST_DB_HOSTNAME}" db.set_connection(url=database_url) - - -@pytest.mark.parametrize( - "url", ["bolt://user:password", "http://user:password@localhost:7687"] -) -def test_wrong_url_format(url): - with pytest.raises( - ValueError, - match=rf"Expecting url format: bolt://user:password@localhost:7687 got {url}", - ): - db.set_connection(url=url) diff --git a/test/test_multiprocessing.py b/test/test_multiprocessing.py index 9ae4340b..9e638862 100644 --- a/test/test_multiprocessing.py +++ b/test/test_multiprocessing.py @@ -1,6 +1,6 @@ from multiprocessing.pool import ThreadPool as Pool -from neomodel import StringProperty, StructuredNode, db +from neomodel import StringProperty, StructuredNode class ThingyMaBob(StructuredNode): diff --git a/test/test_transactions.py b/test/test_transactions.py index 9cbfb8f3..0481e2a7 100644 --- a/test/test_transactions.py +++ b/test/test_transactions.py @@ -110,20 +110,6 @@ def double_transaction(): db.rollback() -def test_set_connection_works(): - assert APerson(name="New guy 1").save() - db.close_connection() - - with raises(ValueError): - db.set_connection(url="bolt://user:password@6.6.6.6.6.6.6.6:7687") - APerson(name="New guy 2").save() - - db.close_connection() - db.set_connection(url=config.DATABASE_URL) - # set connection back - assert APerson(name="New guy 3").save() - - @db.transaction.with_bookmark def in_a_tx(*names): for n in names: From 88237be76d56ed8496ef68a57d5bd85ce844a8a8 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Wed, 6 Sep 2023 09:58:41 +0200 Subject: [PATCH 04/76] Add test for passing database name --- neomodel/config.py | 3 --- neomodel/util.py | 14 ++++++++---- test/test_connection.py | 44 ++++++++++++++++++++++++++++++++++++ test/test_multiprocessing.py | 3 ++- 4 files changed, 56 insertions(+), 8 deletions(-) diff --git a/neomodel/config.py b/neomodel/config.py index 2cc5539d..b54aa806 100644 --- a/neomodel/config.py +++ b/neomodel/config.py @@ -24,6 +24,3 @@ # DRIVER = neo4j.GraphDatabase().driver( # "bolt://localhost:7687", auth=("neo4j", "foobarbaz") # ) - -# TODO : Try passing a different database name -# DATABASE_NAME = "testdatabase" diff --git a/neomodel/util.py b/neomodel/util.py index c32945aa..51cdbcf3 100644 --- a/neomodel/util.py +++ b/neomodel/util.py @@ -87,7 +87,7 @@ def set_connection(self, url: str = None, driver: Driver = None): """ if driver: self.driver = driver - if hasattr(config, "DATABASE_NAME"): + if hasattr(config, "DATABASE_NAME") and config.DATABASE_NAME: self._database_name = config.DATABASE_NAME elif url: p_start = url.replace(":", "", 1).find(":") + 2 @@ -136,12 +136,18 @@ def set_connection(self, url: str = None, driver: Driver = None): parsed_url.scheme + "://" + hostname, **options ) self.url = url - self._database_name = ( - DEFAULT_DATABASE if database_name == "" else database_name - ) + # The database name can be provided through the url or the config + if database_name == "": + if hasattr(config, "DATABASE_NAME") and config.DATABASE_NAME: + self._database_name = config.DATABASE_NAME + else: + self._database_name = database_name self._pid = os.getpid() self._active_transaction = None + # Set to default database if it hasn't been set before + if self._database_name is None: + self._database_name = DEFAULT_DATABASE # Getting the information about the database version requires a connection to the database self._database_version = None diff --git a/test/test_connection.py b/test/test_connection.py index ccac8739..622ec9e6 100644 --- a/test/test_connection.py +++ b/test/test_connection.py @@ -24,6 +24,19 @@ def neo4j_logging(): yield +def get_current_database_name() -> str: + """ + Fetches the name of the currently active database from the Neo4j database. + + Returns: + - str: The name of the current database. + """ + results, meta = db.cypher_query("CALL db.info") + results_as_dict = [dict(zip(meta, row)) for row in results] + + return results_as_dict[0]["name"] + + class Pastry(StructuredNode): name = StringProperty(unique_index=True) @@ -40,6 +53,37 @@ def test_set_connection_driver_works(): assert Pastry(name="Croissant").save() +def test_connect_to_non_default_database(): + database_name = "pastries" + db.cypher_query(f"CREATE DATABASE {database_name} IF NOT EXISTS") + db.close_connection() + + # Set database name in url - for url init only + db.set_connection(url=f"{config.DATABASE_URL}/{database_name}") + assert get_current_database_name() == "pastries" + + db.close_connection() + + # Set database name in config - for both url and driver init + config.DATABASE_NAME = database_name + + # url init + db.set_connection(url=config.DATABASE_URL) + assert get_current_database_name() == "pastries" + + db.close_connection() + + # driver init + db.set_connection( + driver=GraphDatabase().driver(NEO4J_URL, auth=(NEO4J_USERNAME, NEO4J_PASSWORD)) + ) + assert get_current_database_name() == "pastries" + + # Clear config + # No need to close connection - pytest teardown will do it + config.DATABASE_NAME = None + + @pytest.mark.parametrize( "url", ["bolt://user:password", "http://user:password@localhost:7687"] ) diff --git a/test/test_multiprocessing.py b/test/test_multiprocessing.py index 9e638862..fb00675d 100644 --- a/test/test_multiprocessing.py +++ b/test/test_multiprocessing.py @@ -1,6 +1,6 @@ from multiprocessing.pool import ThreadPool as Pool -from neomodel import StringProperty, StructuredNode +from neomodel import StringProperty, StructuredNode, db class ThingyMaBob(StructuredNode): @@ -18,3 +18,4 @@ def test_concurrency(): results = p.map(thing_create, range(50)) for returned, sent in results: assert returned == sent + db.close_connection() From 21e286c011d5e472f501a18dde0a729ee6bbe9f6 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Wed, 6 Sep 2023 11:07:22 +0200 Subject: [PATCH 05/76] Skip non default database test for community edition --- test/test_connection.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/test_connection.py b/test/test_connection.py index 622ec9e6..379556a2 100644 --- a/test/test_connection.py +++ b/test/test_connection.py @@ -53,6 +53,10 @@ def test_set_connection_driver_works(): assert Pastry(name="Croissant").save() +@pytest.mark.skipif( + db.database_edition != "enterprise", + reason="Skipping test for community edition - no multi database in CE", +) def test_connect_to_non_default_database(): database_name = "pastries" db.cypher_query(f"CREATE DATABASE {database_name} IF NOT EXISTS") From 4cc15693d3942563369de5c64ce54cc22f8dedb9 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Wed, 6 Sep 2023 14:00:35 +0200 Subject: [PATCH 06/76] Update doc --- doc/source/conf.py | 2 +- doc/source/configuration.rst | 96 +++++++++++++++++++++++++++++++--- doc/source/getting_started.rst | 35 ------------- 3 files changed, 91 insertions(+), 42 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 41e50bbf..538c5190 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -57,7 +57,7 @@ # General information about the project. project = __package__ -copyright = "2019, " + __author__ +copyright = "2023, " + __author__ # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst index 854af8a8..93dd96ac 100644 --- a/doc/source/configuration.rst +++ b/doc/source/configuration.rst @@ -3,14 +3,22 @@ Configuration This section is covering the Neomodel 'config' module and its variables. -Database --------- +Connection +---------- -Setting the connection URL:: +There are two ways to define your connection to the database : + +1. Provide a Neo4j URL and some options - Driver will be managed by neomodel +2. Create your own Neo4j driver and pass it to neomodel + +neomodel-managed (default) +-------------------------- + +Set the connection URL:: config.DATABASE_URL = 'bolt://neo4j:neo4j@localhost:7687` -Adjust driver configuration:: +Adjust driver configuration - these options are only available for this connection method:: config.MAX_CONNECTION_POOL_SIZE = 100 # default config.CONNECTION_ACQUISITION_TIMEOUT = 60.0 # default @@ -22,12 +30,88 @@ Adjust driver configuration:: config.MAX_TRANSACTION_RETRY_TIME = 30.0 # default config.RESOLVER = None # default config.TRUST = neo4j.TRUST_SYSTEM_CA_SIGNED_CERTIFICATES # default - config.USER_AGENT = neomodel/vNeo4j.Major.minor # default + config.USER_AGENT = neomodel/v5.1.1 # default -Setting the database name, for neo4j >= 4:: +Setting the database name, if different from the default one:: + # Using the URL only config.DATABASE_URL = 'bolt://neo4j:neo4j@localhost:7687/mydb` + # Using config option + config.DATABASE_URL = 'bolt://neo4j:neo4j@localhost:7687` + config.DATABASE_NAME = 'mydb' + +self-managed +------------ + +Create a Neo4j driver:: + + from neo4j import GraphDatabase + my_driver = GraphDatabase().driver('bolt://localhost:7687', auth=('neo4j', 'password')) + config.DRIVER = my_driver + +See the `driver documentation ` here. + +This mode allows you to use all the available driver options that neomodel doesn't implement, for example auth tokens for SSO. +Note that you have to manage the driver's lifecycle yourself. + +However, everything else is still handled by neomodel : sessions, transactions, etc... + +Change/Close the connection +--------------------------- + +Optionally, you can change the connection at any time by calling ``set_connection``:: + + from neomodel import db + # Using URL - auto-managed + db.set_connection(url='bolt://neo4j:neo4j@localhost:7687') + + # Using self-managed driver + db.set_connection(driver=my_driver) + +The new connection url will be applied to the current thread or process. + +Since Neo4j version 5, driver auto-close is deprecated. Make sure to close the connection anytime you want to replace it, +as well as at the end of your application's lifecycle by calling ``close_connection``:: + + from neomodel import db + db.close_connection() + + # If you then want a new connection + db.set_connection(url=url) + +This will close the Neo4j driver, and clean up everything that neomodel creates for its internal workings. + +Protect your credentials +------------------------ + +You should `avoid setting database access credentials in plain sight `_. Neo4J defines a number of +`environment variables `_ that are used in its tools and these can be re-used for other applications +too. + +These are: + +* ``NEO4J_USERNAME`` +* ``NEO4J_PASSWORD`` +* ``NEO4J_BOLT_URL`` + +By setting these with (for example): :: + + $ export NEO4J_USERNAME=neo4j + $ export NEO4J_PASSWORD=neo4j + $ export NEO4J_BOLT_URL="bolt://$NEO4J_USERNAME:$NEO4J_PASSWORD@localhost:7687" + +They can be accessed from a Python script via the ``environ`` dict of module ``os`` and be used to set the connection +with something like: :: + + import os + from neomodel import config + + config.DATABASE_URL = os.environ["NEO4J_BOLT_URL"] + + Enable automatic index and constraint creation ---------------------------------------------- diff --git a/doc/source/getting_started.rst b/doc/source/getting_started.rst index 3ec09ceb..19a2e38b 100644 --- a/doc/source/getting_started.rst +++ b/doc/source/getting_started.rst @@ -10,46 +10,11 @@ Before executing any neomodel code, set the connection url:: from neomodel import config config.DATABASE_URL = 'bolt://neo4j:neo4j@localhost:7687' # default - # You can specify a database name: 'bolt://neo4j:neo4j@localhost:7687/mydb' - This must be called early on in your app, if you are using Django the `settings.py` file is ideal. If you are using your neo4j server for the first time you will need to change the default password. This can be achieved by visiting the neo4j admin panel (default: ``http://localhost:7474`` ). -You can also change the connection url at any time by calling ``set_connection``:: - - from neomodel import db - db.set_connection('bolt://neo4j:neo4j@localhost:7687') - -The new connection url will be applied to the current thread or process. - -In general however, it is better to `avoid setting database access credentials in plain sight `_. Neo4J defines a number of -`environment variables `_ that are used in its tools and these can be re-used for other applications -too. - -These are: - -* ``NEO4J_USERNAME`` -* ``NEO4J_PASSWORD`` -* ``NEO4J_BOLT_URL`` - -By setting these with (for example): :: - - $ export NEO4J_USERNAME=neo4j - $ export NEO4J_PASSWORD=neo4j - $ export NEO4J_BOLT_URL="bolt://$NEO4J_USERNAME:$NEO4J_PASSWORD@localhost:7687" - -They can be accessed from a Python script via the ``environ`` dict of module ``os`` and be used to set the connection -with something like: :: - - import os - from neomodel import config - - config.DATABASE_URL = os.environ["NEO4J_BOLT_URL"] - Defining Node Entities and Relationships ======================================== From 0d355d23d17c8fade3630381f4c498efdc7943f9 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Wed, 6 Sep 2023 14:09:46 +0200 Subject: [PATCH 07/76] Reference Connection in Getting Started --- doc/source/configuration.rst | 2 ++ doc/source/getting_started.rst | 2 ++ 2 files changed, 4 insertions(+) diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst index 93dd96ac..2ddd24d9 100644 --- a/doc/source/configuration.rst +++ b/doc/source/configuration.rst @@ -3,6 +3,8 @@ Configuration This section is covering the Neomodel 'config' module and its variables. +.. _connection_options_doc: + Connection ---------- diff --git a/doc/source/getting_started.rst b/doc/source/getting_started.rst index 19a2e38b..92e08792 100644 --- a/doc/source/getting_started.rst +++ b/doc/source/getting_started.rst @@ -12,6 +12,8 @@ Before executing any neomodel code, set the connection url:: This must be called early on in your app, if you are using Django the `settings.py` file is ideal. +See the Configuration page (:ref:`connection_options_doc`) for config options. + If you are using your neo4j server for the first time you will need to change the default password. This can be achieved by visiting the neo4j admin panel (default: ``http://localhost:7474`` ). From 77adfe9643e577eded13f11ab93e07a9d2e3880f Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Wed, 6 Sep 2023 14:26:52 +0200 Subject: [PATCH 08/76] Add test for driver set through config --- test/test_connection.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/test_connection.py b/test/test_connection.py index 379556a2..61e7e10f 100644 --- a/test/test_connection.py +++ b/test/test_connection.py @@ -53,6 +53,21 @@ def test_set_connection_driver_works(): assert Pastry(name="Croissant").save() +def test_config_driver_works(): + # Verify that current connection is up + assert Pastry(name="Chausson aux pommes").save() + db.close_connection() + + # Test connection using a driver defined in config + driver = GraphDatabase().driver(NEO4J_URL, auth=(NEO4J_USERNAME, NEO4J_PASSWORD)) + config.DRIVER = driver + assert Pastry(name="Grignette").save() + + # Clear config + # No need to close connection - pytest teardown will do it + config.DRIVER = None + + @pytest.mark.skipif( db.database_edition != "enterprise", reason="Skipping test for community edition - no multi database in CE", From c5b3a763e6719382b05a45d68adf3adee10907fa Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Wed, 6 Sep 2023 15:06:29 +0200 Subject: [PATCH 09/76] Fix SessionExpired TODO and add notice in docstring --- neomodel/config.py | 2 +- neomodel/util.py | 6 +++--- test/test_connection.py | 2 ++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/neomodel/config.py b/neomodel/config.py index b54aa806..3396af5f 100644 --- a/neomodel/config.py +++ b/neomodel/config.py @@ -13,7 +13,7 @@ CONNECTION_TIMEOUT = 30.0 ENCRYPTED = False KEEP_ALIVE = True -MAX_CONNECTION_LIFETIME = 3600 +MAX_CONNECTION_LIFETIME = 30 MAX_CONNECTION_POOL_SIZE = 100 MAX_TRANSACTION_RETRY_TIME = 30.0 RESOLVER = None diff --git a/neomodel/util.py b/neomodel/util.py index 51cdbcf3..05795819 100644 --- a/neomodel/util.py +++ b/neomodel/util.py @@ -387,7 +387,8 @@ def cypher_query( :type: dict :param handle_unique: Whether or not to raise UniqueProperty exception on Cypher's ConstraintValidation errors :type: bool - :param retry_on_session_expire: Whether or not to attempt the same query again if the transaction has expired + :param retry_on_session_expire: Whether or not to attempt the same query again if the transaction has expired. + If you use neomodel with your own driver, you must catch SessionExpired exceptions yourself and retry with a new driver instance. :type: bool :param resolve_objects: Whether to attempt to resolve the returned nodes to data model objects automatically :type: bool @@ -449,8 +450,7 @@ def _run_cypher_query( raise exc_info[1].with_traceback(exc_info[2]) except SessionExpired: if retry_on_session_expire: - # TODO : What about if config passes driver instead of url ? - self.set_connection(self.url) + self.set_connection(url=self.url) return self.cypher_query( query=query, params=params, diff --git a/test/test_connection.py b/test/test_connection.py index 61e7e10f..fb3524bb 100644 --- a/test/test_connection.py +++ b/test/test_connection.py @@ -1,4 +1,5 @@ import os +import time import pytest from neo4j import GraphDatabase @@ -60,6 +61,7 @@ def test_config_driver_works(): # Test connection using a driver defined in config driver = GraphDatabase().driver(NEO4J_URL, auth=(NEO4J_USERNAME, NEO4J_PASSWORD)) + config.DRIVER = driver assert Pastry(name="Grignette").save() From 37327c66b05e81934a8e3eab7187d8876984b87a Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Wed, 6 Sep 2023 15:12:44 +0200 Subject: [PATCH 10/76] Update README, changelog and version tag --- Changelog | 5 +++++ README.rst | 14 -------------- doc/source/configuration.rst | 2 +- neomodel/_version.py | 2 +- pyproject.toml | 2 +- 5 files changed, 8 insertions(+), 17 deletions(-) diff --git a/Changelog b/Changelog index 9e0a1486..b9504967 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,8 @@ +Version 5.2.0 2023-09 +* Add an option to pass your own driver instead of relying on the automatically created one. See set_connection method. +* Add a close_connection method to explicitly close the driver to match Neo4j deprecation. +* Add a DATABASE_NAME config option, available for both auto- and self-managed driver modes. + Version 5.1.1 2023-08 * Add impersonation * Bumped neo4j-driver to 5.11.0 diff --git a/README.rst b/README.rst index 1a66076c..c4edceb0 100644 --- a/README.rst +++ b/README.rst @@ -55,20 +55,6 @@ Documentation .. _readthedocs: http://neomodel.readthedocs.org -Upcoming breaking changes notice - >=5.2 -======================================== - -As part of the current quality improvement efforts, we are planning a rework of neomodel's main Database object, which will lead to breaking changes. - -The full scope is not drawn out yet, but here are the main points : - -* Refactoring standalone methods that depend on the Database singleton into the class itself. See issue https://github.com/neo4j-contrib/neomodel/issues/739 - -* Adding an option to pass your own driver to neomodel instead of relying on the one that the library creates for you. This will not be a breaking change. - -We are aiming to release this in neomodel 5.2 - - Installation ============ diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst index 2ddd24d9..018de3fd 100644 --- a/doc/source/configuration.rst +++ b/doc/source/configuration.rst @@ -32,7 +32,7 @@ Adjust driver configuration - these options are only available for this connecti config.MAX_TRANSACTION_RETRY_TIME = 30.0 # default config.RESOLVER = None # default config.TRUST = neo4j.TRUST_SYSTEM_CA_SIGNED_CERTIFICATES # default - config.USER_AGENT = neomodel/v5.1.1 # default + config.USER_AGENT = neomodel/v5.2.0 # default Setting the database name, if different from the default one:: diff --git a/neomodel/_version.py b/neomodel/_version.py index a9c316e2..6c235c59 100644 --- a/neomodel/_version.py +++ b/neomodel/_version.py @@ -1 +1 @@ -__version__ = "5.1.1" +__version__ = "5.2.0" diff --git a/pyproject.toml b/pyproject.toml index 8fbca288..e515ed74 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ dependencies = [ "neobolt==1.7.17", "six==1.16.0", ] -version='5.1.1' +version='5.2.0' [project.urls] documentation = "https://neomodel.readthedocs.io/en/latest/" From 09c5fa80c12cd07f4ec10c7aefb13b56269402f1 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Fri, 8 Sep 2023 09:07:03 +0200 Subject: [PATCH 11/76] Refactor set_connection --- neomodel/util.py | 118 ++++++++++++++++++++++++++--------------------- 1 file changed, 66 insertions(+), 52 deletions(-) diff --git a/neomodel/util.py b/neomodel/util.py index 05795819..b577f8b8 100644 --- a/neomodel/util.py +++ b/neomodel/util.py @@ -90,58 +90,7 @@ def set_connection(self, url: str = None, driver: Driver = None): if hasattr(config, "DATABASE_NAME") and config.DATABASE_NAME: self._database_name = config.DATABASE_NAME elif url: - p_start = url.replace(":", "", 1).find(":") + 2 - p_end = url.rfind("@") - password = url[p_start:p_end] - url = url.replace(password, quote(password)) - parsed_url = urlparse(url) - - valid_schemas = [ - "bolt", - "bolt+s", - "bolt+ssc", - "bolt+routing", - "neo4j", - "neo4j+s", - "neo4j+ssc", - ] - - if parsed_url.netloc.find("@") > -1 and parsed_url.scheme in valid_schemas: - credentials, hostname = parsed_url.netloc.rsplit("@", 1) - username, password = credentials.split(":") - password = unquote(password) - database_name = parsed_url.path.strip("/") - else: - raise ValueError( - f"Expecting url format: bolt://user:password@localhost:7687 got {url}" - ) - - options = { - "auth": basic_auth(username, password), - "connection_acquisition_timeout": config.CONNECTION_ACQUISITION_TIMEOUT, - "connection_timeout": config.CONNECTION_TIMEOUT, - "keep_alive": config.KEEP_ALIVE, - "max_connection_lifetime": config.MAX_CONNECTION_LIFETIME, - "max_connection_pool_size": config.MAX_CONNECTION_POOL_SIZE, - "max_transaction_retry_time": config.MAX_TRANSACTION_RETRY_TIME, - "resolver": config.RESOLVER, - "user_agent": config.USER_AGENT, - } - - if "+s" not in parsed_url.scheme: - options["encrypted"] = config.ENCRYPTED - options["trusted_certificates"] = config.TRUSTED_CERTIFICATES - - self.driver = GraphDatabase.driver( - parsed_url.scheme + "://" + hostname, **options - ) - self.url = url - # The database name can be provided through the url or the config - if database_name == "": - if hasattr(config, "DATABASE_NAME") and config.DATABASE_NAME: - self._database_name = config.DATABASE_NAME - else: - self._database_name = database_name + self._parse_driver_from_url(url=url) self._pid = os.getpid() self._active_transaction = None @@ -154,6 +103,71 @@ def set_connection(self, url: str = None, driver: Driver = None): self._database_edition = None self._update_database_version() + def _parse_driver_from_url(self, url: str) -> None: + """Parse the driver information from the given URL and initialize the driver. + + Args: + url (str): The URL to parse. + + Raises: + ValueError: If the URL format is not as expected. + + Returns: + None - Sets the driver and database_name as class properties + """ + p_start = url.replace(":", "", 1).find(":") + 2 + p_end = url.rfind("@") + password = url[p_start:p_end] + url = url.replace(password, quote(password)) + parsed_url = urlparse(url) + + valid_schemas = [ + "bolt", + "bolt+s", + "bolt+ssc", + "bolt+routing", + "neo4j", + "neo4j+s", + "neo4j+ssc", + ] + + if parsed_url.netloc.find("@") > -1 and parsed_url.scheme in valid_schemas: + credentials, hostname = parsed_url.netloc.rsplit("@", 1) + username, password = credentials.split(":") + password = unquote(password) + database_name = parsed_url.path.strip("/") + else: + raise ValueError( + f"Expecting url format: bolt://user:password@localhost:7687 got {url}" + ) + + options = { + "auth": basic_auth(username, password), + "connection_acquisition_timeout": config.CONNECTION_ACQUISITION_TIMEOUT, + "connection_timeout": config.CONNECTION_TIMEOUT, + "keep_alive": config.KEEP_ALIVE, + "max_connection_lifetime": config.MAX_CONNECTION_LIFETIME, + "max_connection_pool_size": config.MAX_CONNECTION_POOL_SIZE, + "max_transaction_retry_time": config.MAX_TRANSACTION_RETRY_TIME, + "resolver": config.RESOLVER, + "user_agent": config.USER_AGENT, + } + + if "+s" not in parsed_url.scheme: + options["encrypted"] = config.ENCRYPTED + options["trusted_certificates"] = config.TRUSTED_CERTIFICATES + + self.driver = GraphDatabase.driver( + parsed_url.scheme + "://" + hostname, **options + ) + self.url = url + # The database name can be provided through the url or the config + if database_name == "": + if hasattr(config, "DATABASE_NAME") and config.DATABASE_NAME: + self._database_name = config.DATABASE_NAME + else: + self._database_name = database_name + def close_connection(self): """ Closes the currently open driver. From 0348feddaa430d368d789bdb66b526c50519060e Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 18 Sep 2023 10:10:05 +0200 Subject: [PATCH 12/76] Fix default config --- neomodel/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neomodel/config.py b/neomodel/config.py index 3396af5f..b54aa806 100644 --- a/neomodel/config.py +++ b/neomodel/config.py @@ -13,7 +13,7 @@ CONNECTION_TIMEOUT = 30.0 ENCRYPTED = False KEEP_ALIVE = True -MAX_CONNECTION_LIFETIME = 30 +MAX_CONNECTION_LIFETIME = 3600 MAX_CONNECTION_POOL_SIZE = 100 MAX_TRANSACTION_RETRY_TIME = 30.0 RESOLVER = None From 84e3b2df3f56dd998743dbfa93366e85c441057a Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 18 Sep 2023 10:14:17 +0200 Subject: [PATCH 13/76] Improve set_connection docstring --- neomodel/util.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/neomodel/util.py b/neomodel/util.py index b577f8b8..87b6e619 100644 --- a/neomodel/util.py +++ b/neomodel/util.py @@ -83,7 +83,14 @@ def __init__(self): def set_connection(self, url: str = None, driver: Driver = None): """ - Sets the connection URL to the address a Neo4j server is set up at + Sets the connection up and relevant internal. This can be done using a Neo4j URL or a driver instance. + + Args: + url (str): Optionally, Neo4j URL in the form protocol://username:password@hostname:port/dbname. + When provided, a Neo4j driver instance will be created by neomodel. + + driver (neo4j.Driver): Optionally, a pre-created driver instance. + When provided, neomodel will not create a driver instance but use this one instead. """ if driver: self.driver = driver From bac2a0c7c013da37946a9117ca8ccbf625dac889 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 18 Sep 2023 10:15:22 +0200 Subject: [PATCH 14/76] Update other core docstrings --- neomodel/util.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/neomodel/util.py b/neomodel/util.py index 87b6e619..ad4c51b4 100644 --- a/neomodel/util.py +++ b/neomodel/util.py @@ -178,8 +178,7 @@ def _parse_driver_from_url(self, url: str) -> None: def close_connection(self): """ Closes the currently open driver. - The driver should always be called at the end of the application's lifecyle. - If you pass your own driver to neomodel, you can also close it yourself without this method. + The driver should always be closed at the end of the application's lifecyle. """ self._database_version = None self._database_edition = None From 8f7db6efa4c6a099b7e4efb34f24f53b5b972f3b Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Fri, 13 Oct 2023 14:35:32 +0200 Subject: [PATCH 15/76] Add pandas dataframe and series support --- .github/workflows/integration-tests.yml | 2 +- doc/source/cypher.rst | 15 +++++++ doc/source/getting_started.rst | 8 ++++ neomodel/integration/pandas.py | 52 +++++++++++++++++++++++++ pyproject.toml | 1 + test/test_cypher.py | 38 ++++++++++++++++++ 6 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 neomodel/integration/pandas.py diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 8532c799..c794eb57 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -33,7 +33,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e '.[dev]' + pip install -e '.[dev,pandas]' - name: Test with pytest env: AURA_TEST_DB_USER: ${{ secrets.AURA_TEST_DB_USER }} diff --git a/doc/source/cypher.rst b/doc/source/cypher.rst index 98d65f8d..f9a010de 100644 --- a/doc/source/cypher.rst +++ b/doc/source/cypher.rst @@ -24,6 +24,21 @@ Outside of a `StructuredNode`:: The ``resolve_objects`` parameter automatically inflates the returned nodes to their defined classes (this is turned **off** by default). See :ref:`automatic_class_resolution` for details and possible pitfalls. +Integrations +============ + +Pandas +------ + +You can use the `pandas` library to return a `DataFrame` or `Series` object:: + + from neomodel import db + from neomodel.integration.pandas import to_dataframe, to_series + import pandas as pd + + df = to_dataframe(db.cypher_query("MATCH (a:Person) RETURN a.name AS name, a.born AS born")) + series = to_series(db.cypher_query("MATCH (a:Person) RETURN a.name AS name")) + Logging ======= diff --git a/doc/source/getting_started.rst b/doc/source/getting_started.rst index 3ec09ceb..52d61d67 100644 --- a/doc/source/getting_started.rst +++ b/doc/source/getting_started.rst @@ -50,6 +50,14 @@ with something like: :: config.DATABASE_URL = os.environ["NEO4J_BOLT_URL"] +Querying the graph +================== + +neomodel is mainly used as an OGM (see next section), but you also use it for direct Cypher queries : :: + + results, meta = db.cypher_query("RETURN 'Hello World' as message") + + Defining Node Entities and Relationships ======================================== diff --git a/neomodel/integration/pandas.py b/neomodel/integration/pandas.py new file mode 100644 index 00000000..983f91b7 --- /dev/null +++ b/neomodel/integration/pandas.py @@ -0,0 +1,52 @@ +""" +Provides integration with `pandas `_. + +.. note:: + This module requires pandas to be installed, and will raise a + warning if this is not available. + +Example: + + >>> from neomodel import db + >>> from neomodel.integration.pandas import to_dataframe + >>> db.set_connection('bolt://neo4j:secret@localhost:7687') + >>> df = to_dataframe(db.cypher_query("MATCH (a:Person) RETURN a.name AS name, a.born AS born")) + >>> df + email name + 0 jimla@test.com jimla + 1 jimlo@test.com jimlo + + [2 rows x 2 columns] + +""" + + +from warnings import warn + +try: + # noinspection PyPackageRequirements + from pandas import DataFrame, Series +except ImportError: + warn( + "The neomodel.integration.pandas module expects pandas to be installed " + "but it does not appear to be available." + ) + raise + + +def to_dataframe(query_results: tuple, index=None, dtype=None): + """Convert the results of a db.cypher_query call and associated metadata + into a pandas DataFrame. + Optionally, specify an index and/or a datatype for the columns. + """ + results, meta = query_results + return DataFrame(results, columns=meta, index=index, dtype=dtype) + + +def to_series(query_results: tuple, field=0, index=None, dtype=None): + """Convert the results of a db.cypher_query call + into a pandas Series for the given field. + Optionally, specify an index and/or a datatype for the columns. + """ + results, _ = query_results + return Series([record[field] for record in results], index=index, dtype=dtype) diff --git a/pyproject.toml b/pyproject.toml index c87f925c..26f30fe4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ dev = [ "isort", "Shapely>=2.0.0" ] +pandas = ["pandas"] [tool.pytest.ini_options] addopts = "--resetdb" diff --git a/test/test_cypher.py b/test/test_cypher.py index dfeccb4b..e5a974e5 100644 --- a/test/test_cypher.py +++ b/test/test_cypher.py @@ -1,10 +1,18 @@ from neo4j.exceptions import ClientError as CypherError +from pandas import DataFrame, Series from neomodel import StringProperty, StructuredNode from neomodel.core import db +from neomodel.integration.pandas import to_dataframe, to_series class User2(StructuredNode): + name = StringProperty() + email = StringProperty() + + +class User3(StructuredNode): + name = StringProperty() email = StringProperty() @@ -39,3 +47,33 @@ def test_cypher_syntax_error(): assert hasattr(e, "code") else: assert False, "CypherError not raised." + + +def test_pandas_dataframe_integration(): + jimla = User2(email="jimla@test.com", name="jimla").save() + jimlo = User2(email="jimlo@test.com", name="jimlo").save() + df = to_dataframe( + db.cypher_query("MATCH (a:User2) RETURN a.name AS name, a.email AS email") + ) + + assert isinstance(df, DataFrame) + assert df.shape == (2, 2) + assert df["name"].tolist() == ["jimla", "jimlo"] + + # Also test passing an index and dtype to to_dataframe + df = to_dataframe( + db.cypher_query("MATCH (a:User2) RETURN a.name AS name, a.email AS email"), + index=df["email"], + dtype=str, + ) + + assert df.index.inferred_type == "string" + + +def test_pandas_series_integration(): + jimly = User3(email="jimly@test.com", name="jimly").save() + series = to_series(db.cypher_query("MATCH (a:User3) RETURN a.name AS name")) + + assert isinstance(series, Series) + assert series.shape == (1,) + assert series.tolist() == ["jimly"] From b5f4fde8fc6f33622d257e42140796f6173fe791 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Fri, 13 Oct 2023 14:49:57 +0200 Subject: [PATCH 16/76] Update pandas integration doc --- doc/source/cypher.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/source/cypher.rst b/doc/source/cypher.rst index f9a010de..7b040b4a 100644 --- a/doc/source/cypher.rst +++ b/doc/source/cypher.rst @@ -30,6 +30,14 @@ Integrations Pandas ------ +First, you need to install pandas by yourself. We do not include it by default to keep that package size controlled:: + + # When installing neomodel + pip install neomodel[pandas] + + # Or separately + pip install pandas + You can use the `pandas` library to return a `DataFrame` or `Series` object:: from neomodel import db From 6bb7c900bf9ba8495f048b10800757d1b072724b Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Fri, 13 Oct 2023 14:50:19 +0200 Subject: [PATCH 17/76] Fix typo in pandas integration doc --- doc/source/cypher.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/cypher.rst b/doc/source/cypher.rst index 7b040b4a..c698c60e 100644 --- a/doc/source/cypher.rst +++ b/doc/source/cypher.rst @@ -30,7 +30,7 @@ Integrations Pandas ------ -First, you need to install pandas by yourself. We do not include it by default to keep that package size controlled:: +First, you need to install pandas by yourself. We do not include it by default to keep the package size controlled:: # When installing neomodel pip install neomodel[pandas] From 23c7491e6fb96fb623307adba9589a7e1e8c1eab Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Fri, 13 Oct 2023 15:08:20 +0200 Subject: [PATCH 18/76] Fix typo in pandas integration docstring --- neomodel/integration/pandas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neomodel/integration/pandas.py b/neomodel/integration/pandas.py index 983f91b7..1ad19871 100644 --- a/neomodel/integration/pandas.py +++ b/neomodel/integration/pandas.py @@ -10,7 +10,7 @@ >>> from neomodel import db >>> from neomodel.integration.pandas import to_dataframe >>> db.set_connection('bolt://neo4j:secret@localhost:7687') - >>> df = to_dataframe(db.cypher_query("MATCH (a:Person) RETURN a.name AS name, a.born AS born")) + >>> df = to_dataframe(db.cypher_query("MATCH (u:User) RETURN u.email AS email, u.name AS name")) >>> df email name 0 jimla@test.com jimla From e3352ba61bc143af0451fc9b4b9d7d1c0aa607c3 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Fri, 13 Oct 2023 15:23:00 +0200 Subject: [PATCH 19/76] Add numpy ndarray integration --- .github/workflows/integration-tests.yml | 2 +- doc/source/cypher.rst | 21 ++++++++++++-- neomodel/integration/numpy.py | 37 +++++++++++++++++++++++++ pyproject.toml | 1 + test/test_cypher.py | 27 ++++++++++++++---- 5 files changed, 79 insertions(+), 9 deletions(-) create mode 100644 neomodel/integration/numpy.py diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index c794eb57..3469f200 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -33,7 +33,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e '.[dev,pandas]' + pip install -e '.[dev,pandas,numpy]' - name: Test with pytest env: AURA_TEST_DB_USER: ${{ secrets.AURA_TEST_DB_USER }} diff --git a/doc/source/cypher.rst b/doc/source/cypher.rst index c698c60e..f8c7ccaf 100644 --- a/doc/source/cypher.rst +++ b/doc/source/cypher.rst @@ -38,15 +38,32 @@ First, you need to install pandas by yourself. We do not include it by default t # Or separately pip install pandas -You can use the `pandas` library to return a `DataFrame` or `Series` object:: +You can use the `pandas` integration to return a `DataFrame` or `Series` object:: from neomodel import db from neomodel.integration.pandas import to_dataframe, to_series - import pandas as pd df = to_dataframe(db.cypher_query("MATCH (a:Person) RETURN a.name AS name, a.born AS born")) series = to_series(db.cypher_query("MATCH (a:Person) RETURN a.name AS name")) +Numpy +------ + +First, you need to install numpy by yourself. We do not include it by default to keep the package size controlled:: + + # When installing neomodel + pip install neomodel[numpy] + + # Or separately + pip install numpy + +You can use the `numpy` integration to return a `ndarray` object:: + + from neomodel import db + from neomodel.integration.numpy import to_ndarray + + array = to_ndarray(db.cypher_query("MATCH (a:Person) RETURN a.name AS name, a.born AS born")) + Logging ======= diff --git a/neomodel/integration/numpy.py b/neomodel/integration/numpy.py new file mode 100644 index 00000000..a04508c4 --- /dev/null +++ b/neomodel/integration/numpy.py @@ -0,0 +1,37 @@ +""" +Provides integration with `numpy `_. + +.. note:: + This module requires numpy to be installed, and will raise a + warning if this is not available. + +Example: + + >>> from neomodel import db + >>> from neomodel.integration.numpy import to_nparray + >>> db.set_connection('bolt://neo4j:secret@localhost:7687') + >>> df = to_nparray(db.cypher_query("MATCH (u:User) RETURN u.email AS email, u.name AS name")) + >>> df + array([['jimla@test.com', 'jimla'], ['jimlo@test.com', 'jimlo']]) +""" + + +from warnings import warn + +try: + # noinspection PyPackageRequirements + from numpy import array as nparray +except ImportError: + warn( + "The neomodel.integration.numpy module expects numpy to be installed " + "but it does not appear to be available." + ) + raise + + +def to_ndarray(query_results: tuple, dtype=None, order="K"): + """Convert the results of a db.cypher_query call into a numpy array. + Optionally, specify a datatype and/or an order for the columns. + """ + results, _ = query_results + return nparray(results, dtype=dtype, order=order) diff --git a/pyproject.toml b/pyproject.toml index 26f30fe4..743182bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ dev = [ "Shapely>=2.0.0" ] pandas = ["pandas"] +numpy = ["numpy"] [tool.pytest.ini_options] addopts = "--resetdb" diff --git a/test/test_cypher.py b/test/test_cypher.py index e5a974e5..f5be431a 100644 --- a/test/test_cypher.py +++ b/test/test_cypher.py @@ -1,8 +1,10 @@ from neo4j.exceptions import ClientError as CypherError +from numpy import ndarray from pandas import DataFrame, Series from neomodel import StringProperty, StructuredNode from neomodel.core import db +from neomodel.integration.numpy import to_ndarray from neomodel.integration.pandas import to_dataframe, to_series @@ -49,9 +51,11 @@ def test_cypher_syntax_error(): assert False, "CypherError not raised." -def test_pandas_dataframe_integration(): +def test_pandas_integration(): jimla = User2(email="jimla@test.com", name="jimla").save() jimlo = User2(email="jimlo@test.com", name="jimlo").save() + + # Test to_dataframe df = to_dataframe( db.cypher_query("MATCH (a:User2) RETURN a.name AS name, a.email AS email") ) @@ -69,11 +73,22 @@ def test_pandas_dataframe_integration(): assert df.index.inferred_type == "string" + # Next test to_series + series = to_series(db.cypher_query("MATCH (a:User2) RETURN a.name AS name")) + + assert isinstance(series, Series) + assert series.shape == (2,) + assert df["name"].tolist() == ["jimla", "jimlo"] + -def test_pandas_series_integration(): +def test_numpy_integration(): jimly = User3(email="jimly@test.com", name="jimly").save() - series = to_series(db.cypher_query("MATCH (a:User3) RETURN a.name AS name")) + jimlu = User3(email="jimlu@test.com", name="jimlu").save() - assert isinstance(series, Series) - assert series.shape == (1,) - assert series.tolist() == ["jimly"] + array = to_ndarray( + db.cypher_query("MATCH (a:User3) RETURN a.name AS name, a.email AS email") + ) + + assert isinstance(array, ndarray) + assert array.shape == (2, 2) + assert array[0][0] == "jimly" From d7fda767f6c35e03cf7d775d51d4c60ba508b827 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Fri, 13 Oct 2023 17:35:47 +0200 Subject: [PATCH 20/76] Add tests for install/remove labels scripts --- test/test_scripts.py | 84 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 test/test_scripts.py diff --git a/test/test_scripts.py b/test/test_scripts.py new file mode 100644 index 00000000..017f36e8 --- /dev/null +++ b/test/test_scripts.py @@ -0,0 +1,84 @@ +import subprocess + +from neomodel import ( + RelationshipTo, + StringProperty, + StructuredNode, + StructuredRel, + config, + db, +) + + +class ScriptsTestRel(StructuredRel): + some_unique_property = StringProperty(unique_index=True) + some_index_property = StringProperty(index=True) + + +class ScriptsTestNode(StructuredNode): + personal_id = StringProperty(unique_index=True) + name = StringProperty(index=True) + rel = RelationshipTo("ScriptsTestNode", "REL", model=ScriptsTestRel) + + +def test_neomodel_install_labels(): + result = subprocess.run( + ["neomodel_install_labels", "--help"], + capture_output=True, + text=True, + check=False, + ) + assert "usage: neomodel_install_labels" in result.stdout + assert result.returncode == 0 + + result = subprocess.run( + ["neomodel_install_labels", "test/test_scripts.py", "--db", db.url], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0 + assert "Setting up indexes and constraints" in result.stdout + constraints = db.list_constraints() + parsed_constraints = [ + (element["type"], element["labelsOrTypes"], element["properties"]) + for element in constraints + ] + assert ("UNIQUENESS", ["ScriptsTestNode"], ["personal_id"]) in parsed_constraints + assert ( + "RELATIONSHIP_UNIQUENESS", + ["REL"], + ["some_unique_property"], + ) in parsed_constraints + indexes = db.list_indexes() + parsed_indexes = [ + (element["labelsOrTypes"], element["properties"]) for element in indexes + ] + assert (["ScriptsTestNode"], ["name"]) in parsed_indexes + assert (["REL"], ["some_index_property"]) in parsed_indexes + + +def test_neomodel_remove_labels(): + result = subprocess.run( + ["neomodel_remove_labels", "--help"], + capture_output=True, + text=True, + check=False, + ) + assert "usage: neomodel_remove_labels" in result.stdout + assert result.returncode == 0 + + result = subprocess.run( + ["neomodel_remove_labels", "--db", config.DATABASE_URL], + capture_output=True, + text=True, + check=False, + ) + assert ( + "Dropping unique constraint and index on label ScriptsTestNode" in result.stdout + ) + assert result.returncode == 0 + constraints = db.list_constraints() + indexes = db.list_indexes(exclude_token_lookup=True) + assert len(constraints) == 0 + assert len(indexes) == 0 From 3452614a0af7efbe4317b57e07720f435574e150 Mon Sep 17 00:00:00 2001 From: curious-broccoli <77789413+curious-broccoli@users.noreply.github.com> Date: Tue, 17 Oct 2023 17:18:49 +0200 Subject: [PATCH 21/76] Fix typo in variable name --- neomodel/properties.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/neomodel/properties.py b/neomodel/properties.py index 02682f0f..2fdbaf34 100644 --- a/neomodel/properties.py +++ b/neomodel/properties.py @@ -378,16 +378,16 @@ def __init__(self, base_property=None, **kwargs): if isinstance(base_property, ArrayProperty): raise TypeError("Cannot have nested ArrayProperty") - for ilegal_attr in [ + for illegal_attr in [ "default", "index", "unique_index", "required", ]: - if getattr(base_property, ilegal_attr, None): + if getattr(base_property, illegal_attr, None): raise ValueError( 'ArrayProperty base_property cannot have "{0}" set'.format( - ilegal_attr + illegal_attr ) ) From 97b22044644e41a2147ee07710559621ae5a37eb Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 24 Oct 2023 09:48:58 +0200 Subject: [PATCH 22/76] Add relationship unique index for >=5.5 only --- test/test_scripts.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/test_scripts.py b/test/test_scripts.py index 017f36e8..4b131df3 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -11,7 +11,7 @@ class ScriptsTestRel(StructuredRel): - some_unique_property = StringProperty(unique_index=True) + some_unique_property = StringProperty(unique_index=db.version_is_higher_than("5.5")) some_index_property = StringProperty(index=True) @@ -45,11 +45,12 @@ def test_neomodel_install_labels(): for element in constraints ] assert ("UNIQUENESS", ["ScriptsTestNode"], ["personal_id"]) in parsed_constraints - assert ( - "RELATIONSHIP_UNIQUENESS", - ["REL"], - ["some_unique_property"], - ) in parsed_constraints + if db.version_is_higher_than("5.5"): + assert ( + "RELATIONSHIP_UNIQUENESS", + ["REL"], + ["some_unique_property"], + ) in parsed_constraints indexes = db.list_indexes() parsed_indexes = [ (element["labelsOrTypes"], element["properties"]) for element in indexes From e011daae5b886993c70ae877f3430c029681ee47 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 24 Oct 2023 09:57:09 +0200 Subject: [PATCH 23/76] Fix Rel class definition --- test/test_scripts.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/test_scripts.py b/test/test_scripts.py index 4b131df3..9c7df526 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -11,7 +11,11 @@ class ScriptsTestRel(StructuredRel): - some_unique_property = StringProperty(unique_index=db.version_is_higher_than("5.5")) + some_unique_property = ( + StringProperty(unique_index=True) + if db.version_is_higher_than("5.5") + else StringProperty() + ) some_index_property = StringProperty(index=True) From fa60e5cd82f2b8785fb04df748101fe6f92a87c2 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 24 Oct 2023 09:59:55 +0200 Subject: [PATCH 24/76] Fix version tag for rel constraint --- test/test_scripts.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/test/test_scripts.py b/test/test_scripts.py index 9c7df526..e4cd8ec6 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -11,11 +11,7 @@ class ScriptsTestRel(StructuredRel): - some_unique_property = ( - StringProperty(unique_index=True) - if db.version_is_higher_than("5.5") - else StringProperty() - ) + some_unique_property = StringProperty(unique_index=db.version_is_higher_than("5.7")) some_index_property = StringProperty(index=True) @@ -49,7 +45,7 @@ def test_neomodel_install_labels(): for element in constraints ] assert ("UNIQUENESS", ["ScriptsTestNode"], ["personal_id"]) in parsed_constraints - if db.version_is_higher_than("5.5"): + if db.version_is_higher_than("5.7"): assert ( "RELATIONSHIP_UNIQUENESS", ["REL"], From be434c28357e4d2637c46ee9c5eb5704f9742a65 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Thu, 26 Oct 2023 11:54:03 +0200 Subject: [PATCH 25/76] Fix typo in docs --- doc/source/getting_started.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/getting_started.rst b/doc/source/getting_started.rst index 52d61d67..38bf1c3e 100644 --- a/doc/source/getting_started.rst +++ b/doc/source/getting_started.rst @@ -53,7 +53,7 @@ with something like: :: Querying the graph ================== -neomodel is mainly used as an OGM (see next section), but you also use it for direct Cypher queries : :: +neomodel is mainly used as an OGM (see next section), but you can also use it for direct Cypher queries : :: results, meta = db.cypher_query("RETURN 'Hello World' as message") From a6f3730cc8c5ee62faf1be45f3941391204fefb9 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Thu, 26 Oct 2023 12:45:49 +0200 Subject: [PATCH 26/76] Fix tests --- test/test_cypher.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/test/test_cypher.py b/test/test_cypher.py index f5be431a..2ca39a51 100644 --- a/test/test_cypher.py +++ b/test/test_cypher.py @@ -13,7 +13,12 @@ class User2(StructuredNode): email = StringProperty() -class User3(StructuredNode): +class UserPandas(StructuredNode): + name = StringProperty() + email = StringProperty() + + +class UserNP(StructuredNode): name = StringProperty() email = StringProperty() @@ -52,12 +57,12 @@ def test_cypher_syntax_error(): def test_pandas_integration(): - jimla = User2(email="jimla@test.com", name="jimla").save() - jimlo = User2(email="jimlo@test.com", name="jimlo").save() + jimla = UserPandas(email="jimla@test.com", name="jimla").save() + jimlo = UserPandas(email="jimlo@test.com", name="jimlo").save() # Test to_dataframe df = to_dataframe( - db.cypher_query("MATCH (a:User2) RETURN a.name AS name, a.email AS email") + db.cypher_query("MATCH (a:UserPandas) RETURN a.name AS name, a.email AS email") ) assert isinstance(df, DataFrame) @@ -66,7 +71,7 @@ def test_pandas_integration(): # Also test passing an index and dtype to to_dataframe df = to_dataframe( - db.cypher_query("MATCH (a:User2) RETURN a.name AS name, a.email AS email"), + db.cypher_query("MATCH (a:UserPandas) RETURN a.name AS name, a.email AS email"), index=df["email"], dtype=str, ) @@ -74,7 +79,7 @@ def test_pandas_integration(): assert df.index.inferred_type == "string" # Next test to_series - series = to_series(db.cypher_query("MATCH (a:User2) RETURN a.name AS name")) + series = to_series(db.cypher_query("MATCH (a:UserPandas) RETURN a.name AS name")) assert isinstance(series, Series) assert series.shape == (2,) @@ -82,11 +87,11 @@ def test_pandas_integration(): def test_numpy_integration(): - jimly = User3(email="jimly@test.com", name="jimly").save() - jimlu = User3(email="jimlu@test.com", name="jimlu").save() + jimly = UserNP(email="jimly@test.com", name="jimly").save() + jimlu = UserNP(email="jimlu@test.com", name="jimlu").save() array = to_ndarray( - db.cypher_query("MATCH (a:User3) RETURN a.name AS name, a.email AS email") + db.cypher_query("MATCH (a:UserNP) RETURN a.name AS name, a.email AS email") ) assert isinstance(array, ndarray) From b679c52f8c51365e62bb811d9a11333c1f6262ff Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Thu, 26 Oct 2023 16:11:15 +0200 Subject: [PATCH 27/76] Add note that self-managed driver is synchronous only --- doc/source/configuration.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst index 018de3fd..d5299f54 100644 --- a/doc/source/configuration.rst +++ b/doc/source/configuration.rst @@ -59,6 +59,8 @@ Note that you have to manage the driver's lifecycle yourself. However, everything else is still handled by neomodel : sessions, transactions, etc... +NB : Only the synchronous driver will work in this way. The asynchronous driver is not supported yet. + Change/Close the connection --------------------------- From 685b2707d9bd1e548bad5ccc3283b6c3cf8a4679 Mon Sep 17 00:00:00 2001 From: Athanasios Anastasiou Date: Sun, 29 Oct 2023 02:12:26 +0200 Subject: [PATCH 28/76] Corrected bug in element_id. If the object is initialised but not yet saved then its element_id should return None. Previously was failing because element_id_property is a dynamic one and did not exist upon object creation. Also corrected tests because element_id was always tested AFTER a node model was getting saved (added test to check the case of interrogating element_id on an unsaved node model) --- neomodel/core.py | 4 ++-- test/test_models.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/neomodel/core.py b/neomodel/core.py index e89b3a26..b293dc5d 100644 --- a/neomodel/core.py +++ b/neomodel/core.py @@ -413,7 +413,7 @@ def element_id(self): int(self.element_id_property) if db.database_version.startswith("4") else self.element_id_property - ) + ) if hasattr(self, "element_id_property") else None # Version 4.4 support - id is deprecated in version 5.x @property @@ -758,7 +758,7 @@ def save(self): """ # create or update instance node - if hasattr(self, "element_id"): + if hasattr(self, "element_id_property"): # update params = self.deflate(self.__properties__, self) query = f"MATCH (n) WHERE {db.get_id_method()}(n)=$self\n" diff --git a/test/test_models.py b/test/test_models.py index 7fbd80a4..827c705a 100644 --- a/test/test_models.py +++ b/test/test_models.py @@ -98,10 +98,18 @@ def test_first_and_first_or_none(): assert n is None +def test_bare_init_without_save(): + """ + If a node model is initialised without being saved, accessing its `element_id` should + return None. + """ + assert(User().element_id is None) + + def test_save_to_model(): u = User(email="jim@test.com", age=3) assert u.save() - assert u.element_id != "" + assert u.element_id is not None assert u.email == "jim@test.com" assert u.age == 3 @@ -109,7 +117,7 @@ def test_save_to_model(): def test_save_node_without_properties(): n = NodeWithoutProperty() assert n.save() - assert n.element_id != "" + assert n.element_id is not None def test_unique(): From 7876b88e57b20d73c5318f73c8e1f52138bbda57 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 10:48:07 +0100 Subject: [PATCH 29/76] Add inspection script --- neomodel/scripts/neomodel_inspect_database.py | 356 ++++++++++++++++++ pyproject.toml | 1 + 2 files changed, 357 insertions(+) create mode 100644 neomodel/scripts/neomodel_inspect_database.py diff --git a/neomodel/scripts/neomodel_inspect_database.py b/neomodel/scripts/neomodel_inspect_database.py new file mode 100644 index 00000000..6d28c68d --- /dev/null +++ b/neomodel/scripts/neomodel_inspect_database.py @@ -0,0 +1,356 @@ +""" +.. _neomodel_inspect_database: + +``_neomodel_inspect_database`` +--------------------------- + +:: + + usage: _neomodel_inspect_database [-h] [--db bolt://neo4j:neo4j@localhost:7687] [--write-to ...] + + Connects to a Neo4j database and inspects existing nodes and relationships. + Infers the schema of the database and generates Python class definitions. + + If a connection URL is not specified, the tool will look up the environment + variable NEO4J_BOLT_URL. If that environment variable is not set, the tool + will attempt to connect to the default URL bolt://neo4j:neo4j@localhost:7687 + + If a file is specified, the tool will write the class definitions to that file. + If no file is specified, the tool will print the class definitions to stdout. + + options: + -h, --help show this help message and exit + --db bolt://neo4j:neo4j@localhost:7687 + Neo4j Server URL + -T, --write-to someapp/models.py + File where to write output. +""" + +import argparse +import string +import textwrap +from os import environ + +from neomodel import db + +IMPORTS = [] + + +def parse_prop_class(prop_type): + if prop_type.startswith("LIST OF"): + if "ArrayProperty" not in IMPORTS: + IMPORTS.append("ArrayProperty") + return f"ArrayProperty({parse_prop_class(prop_type.replace('LIST OF ', ''))})" + else: + if prop_type == "STRING": + if "StringProperty" not in IMPORTS: + IMPORTS.append("StringProperty") + return "StringProperty(" + elif prop_type == "BOOLEAN": + if "BooleanProperty" not in IMPORTS: + IMPORTS.append("BooleanProperty") + return "BooleanProperty(" + elif prop_type == "DATE_TIME": + if "DateTimeProperty" not in IMPORTS: + IMPORTS.append("DateTimeProperty") + return "DateTimeProperty(" + elif prop_type == "INTEGER": + if "IntegerProperty" not in IMPORTS: + IMPORTS.append("IntegerProperty") + return "IntegerProperty(" + elif prop_type == "FLOAT": + if "FloatProperty" not in IMPORTS: + IMPORTS.append("FloatProperty") + return "FloatProperty(" + elif prop_type == "POINT": + if "PointProperty" not in IMPORTS: + IMPORTS.append("PointProperty") + return "PointProperty(" + + +class NodeInspector: + @staticmethod + def get_properties_for_label(label): + query = f""" + MATCH (n:`{label}`) + WITH DISTINCT keys(n) as properties, head(collect(n)) AS sampleNode + ORDER BY size(properties) DESC + RETURN apoc.meta.cypher.types(properties(sampleNode)) AS properties LIMIT 1 + """ + result, _ = db.cypher_query(query) + if result is not None and len(result) > 0: + return result[0][0] + return {} + + @staticmethod + def get_constraints_for_label(label): + constraints, meta_constraints = db.cypher_query( + f"SHOW CONSTRAINTS WHERE entityType='NODE' AND '{label}' IN labelsOrTypes AND type='UNIQUENESS'" + ) + constraints_as_dict = [dict(zip(meta_constraints, row)) for row in constraints] + constrained_properties = [ + item.get("properties")[0] + for item in constraints_as_dict + if len(item.get("properties")) == 1 + ] + return constrained_properties + + @staticmethod + def get_indexed_properties_for_label(label): + indexes, meta_indexes = db.cypher_query( + f"SHOW INDEXES WHERE entityType='NODE' AND '{label}' IN labelsOrTypes AND type='RANGE' AND owningConstraint IS NULL" + ) + indexes_as_dict = [dict(zip(meta_indexes, row)) for row in indexes] + indexed_properties = [ + item.get("properties")[0] + for item in indexes_as_dict + if len(item.get("properties")) == 1 + ] + return indexed_properties + + +class RelationshipInspector: + @classmethod + def outgoing_relationships(cls, start_label): + query = f""" + MATCH (n:`{start_label}`)-[r]->(m) + WITH DISTINCT type(r) as rel_type, head(labels(m)) AS target_label, keys(r) AS properties, head(collect(r)) AS sampleRel + ORDER BY size(properties) DESC + RETURN rel_type, target_label, apoc.meta.cypher.types(properties(sampleRel)) AS properties LIMIT 1 + """ + result, _ = db.cypher_query(query) + return [(record[0], record[1], record[2]) for record in result] + + @staticmethod + def get_constraints_for_type(rel_type): + constraints, meta_constraints = db.cypher_query( + f"SHOW CONSTRAINTS WHERE entityType='RELATIONSHIP' AND '{rel_type}' IN labelsOrTypes AND type='UNIQUENESS'" + ) + constraints_as_dict = [dict(zip(meta_constraints, row)) for row in constraints] + constrained_properties = [ + item.get("properties")[0] + for item in constraints_as_dict + if len(item.get("properties")) == 1 + ] + return constrained_properties + + @staticmethod + def get_indexed_properties_for_type(rel_type): + indexes, meta_indexes = db.cypher_query( + f"SHOW INDEXES WHERE entityType='RELATIONSHIP' AND '{rel_type}' IN labelsOrTypes AND type='RANGE' AND owningConstraint IS NULL" + ) + indexes_as_dict = [dict(zip(meta_indexes, row)) for row in indexes] + indexed_properties = [ + item.get("properties")[0] + for item in indexes_as_dict + if len(item.get("properties")) == 1 + ] + return indexed_properties + + @staticmethod + def infer_cardinality(rel_type, start_label): + range_start_query = f"MATCH (n:`{start_label}`) WHERE NOT EXISTS ((n)-[:`{rel_type}`]->()) WITH n LIMIT 1 RETURN count(n)" + result, _ = db.cypher_query(range_start_query) + is_start_zero = result[0][0] > 0 + + range_end_query = f""" + MATCH (n:`{start_label}`)-[rel:`{rel_type}`]->() + WITH n, count(rel) AS rel_count + WHERE rel_count > 1 + WITH n LIMIT 1 + RETURN count(n) + """ + result, _ = db.cypher_query(range_end_query) + is_end_one = result[0][0] == 0 + + cardinality = "Zero" if is_start_zero else "One" + cardinality += "OrOne" if is_end_one and is_start_zero else "OrMore" + + if cardinality not in IMPORTS: + IMPORTS.append(cardinality) + + return cardinality + + +def get_node_labels(): + query = "CALL db.labels()" + result, _ = db.cypher_query(query) + return [record[0] for record in result] + + +def get_relationship_types(): + query = "CALL db.relationshipTypes()" + result, _ = db.cypher_query(query) + return [record[0] for record in result] + + +def build_prop_string(unique_properties, indexed_properties, prop, prop_type): + is_unique = prop in unique_properties + is_indexed = prop in indexed_properties + index_str = "" + if is_unique: + index_str = "unique_index=True" + elif is_indexed: + index_str = "index=True" + return f" {prop.replace(' ', '_')} = {parse_prop_class(prop_type)}{index_str})\n" + + +def clean_class_member_key(key): + return key.replace(" ", "_") + + +def generate_rel_class_name(rel_type): + # Relationship type best practices are like FRIENDS_WITH + # We want to convert that to FriendsWithRel + return string.capwords(rel_type.replace("_", " ")).replace(" ", "") + "Rel" + + +def inspect_database(bolt_url): + # Connect to the database + print(f"Connecting to {bolt_url}") + db.set_connection(bolt_url) + + node_labels = get_node_labels() + defined_rel_types = [] + class_definitions = "" + + if node_labels: + IMPORTS.append("StructuredNode") + + for label in node_labels: + class_name = clean_class_member_key(label) + properties = NodeInspector.get_properties_for_label(label) + unique_properties = NodeInspector.get_constraints_for_label(label) + indexed_properties = NodeInspector.get_indexed_properties_for_label(label) + + class_definition = f"class {class_name}(StructuredNode):\n" + class_definition += "".join( + [ + build_prop_string( + unique_properties, indexed_properties, prop, prop_type + ) + for prop, prop_type in properties.items() + ] + ) + + outgoing_relationships = RelationshipInspector.outgoing_relationships(label) + rel_type_definitions = "" + + if outgoing_relationships and "StructuredRel" not in IMPORTS: + IMPORTS.append("RelationshipTo") + IMPORTS.append("StructuredRel") + + for rel in outgoing_relationships: + rel_type = rel[0] + rel_name = rel_type.lower() + target_label = rel[1] + rel_props = rel[2] + + unique_properties = RelationshipInspector.get_constraints_for_type(rel_type) + indexed_properties = RelationshipInspector.get_indexed_properties_for_type( + rel_type + ) + + cardinality = RelationshipInspector.infer_cardinality(rel_type, label) + + class_definition += f' {clean_class_member_key(rel_name)} = RelationshipTo("{target_label}", "{rel_type}", cardinality={cardinality}' + + if rel_props and rel_type not in defined_rel_types: + rel_model_name = generate_rel_class_name(rel_type) + class_definition += f', model="{rel_model_name}"' + rel_type_definitions = f"\n\nclass {rel_model_name}(StructuredRel):\n" + if properties: + rel_type_definitions += "".join( + [ + build_prop_string( + unique_properties, indexed_properties, prop, prop_type + ) + for prop, prop_type in properties.items() + ] + ) + else: + rel_type_definitions += " pass\n" + defined_rel_types.append(rel_type) + + class_definition += ")\n" + + class_definition += rel_type_definitions + + if not properties and not outgoing_relationships: + class_definition += " pass\n" + + class_definition += "\n\n" + + class_definitions += class_definition + + # Finally, parse imports + if IMPORTS: + special_imports = "" + if "PointProperty" in IMPORTS: + IMPORTS.remove("PointProperty") + special_imports += ( + "from neomodel.contrib.spatial_properties import PointProperty\n" + ) + imports = f"from neomodel import {', '.join(IMPORTS)}\n" + special_imports + output = "\n".join([imports, class_definitions]) + + return output + + +def main(): + # Parse command line arguments + parser = argparse.ArgumentParser( + description=textwrap.dedent( + """ + Connects to a Neo4j database and inspects existing nodes and relationships. + Infers the schema of the database and generates Python class definitions. + + If a connection URL is not specified, the tool will look up the environment + variable NEO4J_BOLT_URL. If that environment variable is not set, the tool + will attempt to connect to the default URL bolt://neo4j:neo4j@localhost:7687 + + If a file is specified, the tool will write the class definitions to that file. + If no file is specified, the tool will print the class definitions to stdout. + """ + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + "--db", + metavar="bolt://neo4j:neo4j@localhost:7687", + dest="neo4j_bolt_url", + type=str, + default="", + help="Neo4j Server URL", + ) + + parser.add_argument( + "-T", + "--write-to", + metavar="", + type=str, + help="File where to write output.", + ) + + args = parser.parse_args() + + bolt_url = args.neo4j_bolt_url + if len(bolt_url) == 0: + bolt_url = environ.get("NEO4J_BOLT_URL", "bolt://neo4j:neo4j@localhost:7687") + + # If a file is specified, write to that file + # First try to open the file for writing to make sure it is writable + # Before connecting to the database + if args.write_to: + with open(args.write_to, "w") as file: + output = inspect_database(bolt_url=bolt_url) + print(f"Writing to {args.write_to}") + file.write(output) + # If no file is specified, print to stdout + else: + print(inspect_database(bolt_url=bolt_url)) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index c87f925c..8e164f67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,3 +76,4 @@ max-args = 8 [project.scripts] neomodel_install_labels = "neomodel.scripts.neomodel_install_labels:main" neomodel_remove_labels = "neomodel.scripts.neomodel_remove_labels:main" +neomodel_inspect_database = "neomodel.scripts.neomodel_inspect_database:main" From e62c0fa54a09d1b1c8c392dce4ee270e9d75c9c3 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 13:32:36 +0100 Subject: [PATCH 30/76] Create a database inspection script, with tests --- neomodel/scripts/neomodel_inspect_database.py | 2 +- .../data/neomodel_inspect_database_output.txt | 24 ++++ test/test_scripts.py | 103 ++++++++++++++++++ 3 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 test/data/neomodel_inspect_database_output.txt diff --git a/neomodel/scripts/neomodel_inspect_database.py b/neomodel/scripts/neomodel_inspect_database.py index 6d28c68d..fafccf86 100644 --- a/neomodel/scripts/neomodel_inspect_database.py +++ b/neomodel/scripts/neomodel_inspect_database.py @@ -65,7 +65,7 @@ def parse_prop_class(prop_type): elif prop_type == "POINT": if "PointProperty" not in IMPORTS: IMPORTS.append("PointProperty") - return "PointProperty(" + return "PointProperty(crs='wgs-84'" class NodeInspector: diff --git a/test/data/neomodel_inspect_database_output.txt b/test/data/neomodel_inspect_database_output.txt new file mode 100644 index 00000000..10bd55f9 --- /dev/null +++ b/test/data/neomodel_inspect_database_output.txt @@ -0,0 +1,24 @@ +from neomodel import StructuredNode, StringProperty, RelationshipTo, StructuredRel, ZeroOrOne, ArrayProperty, FloatProperty, BooleanProperty, DateTimeProperty, IntegerProperty +from neomodel.contrib.spatial_properties import PointProperty + +class ScriptsTestNode(StructuredNode): + personal_id = StringProperty(unique_index=True) + name = StringProperty(index=True) + rel = RelationshipTo("ScriptsTestNode", "REL", cardinality=ZeroOrOne, model="RelRel") + + +class RelRel(StructuredRel): + personal_id = StringProperty() + name = StringProperty() + + +class EveryPropertyTypeNode(StructuredNode): + array_property = ArrayProperty(StringProperty()) + float_property = FloatProperty() + boolean_property = BooleanProperty() + point_property = PointProperty(crs='wgs-84') + string_property = StringProperty() + datetime_property = DateTimeProperty() + integer_property = IntegerProperty() + + diff --git a/test/test_scripts.py b/test/test_scripts.py index e4cd8ec6..3e8f7668 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -1,6 +1,11 @@ import subprocess from neomodel import ( + ArrayProperty, + BooleanProperty, + DateTimeProperty, + FloatProperty, + IntegerProperty, RelationshipTo, StringProperty, StructuredNode, @@ -8,6 +13,7 @@ config, db, ) +from neomodel.contrib.spatial_properties import NeomodelPoint, PointProperty class ScriptsTestRel(StructuredRel): @@ -21,6 +27,16 @@ class ScriptsTestNode(StructuredNode): rel = RelationshipTo("ScriptsTestNode", "REL", model=ScriptsTestRel) +class EveryPropertyTypeNode(StructuredNode): + string_property = StringProperty() + boolean_property = BooleanProperty() + datetime_property = DateTimeProperty() + integer_property = IntegerProperty() + float_property = FloatProperty() + point_property = PointProperty(crs="wgs-84") + array_property = ArrayProperty(StringProperty()) + + def test_neomodel_install_labels(): result = subprocess.run( ["neomodel_install_labels", "--help"], @@ -83,3 +99,90 @@ def test_neomodel_remove_labels(): indexes = db.list_indexes(exclude_token_lookup=True) assert len(constraints) == 0 assert len(indexes) == 0 + + +def test_neomodel_inspect_database(): + # Check that the help option works + result = subprocess.run( + ["neomodel_inspect_database", "--help"], + capture_output=True, + text=True, + check=False, + ) + assert "usage: neomodel_inspect_database" in result.stdout + assert result.returncode == 0 + + # Create a few nodes and a rel, with indexes and constraints + node1 = ScriptsTestNode(personal_id="1", name="test").save() + node2 = ScriptsTestNode(personal_id="2", name="test").save() + node1.rel.connect(node2, {"some_unique_property": "1", "some_index_property": "2"}) + + # Create a node with all the parsable property types + db.cypher_query( + """ + CREATE (n:EveryPropertyTypeNode { + string_property: "Hello World", + boolean_property: true, + datetime_property: datetime("2020-01-01T00:00:00.000Z"), + integer_property: 1, + float_property: 1.0, + point_property: point({x: 0.0, y: 0.0, crs: "wgs-84"}), + array_property: ["test"] + }) + """ + ) + + # Test the console output version of the script + result = subprocess.run( + ["neomodel_inspect_database", "--db", config.DATABASE_URL], + capture_output=True, + text=True, + check=False, + ) + + console_output = result.stdout + wrapped_console_output = console_output.split("\n") + assert wrapped_console_output[0].startswith("Connecting to") + # Check that all the expected lines are here + with open("test/data/neomodel_inspect_database_output.txt", "r") as f: + for line in f.read().split("\n"): + # The neomodel components import order might differ + # So let's check that every import that should be added is added, regardless of order + if line.startswith("from neomodel import"): + parsed_imports = line.replace("from neomodel import ", "").split(", ") + expected_imports = ( + wrapped_console_output[1] + .replace("from neomodel import ", "") + .split(", ") + ) + assert set(parsed_imports) == set(expected_imports) + else: + assert line in wrapped_console_output + + # Test the file output version of the script + result = subprocess.run( + [ + "neomodel_inspect_database", + "--db", + config.DATABASE_URL, + "--write-to", + "test/data/neomodel_inspect_database_test_output.py", + ], + capture_output=True, + text=True, + check=False, + ) + + # Check that the file was written + # And that the file output has the same content as the console output + # Again, regardless of order and redundance + wrapped_file_console_output = result.stdout.split("\n") + assert wrapped_file_console_output[0].startswith("Connecting to") + assert wrapped_file_console_output[1].startswith("Writing to") + with open("test/data/neomodel_inspect_database_test_output.py", "r") as f: + assert set(f.read().split("\n")) == set(wrapped_console_output[1:]) + + # Finally, delete the file created by the script + subprocess.run( + ["rm", "test/data/neomodel_inspect_database_test_output.py"], + ) From b85dd31ee00654e30b80b991497783f455e8a463 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 13:58:57 +0100 Subject: [PATCH 31/76] Add docs --- doc/source/getting_started.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/doc/source/getting_started.rst b/doc/source/getting_started.rst index 3ec09ceb..2a1a20a8 100644 --- a/doc/source/getting_started.rst +++ b/doc/source/getting_started.rst @@ -97,6 +97,28 @@ in the case of ``Relationship`` it will be possible to be queried in either dire Neomodel automatically creates a label for each ``StructuredNode`` class in the database with the corresponding indexes and constraints. +Database Inspection +=================== +You can inspect an existing Neo4j database to generate a neomodel definition file using the ``inspect`` command:: + + $ neomodel_inspect_database -db bolt://neo4j:neo4j@localhost:7687 --write-to yourapp/models.py + +This will generate a file called ``models.py`` in the ``yourapp`` directory. This file can be used as a starting point, +and will contain the necessary module imports, as well as class definition for nodes and, if relevant, relationships. + +Note that you can also print the output to the console instead of writing a file by omitting the ``--write-to`` option. + +.. note:: + + This command will only generate the definition for nodes and relationships that are present in the + database. If you want to generate a complete definition file, you will need to add the missing classes manually. + + Also, this has only been tested with single-label nodes. If you have multi-label nodes, you will need to double check, + and add the missing labels manually in the relevant way. + + Finally, relationship cardinality is guessed from the database by looking at existing relationships, so it might + guess wrong on edge cases. + Applying constraints and indexes ================================ After creating a model in Python, any constraints or indexes must be applied to Neo4j and ``neomodel`` provides a From 1aaef59d9b665eec34461f95e4fb2b13653f8a5b Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 14:44:41 +0100 Subject: [PATCH 32/76] Clean up database before inspection test --- test/test_scripts.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/test_scripts.py b/test/test_scripts.py index 3e8f7668..37725b26 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -12,6 +12,8 @@ StructuredRel, config, db, + install_labels, + util, ) from neomodel.contrib.spatial_properties import NeomodelPoint, PointProperty @@ -112,6 +114,10 @@ def test_neomodel_inspect_database(): assert "usage: neomodel_inspect_database" in result.stdout assert result.returncode == 0 + util.clear_neo4j_database(db) + install_labels(ScriptsTestNode) + install_labels(ScriptsTestRel) + # Create a few nodes and a rel, with indexes and constraints node1 = ScriptsTestNode(personal_id="1", name="test").save() node2 = ScriptsTestNode(personal_id="2", name="test").save() From c2ad515ac958fd4fd678c6b2f98a0175397398ed Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 14:49:57 +0100 Subject: [PATCH 33/76] Improve method readability --- neomodel/scripts/neomodel_inspect_database.py | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/neomodel/scripts/neomodel_inspect_database.py b/neomodel/scripts/neomodel_inspect_database.py index fafccf86..442d164e 100644 --- a/neomodel/scripts/neomodel_inspect_database.py +++ b/neomodel/scripts/neomodel_inspect_database.py @@ -37,35 +37,36 @@ def parse_prop_class(prop_type): + _import = "" + prop_class = "" if prop_type.startswith("LIST OF"): - if "ArrayProperty" not in IMPORTS: - IMPORTS.append("ArrayProperty") - return f"ArrayProperty({parse_prop_class(prop_type.replace('LIST OF ', ''))})" + _import = "ArrayProperty" + prop_class = ( + f"ArrayProperty({parse_prop_class(prop_type.replace('LIST OF ', ''))})" + ) else: if prop_type == "STRING": - if "StringProperty" not in IMPORTS: - IMPORTS.append("StringProperty") - return "StringProperty(" + _import = "StringProperty" + prop_class = f"{_import}(" elif prop_type == "BOOLEAN": - if "BooleanProperty" not in IMPORTS: - IMPORTS.append("BooleanProperty") - return "BooleanProperty(" + _import = "BooleanProperty" + prop_class = f"{_import}(" elif prop_type == "DATE_TIME": - if "DateTimeProperty" not in IMPORTS: - IMPORTS.append("DateTimeProperty") - return "DateTimeProperty(" + _import = "DateTimeProperty" + prop_class = f"{_import}(" elif prop_type == "INTEGER": - if "IntegerProperty" not in IMPORTS: - IMPORTS.append("IntegerProperty") - return "IntegerProperty(" + _import = "IntegerProperty" + prop_class = f"{_import}(" elif prop_type == "FLOAT": - if "FloatProperty" not in IMPORTS: - IMPORTS.append("FloatProperty") - return "FloatProperty(" + _import = "FloatProperty" + prop_class = f"{_import}(" elif prop_type == "POINT": - if "PointProperty" not in IMPORTS: - IMPORTS.append("PointProperty") - return "PointProperty(crs='wgs-84'" + _import = "PointProperty" + prop_class = f"{_import}(crs='wgs-84'" + + if _import not in IMPORTS: + IMPORTS.append(_import) + return prop_class class NodeInspector: From 9bedf78660be7f329a619ba64b72cbd108f5244d Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 14:59:29 +0100 Subject: [PATCH 34/76] Add a temp print to debug remote test --- test/test_scripts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_scripts.py b/test/test_scripts.py index 37725b26..8be4402d 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -154,6 +154,7 @@ def test_neomodel_inspect_database(): for line in f.read().split("\n"): # The neomodel components import order might differ # So let's check that every import that should be added is added, regardless of order + print(line) if line.startswith("from neomodel import"): parsed_imports = line.replace("from neomodel import ", "").split(", ") expected_imports = ( From 29dbd047cde9fdd087a40d0bfec6a2f4893d38e2 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 15:13:42 +0100 Subject: [PATCH 35/76] Try a different set of prints --- test/test_scripts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_scripts.py b/test/test_scripts.py index 8be4402d..44ffa197 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -154,8 +154,9 @@ def test_neomodel_inspect_database(): for line in f.read().split("\n"): # The neomodel components import order might differ # So let's check that every import that should be added is added, regardless of order - print(line) if line.startswith("from neomodel import"): + print(line) + print(wrapped_console_output[1]) parsed_imports = line.replace("from neomodel import ", "").split(", ") expected_imports = ( wrapped_console_output[1] From 740cbdf794193b99a328c7d633c3b50cb58631e9 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 15:19:45 +0100 Subject: [PATCH 36/76] Still debugging --- test/test_scripts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_scripts.py b/test/test_scripts.py index 44ffa197..d91ec3c1 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -156,7 +156,7 @@ def test_neomodel_inspect_database(): # So let's check that every import that should be added is added, regardless of order if line.startswith("from neomodel import"): print(line) - print(wrapped_console_output[1]) + print(console_output) parsed_imports = line.replace("from neomodel import ", "").split(", ") expected_imports = ( wrapped_console_output[1] From 02144d080e86162e8d9b20f03c62dc8d94ceb119 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 15:32:24 +0100 Subject: [PATCH 37/76] Other prints --- test/test_scripts.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/test_scripts.py b/test/test_scripts.py index d91ec3c1..1142ad6d 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -147,6 +147,12 @@ def test_neomodel_inspect_database(): ) console_output = result.stdout + print(config.DATABASE_URL) + print(db.url) + print(console_output) + + results, _ = db.cypher_query("MATCH (n) RETURN count(n)") + print(results[0][0]) wrapped_console_output = console_output.split("\n") assert wrapped_console_output[0].startswith("Connecting to") # Check that all the expected lines are here @@ -155,8 +161,6 @@ def test_neomodel_inspect_database(): # The neomodel components import order might differ # So let's check that every import that should be added is added, regardless of order if line.startswith("from neomodel import"): - print(line) - print(console_output) parsed_imports = line.replace("from neomodel import ", "").split(", ") expected_imports = ( wrapped_console_output[1] From d9564e79538fb954eff474867457ac7f84de6815 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 15:38:36 +0100 Subject: [PATCH 38/76] Yet another print --- test/test_scripts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_scripts.py b/test/test_scripts.py index 1142ad6d..f76ce8b4 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -151,8 +151,8 @@ def test_neomodel_inspect_database(): print(db.url) print(console_output) - results, _ = db.cypher_query("MATCH (n) RETURN count(n)") - print(results[0][0]) + results, _ = db.cypher_query("CALL db.labels()") + print(",".join(results[0])) wrapped_console_output = console_output.split("\n") assert wrapped_console_output[0].startswith("Connecting to") # Check that all the expected lines are here From c240abac6ada0fd4f414bb6f4ca512a143ea506a Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 15:42:37 +0100 Subject: [PATCH 39/76] YAP --- test/test_scripts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_scripts.py b/test/test_scripts.py index f76ce8b4..81b27af3 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -151,8 +151,8 @@ def test_neomodel_inspect_database(): print(db.url) print(console_output) - results, _ = db.cypher_query("CALL db.labels()") - print(",".join(results[0])) + results, _ = db.cypher_query("MATCH (n) RETURN n") + print(",".join([result[0] for result in results])) wrapped_console_output = console_output.split("\n") assert wrapped_console_output[0].startswith("Connecting to") # Check that all the expected lines are here From 24acaaf0496a7f2577836c9b312601a7b7b7a785 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 15:46:58 +0100 Subject: [PATCH 40/76] Other label try --- test/test_scripts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_scripts.py b/test/test_scripts.py index 81b27af3..96c3b801 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -151,7 +151,7 @@ def test_neomodel_inspect_database(): print(db.url) print(console_output) - results, _ = db.cypher_query("MATCH (n) RETURN n") + results, _ = db.cypher_query("CALL db.labels()") print(",".join([result[0] for result in results])) wrapped_console_output = console_output.split("\n") assert wrapped_console_output[0].startswith("Connecting to") From e959ab6e24b7025b6600e0976560d4468f9d400f Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 16:02:22 +0100 Subject: [PATCH 41/76] Remove prints --- test/test_scripts.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/test_scripts.py b/test/test_scripts.py index 96c3b801..37725b26 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -147,12 +147,6 @@ def test_neomodel_inspect_database(): ) console_output = result.stdout - print(config.DATABASE_URL) - print(db.url) - print(console_output) - - results, _ = db.cypher_query("CALL db.labels()") - print(",".join([result[0] for result in results])) wrapped_console_output = console_output.split("\n") assert wrapped_console_output[0].startswith("Connecting to") # Check that all the expected lines are here From 1ed65475a298af1d767a56605f9bc5d08bc6ec32 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 16:07:24 +0100 Subject: [PATCH 42/76] Skip console output test for debugging --- test/test_scripts.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/test/test_scripts.py b/test/test_scripts.py index 37725b26..fd8b2e3e 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -150,20 +150,20 @@ def test_neomodel_inspect_database(): wrapped_console_output = console_output.split("\n") assert wrapped_console_output[0].startswith("Connecting to") # Check that all the expected lines are here - with open("test/data/neomodel_inspect_database_output.txt", "r") as f: - for line in f.read().split("\n"): - # The neomodel components import order might differ - # So let's check that every import that should be added is added, regardless of order - if line.startswith("from neomodel import"): - parsed_imports = line.replace("from neomodel import ", "").split(", ") - expected_imports = ( - wrapped_console_output[1] - .replace("from neomodel import ", "") - .split(", ") - ) - assert set(parsed_imports) == set(expected_imports) - else: - assert line in wrapped_console_output + # with open("test/data/neomodel_inspect_database_output.txt", "r") as f: + # for line in f.read().split("\n"): + # # The neomodel components import order might differ + # # So let's check that every import that should be added is added, regardless of order + # if line.startswith("from neomodel import"): + # parsed_imports = line.replace("from neomodel import ", "").split(", ") + # expected_imports = ( + # wrapped_console_output[1] + # .replace("from neomodel import ", "") + # .split(", ") + # ) + # assert set(parsed_imports) == set(expected_imports) + # else: + # assert line in wrapped_console_output # Test the file output version of the script result = subprocess.run( @@ -186,7 +186,8 @@ def test_neomodel_inspect_database(): assert wrapped_file_console_output[0].startswith("Connecting to") assert wrapped_file_console_output[1].startswith("Writing to") with open("test/data/neomodel_inspect_database_test_output.py", "r") as f: - assert set(f.read().split("\n")) == set(wrapped_console_output[1:]) + # assert set(f.read().split("\n")) == set(wrapped_console_output[1:]) + print(f.read()) # Finally, delete the file created by the script subprocess.run( From 0c025ff45eb65ce5aa12a67f9c6c1fc90e8d94b9 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 16:10:33 +0100 Subject: [PATCH 43/76] Add one print --- test/test_scripts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_scripts.py b/test/test_scripts.py index fd8b2e3e..e1ee085a 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -182,6 +182,7 @@ def test_neomodel_inspect_database(): # Check that the file was written # And that the file output has the same content as the console output # Again, regardless of order and redundance + print(result.stdout) wrapped_file_console_output = result.stdout.split("\n") assert wrapped_file_console_output[0].startswith("Connecting to") assert wrapped_file_console_output[1].startswith("Writing to") From fadeaec65e7b32fad1f079ebb504e45156ce57a5 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 16:20:58 +0100 Subject: [PATCH 44/76] Ignore empty lines in console output and test/generated file --- test/test_scripts.py | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/test/test_scripts.py b/test/test_scripts.py index e1ee085a..15a6c413 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -146,24 +146,26 @@ def test_neomodel_inspect_database(): check=False, ) - console_output = result.stdout - wrapped_console_output = console_output.split("\n") + wrapped_console_output = [ + line for line in result.stdout.splitlines() if line.strip() + ] assert wrapped_console_output[0].startswith("Connecting to") # Check that all the expected lines are here - # with open("test/data/neomodel_inspect_database_output.txt", "r") as f: - # for line in f.read().split("\n"): - # # The neomodel components import order might differ - # # So let's check that every import that should be added is added, regardless of order - # if line.startswith("from neomodel import"): - # parsed_imports = line.replace("from neomodel import ", "").split(", ") - # expected_imports = ( - # wrapped_console_output[1] - # .replace("from neomodel import ", "") - # .split(", ") - # ) - # assert set(parsed_imports) == set(expected_imports) - # else: - # assert line in wrapped_console_output + with open("test/data/neomodel_inspect_database_output.txt", "r") as f: + wrapped_test_file = [line for line in f.read().split("\n") if line.strip()] + for line in wrapped_test_file: + # The neomodel components import order might differ + # So let's check that every import that should be added is added, regardless of order + if line.startswith("from neomodel import"): + parsed_imports = line.replace("from neomodel import ", "").split(", ") + expected_imports = ( + wrapped_console_output[1] + .replace("from neomodel import ", "") + .split(", ") + ) + assert set(parsed_imports) == set(expected_imports) + else: + assert line in wrapped_console_output # Test the file output version of the script result = subprocess.run( @@ -182,13 +184,14 @@ def test_neomodel_inspect_database(): # Check that the file was written # And that the file output has the same content as the console output # Again, regardless of order and redundance - print(result.stdout) - wrapped_file_console_output = result.stdout.split("\n") + wrapped_file_console_output = [ + line for line in result.stdout.splitlines() if line.strip() + ] assert wrapped_file_console_output[0].startswith("Connecting to") assert wrapped_file_console_output[1].startswith("Writing to") with open("test/data/neomodel_inspect_database_test_output.py", "r") as f: - # assert set(f.read().split("\n")) == set(wrapped_console_output[1:]) - print(f.read()) + wrapped_output_file = [line for line in f.read().split("\n") if line.strip()] + assert set(wrapped_output_file) == set(wrapped_console_output[1:]) # Finally, delete the file created by the script subprocess.run( From 1f1aac567156c274df2cc5273a0736296214e30b Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 16:26:10 +0100 Subject: [PATCH 45/76] Add a print for debugging --- test/test_scripts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_scripts.py b/test/test_scripts.py index 15a6c413..f216fcb7 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -146,6 +146,7 @@ def test_neomodel_inspect_database(): check=False, ) + print(result.stdout) wrapped_console_output = [ line for line in result.stdout.splitlines() if line.strip() ] From 50a829d09d421b5774d447ac1296f1aa7fe8cbec Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 16:29:39 +0100 Subject: [PATCH 46/76] Add some prints in the script itself --- neomodel/scripts/neomodel_inspect_database.py | 1 + test/test_scripts.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/neomodel/scripts/neomodel_inspect_database.py b/neomodel/scripts/neomodel_inspect_database.py index 442d164e..ea82a68c 100644 --- a/neomodel/scripts/neomodel_inspect_database.py +++ b/neomodel/scripts/neomodel_inspect_database.py @@ -220,6 +220,7 @@ def inspect_database(bolt_url): for label in node_labels: class_name = clean_class_member_key(label) + print(class_name) properties = NodeInspector.get_properties_for_label(label) unique_properties = NodeInspector.get_constraints_for_label(label) indexed_properties = NodeInspector.get_indexed_properties_for_label(label) diff --git a/test/test_scripts.py b/test/test_scripts.py index f216fcb7..15a6c413 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -146,7 +146,6 @@ def test_neomodel_inspect_database(): check=False, ) - print(result.stdout) wrapped_console_output = [ line for line in result.stdout.splitlines() if line.strip() ] From a0b02aa5935bd4f72ac4162908bc79fc1a5311b6 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 16:37:39 +0100 Subject: [PATCH 47/76] Compare prints --- test/test_scripts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_scripts.py b/test/test_scripts.py index 15a6c413..f216fcb7 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -146,6 +146,7 @@ def test_neomodel_inspect_database(): check=False, ) + print(result.stdout) wrapped_console_output = [ line for line in result.stdout.splitlines() if line.strip() ] From 129a5bc269ce4c97acc6e6a7a9340bdf67cf02e5 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 16:42:07 +0100 Subject: [PATCH 48/76] Skip everything except dumping to file and print file content to stdout --- neomodel/scripts/neomodel_inspect_database.py | 1 - test/test_scripts.py | 68 +++++++++---------- 2 files changed, 34 insertions(+), 35 deletions(-) diff --git a/neomodel/scripts/neomodel_inspect_database.py b/neomodel/scripts/neomodel_inspect_database.py index ea82a68c..442d164e 100644 --- a/neomodel/scripts/neomodel_inspect_database.py +++ b/neomodel/scripts/neomodel_inspect_database.py @@ -220,7 +220,6 @@ def inspect_database(bolt_url): for label in node_labels: class_name = clean_class_member_key(label) - print(class_name) properties = NodeInspector.get_properties_for_label(label) unique_properties = NodeInspector.get_constraints_for_label(label) indexed_properties = NodeInspector.get_indexed_properties_for_label(label) diff --git a/test/test_scripts.py b/test/test_scripts.py index f216fcb7..d9c56dd1 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -139,34 +139,33 @@ def test_neomodel_inspect_database(): ) # Test the console output version of the script - result = subprocess.run( - ["neomodel_inspect_database", "--db", config.DATABASE_URL], - capture_output=True, - text=True, - check=False, - ) - - print(result.stdout) - wrapped_console_output = [ - line for line in result.stdout.splitlines() if line.strip() - ] - assert wrapped_console_output[0].startswith("Connecting to") - # Check that all the expected lines are here - with open("test/data/neomodel_inspect_database_output.txt", "r") as f: - wrapped_test_file = [line for line in f.read().split("\n") if line.strip()] - for line in wrapped_test_file: - # The neomodel components import order might differ - # So let's check that every import that should be added is added, regardless of order - if line.startswith("from neomodel import"): - parsed_imports = line.replace("from neomodel import ", "").split(", ") - expected_imports = ( - wrapped_console_output[1] - .replace("from neomodel import ", "") - .split(", ") - ) - assert set(parsed_imports) == set(expected_imports) - else: - assert line in wrapped_console_output + # result = subprocess.run( + # ["neomodel_inspect_database", "--db", config.DATABASE_URL], + # capture_output=True, + # text=True, + # check=False, + # ) + + # wrapped_console_output = [ + # line for line in result.stdout.splitlines() if line.strip() + # ] + # assert wrapped_console_output[0].startswith("Connecting to") + # # Check that all the expected lines are here + # with open("test/data/neomodel_inspect_database_output.txt", "r") as f: + # wrapped_test_file = [line for line in f.read().split("\n") if line.strip()] + # for line in wrapped_test_file: + # # The neomodel components import order might differ + # # So let's check that every import that should be added is added, regardless of order + # if line.startswith("from neomodel import"): + # parsed_imports = line.replace("from neomodel import ", "").split(", ") + # expected_imports = ( + # wrapped_console_output[1] + # .replace("from neomodel import ", "") + # .split(", ") + # ) + # assert set(parsed_imports) == set(expected_imports) + # else: + # assert line in wrapped_console_output # Test the file output version of the script result = subprocess.run( @@ -185,14 +184,15 @@ def test_neomodel_inspect_database(): # Check that the file was written # And that the file output has the same content as the console output # Again, regardless of order and redundance - wrapped_file_console_output = [ - line for line in result.stdout.splitlines() if line.strip() - ] - assert wrapped_file_console_output[0].startswith("Connecting to") - assert wrapped_file_console_output[1].startswith("Writing to") + # wrapped_file_console_output = [ + # line for line in result.stdout.splitlines() if line.strip() + # ] + # assert wrapped_file_console_output[0].startswith("Connecting to") + # assert wrapped_file_console_output[1].startswith("Writing to") with open("test/data/neomodel_inspect_database_test_output.py", "r") as f: wrapped_output_file = [line for line in f.read().split("\n") if line.strip()] - assert set(wrapped_output_file) == set(wrapped_console_output[1:]) + print(wrapped_output_file) + # assert set(wrapped_output_file) == set(wrapped_console_output[1:]) # Finally, delete the file created by the script subprocess.run( From 62a43bb2bc56bfdd3e9524b83e75a66eebad87e3 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 16:44:46 +0100 Subject: [PATCH 49/76] Force test failure to get output in GHA --- test/test_scripts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_scripts.py b/test/test_scripts.py index d9c56dd1..a8e48e66 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -192,6 +192,7 @@ def test_neomodel_inspect_database(): with open("test/data/neomodel_inspect_database_test_output.py", "r") as f: wrapped_output_file = [line for line in f.read().split("\n") if line.strip()] print(wrapped_output_file) + assert False # assert set(wrapped_output_file) == set(wrapped_console_output[1:]) # Finally, delete the file created by the script From 9068fed7ba854c8bb34a641737937e086e05f7f1 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 17:36:49 +0100 Subject: [PATCH 50/76] Try with single node --- test/test_scripts.py | 100 +++++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 51 deletions(-) diff --git a/test/test_scripts.py b/test/test_scripts.py index a8e48e66..4d1078bf 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -120,52 +120,52 @@ def test_neomodel_inspect_database(): # Create a few nodes and a rel, with indexes and constraints node1 = ScriptsTestNode(personal_id="1", name="test").save() - node2 = ScriptsTestNode(personal_id="2", name="test").save() - node1.rel.connect(node2, {"some_unique_property": "1", "some_index_property": "2"}) - - # Create a node with all the parsable property types - db.cypher_query( - """ - CREATE (n:EveryPropertyTypeNode { - string_property: "Hello World", - boolean_property: true, - datetime_property: datetime("2020-01-01T00:00:00.000Z"), - integer_property: 1, - float_property: 1.0, - point_property: point({x: 0.0, y: 0.0, crs: "wgs-84"}), - array_property: ["test"] - }) - """ - ) + # node2 = ScriptsTestNode(personal_id="2", name="test").save() + # node1.rel.connect(node2, {"some_unique_property": "1", "some_index_property": "2"}) + + # # Create a node with all the parsable property types + # db.cypher_query( + # """ + # CREATE (n:EveryPropertyTypeNode { + # string_property: "Hello World", + # boolean_property: true, + # datetime_property: datetime("2020-01-01T00:00:00.000Z"), + # integer_property: 1, + # float_property: 1.0, + # point_property: point({x: 0.0, y: 0.0, crs: "wgs-84"}), + # array_property: ["test"] + # }) + # """ + # ) # Test the console output version of the script - # result = subprocess.run( - # ["neomodel_inspect_database", "--db", config.DATABASE_URL], - # capture_output=True, - # text=True, - # check=False, - # ) + result = subprocess.run( + ["neomodel_inspect_database", "--db", config.DATABASE_URL], + capture_output=True, + text=True, + check=False, + ) - # wrapped_console_output = [ - # line for line in result.stdout.splitlines() if line.strip() - # ] - # assert wrapped_console_output[0].startswith("Connecting to") - # # Check that all the expected lines are here - # with open("test/data/neomodel_inspect_database_output.txt", "r") as f: - # wrapped_test_file = [line for line in f.read().split("\n") if line.strip()] - # for line in wrapped_test_file: - # # The neomodel components import order might differ - # # So let's check that every import that should be added is added, regardless of order - # if line.startswith("from neomodel import"): - # parsed_imports = line.replace("from neomodel import ", "").split(", ") - # expected_imports = ( - # wrapped_console_output[1] - # .replace("from neomodel import ", "") - # .split(", ") - # ) - # assert set(parsed_imports) == set(expected_imports) - # else: - # assert line in wrapped_console_output + wrapped_console_output = [ + line for line in result.stdout.splitlines() if line.strip() + ] + assert wrapped_console_output[0].startswith("Connecting to") + # Check that all the expected lines are here + with open("test/data/neomodel_inspect_database_output.txt", "r") as f: + wrapped_test_file = [line for line in f.read().split("\n") if line.strip()] + for line in wrapped_test_file: + # The neomodel components import order might differ + # So let's check that every import that should be added is added, regardless of order + if line.startswith("from neomodel import"): + parsed_imports = line.replace("from neomodel import ", "").split(", ") + expected_imports = ( + wrapped_console_output[1] + .replace("from neomodel import ", "") + .split(", ") + ) + assert set(parsed_imports) == set(expected_imports) + else: + assert line in wrapped_console_output # Test the file output version of the script result = subprocess.run( @@ -184,16 +184,14 @@ def test_neomodel_inspect_database(): # Check that the file was written # And that the file output has the same content as the console output # Again, regardless of order and redundance - # wrapped_file_console_output = [ - # line for line in result.stdout.splitlines() if line.strip() - # ] - # assert wrapped_file_console_output[0].startswith("Connecting to") - # assert wrapped_file_console_output[1].startswith("Writing to") + wrapped_file_console_output = [ + line for line in result.stdout.splitlines() if line.strip() + ] + assert wrapped_file_console_output[0].startswith("Connecting to") + assert wrapped_file_console_output[1].startswith("Writing to") with open("test/data/neomodel_inspect_database_test_output.py", "r") as f: wrapped_output_file = [line for line in f.read().split("\n") if line.strip()] - print(wrapped_output_file) - assert False - # assert set(wrapped_output_file) == set(wrapped_console_output[1:]) + assert set(wrapped_output_file) == set(wrapped_console_output[1:]) # Finally, delete the file created by the script subprocess.run( From 909b5d200504fdf16004ce7fb28bcedccfb8802d Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Mon, 30 Oct 2023 17:40:14 +0100 Subject: [PATCH 51/76] Uncomment tests --- test/test_scripts.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/test/test_scripts.py b/test/test_scripts.py index 4d1078bf..15a6c413 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -120,23 +120,23 @@ def test_neomodel_inspect_database(): # Create a few nodes and a rel, with indexes and constraints node1 = ScriptsTestNode(personal_id="1", name="test").save() - # node2 = ScriptsTestNode(personal_id="2", name="test").save() - # node1.rel.connect(node2, {"some_unique_property": "1", "some_index_property": "2"}) - - # # Create a node with all the parsable property types - # db.cypher_query( - # """ - # CREATE (n:EveryPropertyTypeNode { - # string_property: "Hello World", - # boolean_property: true, - # datetime_property: datetime("2020-01-01T00:00:00.000Z"), - # integer_property: 1, - # float_property: 1.0, - # point_property: point({x: 0.0, y: 0.0, crs: "wgs-84"}), - # array_property: ["test"] - # }) - # """ - # ) + node2 = ScriptsTestNode(personal_id="2", name="test").save() + node1.rel.connect(node2, {"some_unique_property": "1", "some_index_property": "2"}) + + # Create a node with all the parsable property types + db.cypher_query( + """ + CREATE (n:EveryPropertyTypeNode { + string_property: "Hello World", + boolean_property: true, + datetime_property: datetime("2020-01-01T00:00:00.000Z"), + integer_property: 1, + float_property: 1.0, + point_property: point({x: 0.0, y: 0.0, crs: "wgs-84"}), + array_property: ["test"] + }) + """ + ) # Test the console output version of the script result = subprocess.run( From 309eca8963d28632055fdb95539d16e9e6e2ecf3 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 31 Oct 2023 11:13:19 +0100 Subject: [PATCH 52/76] Pass check True for subprocess --- test/test_scripts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_scripts.py b/test/test_scripts.py index 15a6c413..baf24155 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -143,7 +143,7 @@ def test_neomodel_inspect_database(): ["neomodel_inspect_database", "--db", config.DATABASE_URL], capture_output=True, text=True, - check=False, + check=True, ) wrapped_console_output = [ @@ -178,7 +178,7 @@ def test_neomodel_inspect_database(): ], capture_output=True, text=True, - check=False, + check=True, ) # Check that the file was written From 1f3cdc89d4e3ff2059b10507cebb04b83dbe1e46 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 31 Oct 2023 11:19:05 +0100 Subject: [PATCH 53/76] Add APOC to GHA environment --- docker-scripts/docker-neo4j.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-scripts/docker-neo4j.sh b/docker-scripts/docker-neo4j.sh index 3b294256..99aabfff 100644 --- a/docker-scripts/docker-neo4j.sh +++ b/docker-scripts/docker-neo4j.sh @@ -4,4 +4,5 @@ docker run \ -d \ --env NEO4J_AUTH=neo4j/foobarbaz \ --env NEO4J_ACCEPT_LICENSE_AGREEMENT=yes \ + --env NEO4JLABS_PLUGINS='["apoc"]' \ neo4j:$1 \ No newline at end of file From 01049b8c85b54608e0b26a68f75e8d0fe9090f0c Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 31 Oct 2023 12:23:54 +0100 Subject: [PATCH 54/76] Fix inspection script for 4.4 --- neomodel/scripts/neomodel_inspect_database.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/neomodel/scripts/neomodel_inspect_database.py b/neomodel/scripts/neomodel_inspect_database.py index 442d164e..08c9515b 100644 --- a/neomodel/scripts/neomodel_inspect_database.py +++ b/neomodel/scripts/neomodel_inspect_database.py @@ -98,9 +98,14 @@ def get_constraints_for_label(label): @staticmethod def get_indexed_properties_for_label(label): - indexes, meta_indexes = db.cypher_query( - f"SHOW INDEXES WHERE entityType='NODE' AND '{label}' IN labelsOrTypes AND type='RANGE' AND owningConstraint IS NULL" - ) + if db.version_is_higher_than("5.0"): + indexes, meta_indexes = db.cypher_query( + f"SHOW INDEXES WHERE entityType='NODE' AND '{label}' IN labelsOrTypes AND type='RANGE' AND owningConstraint IS NULL" + ) + else: + indexes, meta_indexes = db.cypher_query( + f"SHOW INDEXES WHERE entityType='NODE' AND '{label}' IN labelsOrTypes AND type='BTREE' AND uniqueness='NONUNIQUE'" + ) indexes_as_dict = [dict(zip(meta_indexes, row)) for row in indexes] indexed_properties = [ item.get("properties")[0] From e5091c386210aa2c493af724b29bb6651d8fbd63 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 31 Oct 2023 12:28:05 +0100 Subject: [PATCH 55/76] Skip relationship type constraints for < 5.7 --- neomodel/scripts/neomodel_inspect_database.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/neomodel/scripts/neomodel_inspect_database.py b/neomodel/scripts/neomodel_inspect_database.py index 08c9515b..c5e632d1 100644 --- a/neomodel/scripts/neomodel_inspect_database.py +++ b/neomodel/scripts/neomodel_inspect_database.py @@ -252,7 +252,11 @@ def inspect_database(bolt_url): target_label = rel[1] rel_props = rel[2] - unique_properties = RelationshipInspector.get_constraints_for_type(rel_type) + unique_properties = ( + RelationshipInspector.get_constraints_for_type(rel_type) + if db.version_is_higher_than("5.7") + else [] + ) indexed_properties = RelationshipInspector.get_indexed_properties_for_type( rel_type ) From bcd84af81b54ce7205ddcfae803a7415f9b7e329 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 31 Oct 2023 13:59:13 +0100 Subject: [PATCH 56/76] Try installating apoc 4.4 for Neo4J 4.4 --- docker-scripts/docker-neo4j.sh | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/docker-scripts/docker-neo4j.sh b/docker-scripts/docker-neo4j.sh index 99aabfff..3b42800d 100644 --- a/docker-scripts/docker-neo4j.sh +++ b/docker-scripts/docker-neo4j.sh @@ -1,8 +1,19 @@ -docker run \ - --name neo4j \ - -p7474:7474 -p7687:7687 \ - -d \ - --env NEO4J_AUTH=neo4j/foobarbaz \ - --env NEO4J_ACCEPT_LICENSE_AGREEMENT=yes \ - --env NEO4JLABS_PLUGINS='["apoc"]' \ - neo4j:$1 \ No newline at end of file +if [[ "$1" == 4.4* ]]; then + docker run \ + --name neo4j \ + -p7474:7474 -p7687:7687 \ + -d \ + --env NEO4J_AUTH=neo4j/foobarbaz \ + --env NEO4J_ACCEPT_LICENSE_AGREEMENT=yes \ + --env NEO4JLABS_PLUGINS='["apoc-4.4.0.23-all"]' \ + neo4j:$1 +else + docker run \ + --name neo4j \ + -p7474:7474 -p7687:7687 \ + -d \ + --env NEO4J_AUTH=neo4j/foobarbaz \ + --env NEO4J_ACCEPT_LICENSE_AGREEMENT=yes \ + --env NEO4JLABS_PLUGINS='["apoc"]' \ neo4j:$1 + +fi From 14ccb9cc2a181c1989b5096238d6592f9b03bd05 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 31 Oct 2023 14:04:08 +0100 Subject: [PATCH 57/76] Fix docker script --- docker-scripts/docker-neo4j.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docker-scripts/docker-neo4j.sh b/docker-scripts/docker-neo4j.sh index 3b42800d..ae0bed96 100644 --- a/docker-scripts/docker-neo4j.sh +++ b/docker-scripts/docker-neo4j.sh @@ -1,11 +1,11 @@ -if [[ "$1" == 4.4* ]]; then +if [ $1 = "4.4*" ]; then docker run \ --name neo4j \ -p7474:7474 -p7687:7687 \ -d \ --env NEO4J_AUTH=neo4j/foobarbaz \ --env NEO4J_ACCEPT_LICENSE_AGREEMENT=yes \ - --env NEO4JLABS_PLUGINS='["apoc-4.4.0.23-all"]' \ + --env NEO4JLABS_PLUGINS=["apoc-4.4.0.23-all"] \ neo4j:$1 else docker run \ @@ -14,6 +14,7 @@ else -d \ --env NEO4J_AUTH=neo4j/foobarbaz \ --env NEO4J_ACCEPT_LICENSE_AGREEMENT=yes \ - --env NEO4JLABS_PLUGINS='["apoc"]' \ neo4j:$1 + --env NEO4JLABS_PLUGINS=["apoc"] \ + neo4j:$1 fi From 764af2443618dc6c96a702439b7efe51413cd9e2 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 31 Oct 2023 14:08:45 +0100 Subject: [PATCH 58/76] Reset LABS PLUGINS as string --- docker-scripts/docker-neo4j.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-scripts/docker-neo4j.sh b/docker-scripts/docker-neo4j.sh index ae0bed96..db4eddc1 100644 --- a/docker-scripts/docker-neo4j.sh +++ b/docker-scripts/docker-neo4j.sh @@ -5,7 +5,7 @@ if [ $1 = "4.4*" ]; then -d \ --env NEO4J_AUTH=neo4j/foobarbaz \ --env NEO4J_ACCEPT_LICENSE_AGREEMENT=yes \ - --env NEO4JLABS_PLUGINS=["apoc-4.4.0.23-all"] \ + --env NEO4JLABS_PLUGINS='["apoc-4.4.0.23-all"]' \ neo4j:$1 else docker run \ @@ -14,7 +14,7 @@ else -d \ --env NEO4J_AUTH=neo4j/foobarbaz \ --env NEO4J_ACCEPT_LICENSE_AGREEMENT=yes \ - --env NEO4JLABS_PLUGINS=["apoc"] \ + --env NEO4JLABS_PLUGINS='["apoc"]' \ neo4j:$1 fi From 2dbf73817b15461e0442bde3e717f96bccdd3b72 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 31 Oct 2023 14:22:28 +0100 Subject: [PATCH 59/76] Revert docker script to unversioned apoc --- docker-scripts/docker-neo4j.sh | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/docker-scripts/docker-neo4j.sh b/docker-scripts/docker-neo4j.sh index db4eddc1..99aabfff 100644 --- a/docker-scripts/docker-neo4j.sh +++ b/docker-scripts/docker-neo4j.sh @@ -1,20 +1,8 @@ -if [ $1 = "4.4*" ]; then - docker run \ - --name neo4j \ - -p7474:7474 -p7687:7687 \ - -d \ - --env NEO4J_AUTH=neo4j/foobarbaz \ - --env NEO4J_ACCEPT_LICENSE_AGREEMENT=yes \ - --env NEO4JLABS_PLUGINS='["apoc-4.4.0.23-all"]' \ - neo4j:$1 -else - docker run \ - --name neo4j \ - -p7474:7474 -p7687:7687 \ - -d \ - --env NEO4J_AUTH=neo4j/foobarbaz \ - --env NEO4J_ACCEPT_LICENSE_AGREEMENT=yes \ - --env NEO4JLABS_PLUGINS='["apoc"]' \ - neo4j:$1 - -fi +docker run \ + --name neo4j \ + -p7474:7474 -p7687:7687 \ + -d \ + --env NEO4J_AUTH=neo4j/foobarbaz \ + --env NEO4J_ACCEPT_LICENSE_AGREEMENT=yes \ + --env NEO4JLABS_PLUGINS='["apoc"]' \ + neo4j:$1 \ No newline at end of file From d2e5d4bba069823be2617570e227678e439b8738 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 31 Oct 2023 14:24:03 +0100 Subject: [PATCH 60/76] Fix issue with imports --- neomodel/scripts/neomodel_inspect_database.py | 1 + 1 file changed, 1 insertion(+) diff --git a/neomodel/scripts/neomodel_inspect_database.py b/neomodel/scripts/neomodel_inspect_database.py index c5e632d1..a9cb4fff 100644 --- a/neomodel/scripts/neomodel_inspect_database.py +++ b/neomodel/scripts/neomodel_inspect_database.py @@ -294,6 +294,7 @@ def inspect_database(bolt_url): class_definitions += class_definition # Finally, parse imports + imports = "" if IMPORTS: special_imports = "" if "PointProperty" in IMPORTS: From 4422aefa172190c84255c3ca0deb44aeaefcaf55 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 31 Oct 2023 14:27:09 +0100 Subject: [PATCH 61/76] Fix relationship indexes --- neomodel/scripts/neomodel_inspect_database.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/neomodel/scripts/neomodel_inspect_database.py b/neomodel/scripts/neomodel_inspect_database.py index a9cb4fff..75b0a696 100644 --- a/neomodel/scripts/neomodel_inspect_database.py +++ b/neomodel/scripts/neomodel_inspect_database.py @@ -142,9 +142,14 @@ def get_constraints_for_type(rel_type): @staticmethod def get_indexed_properties_for_type(rel_type): - indexes, meta_indexes = db.cypher_query( - f"SHOW INDEXES WHERE entityType='RELATIONSHIP' AND '{rel_type}' IN labelsOrTypes AND type='RANGE' AND owningConstraint IS NULL" - ) + if db.version_is_higher_than("5.0"): + indexes, meta_indexes = db.cypher_query( + f"SHOW INDEXES WHERE entityType='RELATIONSHIP' AND '{rel_type}' IN labelsOrTypes AND type='RANGE' AND owningConstraint IS NULL" + ) + else: + indexes, meta_indexes = db.cypher_query( + f"SHOW INDEXES WHERE entityType='RELATIONSHIP' AND '{rel_type}' IN labelsOrTypes AND type='BTREE' AND uniqueness='NONUNIQUE'" + ) indexes_as_dict = [dict(zip(meta_indexes, row)) for row in indexes] indexed_properties = [ item.get("properties")[0] From 450315b08b6ca95f7a56460c76750511a7dfa0a0 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 31 Oct 2023 14:37:31 +0100 Subject: [PATCH 62/76] Update APOC and version tag --- doc/source/getting_started.rst | 9 +++++++-- neomodel/_version.py | 2 +- pyproject.toml | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/doc/source/getting_started.rst b/doc/source/getting_started.rst index 7a6b8cfa..d49630fb 100644 --- a/doc/source/getting_started.rst +++ b/doc/source/getting_started.rst @@ -105,8 +105,8 @@ in the case of ``Relationship`` it will be possible to be queried in either dire Neomodel automatically creates a label for each ``StructuredNode`` class in the database with the corresponding indexes and constraints. -Database Inspection -=================== +Database Inspection - Requires APOC +=================================== You can inspect an existing Neo4j database to generate a neomodel definition file using the ``inspect`` command:: $ neomodel_inspect_database -db bolt://neo4j:neo4j@localhost:7687 --write-to yourapp/models.py @@ -127,6 +127,11 @@ Note that you can also print the output to the console instead of writing a file Finally, relationship cardinality is guessed from the database by looking at existing relationships, so it might guess wrong on edge cases. +.. warning:: + + The script relies on the method apoc.meta.cypher.types to parse property types. So APOC must be installed on your Neo4j server + for this script to work. + Applying constraints and indexes ================================ After creating a model in Python, any constraints or indexes must be applied to Neo4j and ``neomodel`` provides a diff --git a/neomodel/_version.py b/neomodel/_version.py index b2e51c32..36887081 100644 --- a/neomodel/_version.py +++ b/neomodel/_version.py @@ -1 +1 @@ -__version__ = "5.1.2" +__version__ = "5.1.3" diff --git a/pyproject.toml b/pyproject.toml index ed4b1554..35a78a5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ dependencies = [ "neobolt==1.7.17", "six==1.16.0", ] -version='5.1.2' +version='5.1.3' [project.urls] documentation = "https://neomodel.readthedocs.io/en/latest/" From ca5498e6358d264e1af922c5954d2d09a319b883 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 31 Oct 2023 15:00:48 +0100 Subject: [PATCH 63/76] Fix unsaved node checks --- neomodel/cardinality.py | 2 +- neomodel/match.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/neomodel/cardinality.py b/neomodel/cardinality.py index 2e2fdf0d..099bf578 100644 --- a/neomodel/cardinality.py +++ b/neomodel/cardinality.py @@ -128,7 +128,7 @@ def connect(self, node, properties=None): :param properties: relationship properties :return: True / rel instance """ - if not hasattr(self.source, "element_id"): + if not hasattr(self.source, "element_id") or self.source.element_id is None: raise ValueError("Node has not been saved cannot connect!") if len(self): raise AttemptedCardinalityViolation("Node already has one relationship") diff --git a/neomodel/match.py b/neomodel/match.py index 022307cf..2773d6e5 100644 --- a/neomodel/match.py +++ b/neomodel/match.py @@ -733,7 +733,7 @@ def __nonzero__(self): def __contains__(self, obj): if isinstance(obj, StructuredNode): - if hasattr(obj, "element_id"): + if hasattr(obj, "element_id") and obj.element_id is not None: return self.query_cls(self).build_ast()._contains(obj.element_id) raise ValueError("Unsaved node: " + repr(obj)) From 970dae613b472cad434e6f1f9ad5950172c9915b Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Tue, 31 Oct 2023 15:31:07 +0100 Subject: [PATCH 64/76] Update Changelog and version tag --- Changelog | 5 +++++ neomodel/_version.py | 2 +- pyproject.toml | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Changelog b/Changelog index 5b82a305..3bea1704 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,8 @@ +Version 5.2.0 2023-11 +* Add support for pandas DataFrame and Series ; numpy Array +* Add relationship uniqueness constraints - for Neo4j >= 5.7 +* Add neomodel_inspect_database script, which inspects an existing database and creates neomodel class definitions for all objects. + Version 5.1.2 2023-09 * Raise ValueError on reserved keywords ; add tests #590 #623 * Add support for relationship property uniqueness constraints. Introduced in Neo4j 5.7. diff --git a/neomodel/_version.py b/neomodel/_version.py index 36887081..6c235c59 100644 --- a/neomodel/_version.py +++ b/neomodel/_version.py @@ -1 +1 @@ -__version__ = "5.1.3" +__version__ = "5.2.0" diff --git a/pyproject.toml b/pyproject.toml index 35a78a5d..1f57d699 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,7 @@ dependencies = [ "neobolt==1.7.17", "six==1.16.0", ] -version='5.1.3' +version='5.2.0' [project.urls] documentation = "https://neomodel.readthedocs.io/en/latest/" From 29c1ffcfa3fcd38b76e4ad6976c693dd372db2ec Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Thu, 2 Nov 2023 10:17:12 +0100 Subject: [PATCH 65/76] Fix typo in Changelog --- Changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog b/Changelog index 442c9c9f..0713c26f 100644 --- a/Changelog +++ b/Changelog @@ -1,4 +1,4 @@ -ersion 5.2.0 2023-11 +Version 5.2.0 2023-11 * Add an option to pass your own driver instead of relying on the automatically created one. See set_connection method. * Add a close_connection method to explicitly close the driver to match Neo4j deprecation. * Add a DATABASE_NAME config option, available for both auto- and self-managed driver modes. From acfb62d4d2bce3d6d43e7a5cb2b257affa446a51 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Thu, 2 Nov 2023 10:17:59 +0100 Subject: [PATCH 66/76] Add notice in Changelog about sync driver --- Changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog b/Changelog index 0713c26f..4fefdc0f 100644 --- a/Changelog +++ b/Changelog @@ -1,5 +1,5 @@ Version 5.2.0 2023-11 -* Add an option to pass your own driver instead of relying on the automatically created one. See set_connection method. +* Add an option to pass your own driver instead of relying on the automatically created one. See set_connection method. NB : only accepts the synchronous driver for now. * Add a close_connection method to explicitly close the driver to match Neo4j deprecation. * Add a DATABASE_NAME config option, available for both auto- and self-managed driver modes. * Add neomodel_inspect_database script, which inspects an existing database and creates neomodel class definitions for all objects. From 9c1bc4fc1d9750747ce3757730cbd5e58a846286 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Thu, 2 Nov 2023 11:12:50 +0100 Subject: [PATCH 67/76] Fix code smells --- neomodel/core.py | 12 +- neomodel/scripts/neomodel_inspect_database.py | 114 ++++++++++-------- 2 files changed, 72 insertions(+), 54 deletions(-) diff --git a/neomodel/core.py b/neomodel/core.py index b293dc5d..415a97af 100644 --- a/neomodel/core.py +++ b/neomodel/core.py @@ -409,11 +409,13 @@ def nodes(cls): @property def element_id(self): - return ( - int(self.element_id_property) - if db.database_version.startswith("4") - else self.element_id_property - ) if hasattr(self, "element_id_property") else None + if hasattr(self, "element_id_property"): + return ( + int(self.element_id_property) + if db.database_version.startswith("4") + else self.element_id_property + ) + return None # Version 4.4 support - id is deprecated in version 5.x @property diff --git a/neomodel/scripts/neomodel_inspect_database.py b/neomodel/scripts/neomodel_inspect_database.py index 75b0a696..147fd7e8 100644 --- a/neomodel/scripts/neomodel_inspect_database.py +++ b/neomodel/scripts/neomodel_inspect_database.py @@ -216,6 +216,67 @@ def generate_rel_class_name(rel_type): return string.capwords(rel_type.replace("_", " ")).replace(" ", "") + "Rel" +def parse_imports(): + imports = "" + if IMPORTS: + special_imports = "" + if "PointProperty" in IMPORTS: + IMPORTS.remove("PointProperty") + special_imports += ( + "from neomodel.contrib.spatial_properties import PointProperty\n" + ) + imports = f"from neomodel import {', '.join(IMPORTS)}\n" + special_imports + return imports + + +def build_rel_type_definition( + label, outgoing_relationships, properties, defined_rel_types +): + class_definition_append = "" + rel_type_definitions = "" + + for rel in outgoing_relationships: + rel_type = rel[0] + rel_name = rel_type.lower() + target_label = rel[1] + rel_props = rel[2] + + unique_properties = ( + RelationshipInspector.get_constraints_for_type(rel_type) + if db.version_is_higher_than("5.7") + else [] + ) + indexed_properties = RelationshipInspector.get_indexed_properties_for_type( + rel_type + ) + + cardinality = RelationshipInspector.infer_cardinality(rel_type, label) + + class_definition_append += f' {clean_class_member_key(rel_name)} = RelationshipTo("{target_label}", "{rel_type}", cardinality={cardinality}' + + if rel_props and rel_type not in defined_rel_types: + rel_model_name = generate_rel_class_name(rel_type) + class_definition_append += f', model="{rel_model_name}"' + rel_type_definitions = f"\n\nclass {rel_model_name}(StructuredRel):\n" + if properties: + rel_type_definitions += "".join( + [ + build_prop_string( + unique_properties, indexed_properties, prop, prop_type + ) + for prop, prop_type in properties.items() + ] + ) + else: + rel_type_definitions += " pass\n" + + class_definition_append += ")\n" + + class_definition_append += rel_type_definitions + + return class_definition_append + + def inspect_database(bolt_url): # Connect to the database print(f"Connecting to {bolt_url}") @@ -245,51 +306,14 @@ def inspect_database(bolt_url): ) outgoing_relationships = RelationshipInspector.outgoing_relationships(label) - rel_type_definitions = "" if outgoing_relationships and "StructuredRel" not in IMPORTS: IMPORTS.append("RelationshipTo") IMPORTS.append("StructuredRel") - for rel in outgoing_relationships: - rel_type = rel[0] - rel_name = rel_type.lower() - target_label = rel[1] - rel_props = rel[2] - - unique_properties = ( - RelationshipInspector.get_constraints_for_type(rel_type) - if db.version_is_higher_than("5.7") - else [] - ) - indexed_properties = RelationshipInspector.get_indexed_properties_for_type( - rel_type - ) - - cardinality = RelationshipInspector.infer_cardinality(rel_type, label) - - class_definition += f' {clean_class_member_key(rel_name)} = RelationshipTo("{target_label}", "{rel_type}", cardinality={cardinality}' - - if rel_props and rel_type not in defined_rel_types: - rel_model_name = generate_rel_class_name(rel_type) - class_definition += f', model="{rel_model_name}"' - rel_type_definitions = f"\n\nclass {rel_model_name}(StructuredRel):\n" - if properties: - rel_type_definitions += "".join( - [ - build_prop_string( - unique_properties, indexed_properties, prop, prop_type - ) - for prop, prop_type in properties.items() - ] - ) - else: - rel_type_definitions += " pass\n" - defined_rel_types.append(rel_type) - - class_definition += ")\n" - - class_definition += rel_type_definitions + class_definition += build_rel_type_definition( + label, outgoing_relationships, properties, defined_rel_types + ) if not properties and not outgoing_relationships: class_definition += " pass\n" @@ -299,15 +323,7 @@ def inspect_database(bolt_url): class_definitions += class_definition # Finally, parse imports - imports = "" - if IMPORTS: - special_imports = "" - if "PointProperty" in IMPORTS: - IMPORTS.remove("PointProperty") - special_imports += ( - "from neomodel.contrib.spatial_properties import PointProperty\n" - ) - imports = f"from neomodel import {', '.join(IMPORTS)}\n" + special_imports + imports = parse_imports() output = "\n".join([imports, class_definitions]) return output From 1d93b9fc97d85d48ef4150d864f83e6ae62627e2 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Thu, 2 Nov 2023 13:36:29 +0100 Subject: [PATCH 68/76] Test warning when pandas/numpy not installed --- test/test_cypher.py | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/test/test_cypher.py b/test/test_cypher.py index 2ca39a51..7c2e6fd6 100644 --- a/test/test_cypher.py +++ b/test/test_cypher.py @@ -1,11 +1,12 @@ +import builtins + +import pytest from neo4j.exceptions import ClientError as CypherError from numpy import ndarray from pandas import DataFrame, Series from neomodel import StringProperty, StructuredNode from neomodel.core import db -from neomodel.integration.numpy import to_ndarray -from neomodel.integration.pandas import to_dataframe, to_series class User2(StructuredNode): @@ -23,6 +24,18 @@ class UserNP(StructuredNode): email = StringProperty() +@pytest.fixture +def hide_available_pkg(monkeypatch, request): + import_orig = builtins.__import__ + + def mocked_import(name, *args, **kwargs): + if name == request.param: + raise ImportError() + return import_orig(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", mocked_import) + + def test_cypher(): """ test result format is backward compatible with earlier versions of neomodel @@ -56,7 +69,21 @@ def test_cypher_syntax_error(): assert False, "CypherError not raised." +@pytest.mark.parametrize("hide_available_pkg", ["pandas"], indirect=True) +def test_pandas_not_installed(hide_available_pkg): + with pytest.raises(ImportError): + with pytest.warns( + UserWarning, + match="The neomodel.integration.pandas module expects pandas to be installed", + ): + from neomodel.integration.pandas import to_dataframe + + _ = to_dataframe(db.cypher_query("MATCH (a) RETURN a.name AS name")) + + def test_pandas_integration(): + from neomodel.integration.pandas import to_dataframe, to_series + jimla = UserPandas(email="jimla@test.com", name="jimla").save() jimlo = UserPandas(email="jimlo@test.com", name="jimlo").save() @@ -86,7 +113,21 @@ def test_pandas_integration(): assert df["name"].tolist() == ["jimla", "jimlo"] +@pytest.mark.parametrize("hide_available_pkg", ["numpy"], indirect=True) +def test_numpy_not_installed(hide_available_pkg): + with pytest.raises(ImportError): + with pytest.warns( + UserWarning, + match="The neomodel.integration.numpy module expects pandas to be installed", + ): + from neomodel.integration.numpy import to_ndarray + + _ = to_ndarray(db.cypher_query("MATCH (a) RETURN a.name AS name")) + + def test_numpy_integration(): + from neomodel.integration.numpy import to_ndarray + jimly = UserNP(email="jimly@test.com", name="jimly").save() jimlu = UserNP(email="jimlu@test.com", name="jimlu").save() From 731c757ae5063ffd8dbab69285e90b4d37374982 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Thu, 2 Nov 2023 13:42:57 +0100 Subject: [PATCH 69/76] Remove obsolete code --- neomodel/scripts/neomodel_inspect_database.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/neomodel/scripts/neomodel_inspect_database.py b/neomodel/scripts/neomodel_inspect_database.py index 147fd7e8..ce000770 100644 --- a/neomodel/scripts/neomodel_inspect_database.py +++ b/neomodel/scripts/neomodel_inspect_database.py @@ -189,12 +189,6 @@ def get_node_labels(): return [record[0] for record in result] -def get_relationship_types(): - query = "CALL db.relationshipTypes()" - result, _ = db.cypher_query(query) - return [record[0] for record in result] - - def build_prop_string(unique_properties, indexed_properties, prop, prop_type): is_unique = prop in unique_properties is_indexed = prop in indexed_properties From d94d9d3bef4923b69d36e2a2e0f6501b25f241f2 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Thu, 2 Nov 2023 14:34:05 +0100 Subject: [PATCH 70/76] Fix bug in inspection script and improve test --- neomodel/scripts/neomodel_inspect_database.py | 27 +++++++---------- .../data/neomodel_inspect_database_output.txt | 11 +++++-- test/test_scripts.py | 30 +++++++------------ 3 files changed, 31 insertions(+), 37 deletions(-) diff --git a/neomodel/scripts/neomodel_inspect_database.py b/neomodel/scripts/neomodel_inspect_database.py index ce000770..07311e51 100644 --- a/neomodel/scripts/neomodel_inspect_database.py +++ b/neomodel/scripts/neomodel_inspect_database.py @@ -130,7 +130,7 @@ def outgoing_relationships(cls, start_label): @staticmethod def get_constraints_for_type(rel_type): constraints, meta_constraints = db.cypher_query( - f"SHOW CONSTRAINTS WHERE entityType='RELATIONSHIP' AND '{rel_type}' IN labelsOrTypes AND type='UNIQUENESS'" + f"SHOW CONSTRAINTS WHERE entityType='RELATIONSHIP' AND '{rel_type}' IN labelsOrTypes AND type='RELATIONSHIP_UNIQUENESS'" ) constraints_as_dict = [dict(zip(meta_constraints, row)) for row in constraints] constrained_properties = [ @@ -223,9 +223,7 @@ def parse_imports(): return imports -def build_rel_type_definition( - label, outgoing_relationships, properties, defined_rel_types -): +def build_rel_type_definition(label, outgoing_relationships, defined_rel_types): class_definition_append = "" rel_type_definitions = "" @@ -252,17 +250,14 @@ def build_rel_type_definition( rel_model_name = generate_rel_class_name(rel_type) class_definition_append += f', model="{rel_model_name}"' rel_type_definitions = f"\n\nclass {rel_model_name}(StructuredRel):\n" - if properties: - rel_type_definitions += "".join( - [ - build_prop_string( - unique_properties, indexed_properties, prop, prop_type - ) - for prop, prop_type in properties.items() - ] - ) - else: - rel_type_definitions += " pass\n" + rel_type_definitions += "".join( + [ + build_prop_string( + unique_properties, indexed_properties, prop, prop_type + ) + for prop, prop_type in rel_props.items() + ] + ) class_definition_append += ")\n" @@ -306,7 +301,7 @@ def inspect_database(bolt_url): IMPORTS.append("StructuredRel") class_definition += build_rel_type_definition( - label, outgoing_relationships, properties, defined_rel_types + label, outgoing_relationships, defined_rel_types ) if not properties and not outgoing_relationships: diff --git a/test/data/neomodel_inspect_database_output.txt b/test/data/neomodel_inspect_database_output.txt index 10bd55f9..8ddc9d39 100644 --- a/test/data/neomodel_inspect_database_output.txt +++ b/test/data/neomodel_inspect_database_output.txt @@ -8,8 +8,8 @@ class ScriptsTestNode(StructuredNode): class RelRel(StructuredRel): - personal_id = StringProperty() - name = StringProperty() + some_index_property = StringProperty(index=True) + some_unique_property = StringProperty(unique_index=True) class EveryPropertyTypeNode(StructuredNode): @@ -22,3 +22,10 @@ class EveryPropertyTypeNode(StructuredNode): integer_property = IntegerProperty() +class NoPropertyNode(StructuredNode): + pass + + +class NoPropertyRelNode(StructuredNode): + no_prop_rel = RelationshipTo("NoPropertyRelNode", "NO_PROP_REL", cardinality=ZeroOrOne) + diff --git a/test/test_scripts.py b/test/test_scripts.py index baf24155..0918bc8d 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -1,11 +1,6 @@ import subprocess from neomodel import ( - ArrayProperty, - BooleanProperty, - DateTimeProperty, - FloatProperty, - IntegerProperty, RelationshipTo, StringProperty, StructuredNode, @@ -15,7 +10,6 @@ install_labels, util, ) -from neomodel.contrib.spatial_properties import NeomodelPoint, PointProperty class ScriptsTestRel(StructuredRel): @@ -29,16 +23,6 @@ class ScriptsTestNode(StructuredNode): rel = RelationshipTo("ScriptsTestNode", "REL", model=ScriptsTestRel) -class EveryPropertyTypeNode(StructuredNode): - string_property = StringProperty() - boolean_property = BooleanProperty() - datetime_property = DateTimeProperty() - integer_property = IntegerProperty() - float_property = FloatProperty() - point_property = PointProperty(crs="wgs-84") - array_property = ArrayProperty(StringProperty()) - - def test_neomodel_install_labels(): result = subprocess.run( ["neomodel_install_labels", "--help"], @@ -124,9 +108,10 @@ def test_neomodel_inspect_database(): node1.rel.connect(node2, {"some_unique_property": "1", "some_index_property": "2"}) # Create a node with all the parsable property types + # Also create a node with no properties db.cypher_query( """ - CREATE (n:EveryPropertyTypeNode { + CREATE (:EveryPropertyTypeNode { string_property: "Hello World", boolean_property: true, datetime_property: datetime("2020-01-01T00:00:00.000Z"), @@ -135,6 +120,10 @@ def test_neomodel_inspect_database(): point_property: point({x: 0.0, y: 0.0, crs: "wgs-84"}), array_property: ["test"] }) + CREATE (:NoPropertyNode) + CREATE (n1:NoPropertyRelNode) + CREATE (n2:NoPropertyRelNode) + CREATE (n1)-[:NO_PROP_REL]->(n2) """ ) @@ -164,8 +153,11 @@ def test_neomodel_inspect_database(): .split(", ") ) assert set(parsed_imports) == set(expected_imports) - else: - assert line in wrapped_console_output + wrapped_test_file.remove(line) + break + + # Check that both outputs have the same set of lines, regardless of order and redundance + assert set(wrapped_test_file) == set(wrapped_console_output[2:]) # Test the file output version of the script result = subprocess.run( From e3f4ae38eb69071ee3753da921d14474b91c51f5 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Thu, 2 Nov 2023 14:39:41 +0100 Subject: [PATCH 71/76] Fix tests pre 5.7 --- ...omodel_inspect_database_output_pre_5_7.txt | 31 +++++++++++++++++++ test/test_scripts.py | 7 ++++- 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 test/data/neomodel_inspect_database_output_pre_5_7.txt diff --git a/test/data/neomodel_inspect_database_output_pre_5_7.txt b/test/data/neomodel_inspect_database_output_pre_5_7.txt new file mode 100644 index 00000000..8ddc9d39 --- /dev/null +++ b/test/data/neomodel_inspect_database_output_pre_5_7.txt @@ -0,0 +1,31 @@ +from neomodel import StructuredNode, StringProperty, RelationshipTo, StructuredRel, ZeroOrOne, ArrayProperty, FloatProperty, BooleanProperty, DateTimeProperty, IntegerProperty +from neomodel.contrib.spatial_properties import PointProperty + +class ScriptsTestNode(StructuredNode): + personal_id = StringProperty(unique_index=True) + name = StringProperty(index=True) + rel = RelationshipTo("ScriptsTestNode", "REL", cardinality=ZeroOrOne, model="RelRel") + + +class RelRel(StructuredRel): + some_index_property = StringProperty(index=True) + some_unique_property = StringProperty(unique_index=True) + + +class EveryPropertyTypeNode(StructuredNode): + array_property = ArrayProperty(StringProperty()) + float_property = FloatProperty() + boolean_property = BooleanProperty() + point_property = PointProperty(crs='wgs-84') + string_property = StringProperty() + datetime_property = DateTimeProperty() + integer_property = IntegerProperty() + + +class NoPropertyNode(StructuredNode): + pass + + +class NoPropertyRelNode(StructuredNode): + no_prop_rel = RelationshipTo("NoPropertyRelNode", "NO_PROP_REL", cardinality=ZeroOrOne) + diff --git a/test/test_scripts.py b/test/test_scripts.py index 0918bc8d..66594489 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -140,7 +140,12 @@ def test_neomodel_inspect_database(): ] assert wrapped_console_output[0].startswith("Connecting to") # Check that all the expected lines are here - with open("test/data/neomodel_inspect_database_output.txt", "r") as f: + file_path = ( + "test/data/neomodel_inspect_database_output.txt" + if db.version_is_higher_than("5.7") + else "test/data/neomodel_inspect_database_output_pre_5_7.txt" + ) + with open(file_path, "r") as f: wrapped_test_file = [line for line in f.read().split("\n") if line.strip()] for line in wrapped_test_file: # The neomodel components import order might differ From 7359fa07dfb83336e8ae020d5944d6d5de9fa3e4 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Thu, 2 Nov 2023 14:45:59 +0100 Subject: [PATCH 72/76] Remove uniqueness constraint in expected file pre 5.7 --- test/data/neomodel_inspect_database_output_pre_5_7.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/data/neomodel_inspect_database_output_pre_5_7.txt b/test/data/neomodel_inspect_database_output_pre_5_7.txt index 8ddc9d39..8dc8e412 100644 --- a/test/data/neomodel_inspect_database_output_pre_5_7.txt +++ b/test/data/neomodel_inspect_database_output_pre_5_7.txt @@ -9,7 +9,7 @@ class ScriptsTestNode(StructuredNode): class RelRel(StructuredRel): some_index_property = StringProperty(index=True) - some_unique_property = StringProperty(unique_index=True) + some_unique_property = StringProperty() class EveryPropertyTypeNode(StructuredNode): From f6f36c90f12ad655f3727305390e59a27f2c2638 Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Thu, 2 Nov 2023 14:58:16 +0100 Subject: [PATCH 73/76] Remove non-reachable line --- neomodel/scripts/neomodel_inspect_database.py | 1 - 1 file changed, 1 deletion(-) diff --git a/neomodel/scripts/neomodel_inspect_database.py b/neomodel/scripts/neomodel_inspect_database.py index 07311e51..1a24675a 100644 --- a/neomodel/scripts/neomodel_inspect_database.py +++ b/neomodel/scripts/neomodel_inspect_database.py @@ -81,7 +81,6 @@ def get_properties_for_label(label): result, _ = db.cypher_query(query) if result is not None and len(result) > 0: return result[0][0] - return {} @staticmethod def get_constraints_for_label(label): From 33cd9bce021c9f36b4b2d59a281a7296e7e1da3a Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Thu, 2 Nov 2023 15:55:00 +0100 Subject: [PATCH 74/76] Add upcoming breaking changes notice --- README.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.rst b/README.rst index c4edceb0..e911feb9 100644 --- a/README.rst +++ b/README.rst @@ -55,6 +55,21 @@ Documentation .. _readthedocs: http://neomodel.readthedocs.org +Upcoming breaking changes notice - >=5.3 +======================================== + +Based on Python version status_, neomodel will be dropping support for Python 3.7 in the next release (5.3). +This does not mean neomodel will stop working on Python 3.7, but it will no longer be tested against it. +Instead, we will try to add support for Python 3.12. + +.. _status: https://devguide.python.org/versions/ + +Another potential breaking change coming up is adding async support to neomodel. But we do not know when this will happen yet, +or if it will actually be a breaking change. We will definitely push this in a major release though. More to come on that later. + +Finally, we are looking at refactoring some standalone methods into the Database() class. More to come on that later. + + Installation ============ From 3a816c9ea8bdd34049081afeec15b9d47b9c2f8c Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Thu, 2 Nov 2023 15:58:34 +0100 Subject: [PATCH 75/76] Convert README to Markdown --- README.md | 116 +++++++++++++++++++++++++++++++++++++++++++++ README.rst | 126 ------------------------------------------------- pyproject.toml | 2 +- 3 files changed, 117 insertions(+), 127 deletions(-) create mode 100644 README.md delete mode 100644 README.rst diff --git a/README.md b/README.md new file mode 100644 index 00000000..abb09588 --- /dev/null +++ b/README.md @@ -0,0 +1,116 @@ +![neomodel](https://raw.githubusercontent.com/neo4j-contrib/neomodel/master/doc/source/_static/neomodel-300.png) + +An Object Graph Mapper (OGM) for the [neo4j](https://neo4j.com/) graph +database, built on the awesome +[neo4j_driver](https://github.com/neo4j/neo4j-python-driver) + +If you need assistance with neomodel, please create an issue on the +GitHub repo found at . + +- Familiar class based model definitions with proper inheritance. +- Powerful query API. +- Schema enforcement through cardinality restrictions. +- Full transaction support. +- Thread safe. +- Pre/post save/delete hooks. +- Django integration via + [django_neomodel](https://github.com/neo4j-contrib/django-neomodel) + +[![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=neo4j-contrib_neomodel&metric=reliability_rating)](https://sonarcloud.io/summary/new_code?id=neo4j-contrib_neomodel) + +[![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=neo4j-contrib_neomodel&metric=security_rating)](https://sonarcloud.io/summary/new_code?id=neo4j-contrib_neomodel) + +[![Documentation Status](https://readthedocs.org/projects/neomodel/badge/?version=latest)](https://neomodel.readthedocs.io/en/latest/?badge=latest) + +# Requirements + +**For neomodel releases 5.x :** + +- Python 3.7+ +- Neo4j 5.x, 4.4 (LTS) + +**For neomodel releases 4.x :** + +- Python 3.7 -\> 3.10 +- Neo4j 4.x (including 4.4 LTS for neomodel version 4.0.10) + +# Documentation + +(Needs an update, but) Available on +[readthedocs](http://neomodel.readthedocs.org). + +# Upcoming breaking changes notice - \>=5.3 + +Based on Python version [status](https://devguide.python.org/versions/), +neomodel will be dropping support for Python 3.7 in the next release +(5.3). This does not mean neomodel will stop working on Python 3.7, but +it will no longer be tested against it. Instead, we will try to add +support for Python 3.12. + +Another potential breaking change coming up is adding async support to +neomodel. But we do not know when this will happen yet, or if it will +actually be a breaking change. We will definitely push this in a major +release though. More to come on that later. + +Finally, we are looking at refactoring some standalone methods into the +Database() class. More to come on that later. + +# Installation + +Install from pypi (recommended): + + $ pip install neomodel ($ source dev # To install all things needed in a Python3 venv) + + # Neomodel has some optional dependencies (including Shapely), to install these use: + + $ pip install neomodel['extras'] + +To install from github: + + $ pip install git+git://github.com/neo4j-contrib/neomodel.git@HEAD#egg=neomodel-dev + +# Contributing + +Ideas, bugs, tests and pull requests always welcome. Please use +GitHub\'s Issues page to track these. + +If you are interested in developing `neomodel` further, pick a subject +from the Issues page and open a Pull Request (PR) for it. If you are +adding a feature that is not captured in that list yet, consider if the +work for it could also contribute towards delivering any of the existing +issues too. + +## Running the test suite + +Make sure you have a Neo4j database version 4 or higher to run the tests +on.: + + $ export NEO4J_BOLT_URL=bolt://:@localhost:7687 # check your username and password + +Ensure `dbms.security.auth_enabled=true` in your database configuration +file. Setup a virtual environment, install neomodel for development and +run the test suite: : + + $ pip install -e '.[dev]' + $ pytest + +The tests in \"test_connection.py\" will fail locally if you don\'t +specify the following environment variables: + + $ export AURA_TEST_DB_USER=username + $ export AURA_TEST_DB_PASSWORD=password + $ export AURA_TEST_DB_HOSTNAME=url + +If you are running a neo4j database for the first time the test suite +will set the password to \'test\'. If the database is already populated, +the test suite will abort with an error message and ask you to re-run it +with the `--resetdb` switch. This is a safeguard to ensure that the test +suite does not accidentally wipe out a database if you happen to not +have restarted your Neo4j server to point to a (usually named) +`debug.db` database. + +If you have `docker-compose` installed, you can run the test suite +against all supported Python interpreters and neo4j versions: : + + # in the project's root folder: + $ sh ./tests-with-docker-compose.sh diff --git a/README.rst b/README.rst deleted file mode 100644 index e911feb9..00000000 --- a/README.rst +++ /dev/null @@ -1,126 +0,0 @@ -.. image:: https://raw.githubusercontent.com/neo4j-contrib/neomodel/master/doc/source/_static/neomodel-300.png - :alt: neomodel - -An Object Graph Mapper (OGM) for the neo4j_ graph database, built on the awesome neo4j_driver_ - -If you need assistance with neomodel, please create an issue on the GitHub repo found at https://github.com/neo4j-contrib/neomodel/. - -- Familiar class based model definitions with proper inheritance. -- Powerful query API. -- Schema enforcement through cardinality restrictions. -- Full transaction support. -- Thread safe. -- Pre/post save/delete hooks. -- Django integration via django_neomodel_ - -.. _django_neomodel: https://github.com/neo4j-contrib/django-neomodel -.. _neo4j: https://neo4j.com/ -.. _neo4j_driver: https://github.com/neo4j/neo4j-python-driver - -.. image:: https://sonarcloud.io/api/project_badges/measure?project=neo4j-contrib_neomodel&metric=reliability_rating - :alt: Reliability Rating - :scale: 100% - :target: https://sonarcloud.io/summary/new_code?id=neo4j-contrib_neomodel - -.. image:: https://sonarcloud.io/api/project_badges/measure?project=neo4j-contrib_neomodel&metric=security_rating - :alt: Security Rating - :scale: 100% - :target: https://sonarcloud.io/summary/new_code?id=neo4j-contrib_neomodel - -.. image:: https://readthedocs.org/projects/neomodel/badge/?version=latest - :alt: Documentation Status - :scale: 100% - :target: https://neomodel.readthedocs.io/en/latest/?badge=latest - -Requirements -============ - -**For neomodel releases 5.x :** - -* Python 3.7+ -* Neo4j 5.x, 4.4 (LTS) - - -**For neomodel releases 4.x :** - -* Python 3.7 -> 3.10 -* Neo4j 4.x (including 4.4 LTS for neomodel version 4.0.10) - - -Documentation -============= - -(Needs an update, but) Available on readthedocs_. - -.. _readthedocs: http://neomodel.readthedocs.org - - -Upcoming breaking changes notice - >=5.3 -======================================== - -Based on Python version status_, neomodel will be dropping support for Python 3.7 in the next release (5.3). -This does not mean neomodel will stop working on Python 3.7, but it will no longer be tested against it. -Instead, we will try to add support for Python 3.12. - -.. _status: https://devguide.python.org/versions/ - -Another potential breaking change coming up is adding async support to neomodel. But we do not know when this will happen yet, -or if it will actually be a breaking change. We will definitely push this in a major release though. More to come on that later. - -Finally, we are looking at refactoring some standalone methods into the Database() class. More to come on that later. - - -Installation -============ - -Install from pypi (recommended):: - - $ pip install neomodel ($ source dev # To install all things needed in a Python3 venv) - - Neomodel has some optional dependencies (including Shapely), to install these use: - - $ pip install neomodel['extras'] - -To install from github:: - - $ pip install git+git://github.com/neo4j-contrib/neomodel.git@HEAD#egg=neomodel-dev - -Contributing -============ - -Ideas, bugs, tests and pull requests always welcome. Please use GitHub's Issues page to track these. - -If you are interested in developing ``neomodel`` further, pick a subject from the Issues page and open a Pull Request (PR) for -it. If you are adding a feature that is not captured in that list yet, consider if the work for it could also -contribute towards delivering any of the existing issues too. - -Running the test suite ----------------------- - -Make sure you have a Neo4j database version 4 or higher to run the tests on.:: - - $ export NEO4J_BOLT_URL=bolt://:@localhost:7687 # check your username and password - -Ensure ``dbms.security.auth_enabled=true`` in your database configuration file. -Setup a virtual environment, install neomodel for development and run the test suite: :: - - $ pip install -e '.[dev]' - $ pytest - -The tests in "test_connection.py" will fail locally if you don't specify the following environment variables:: - - $ export AURA_TEST_DB_USER=username - $ export AURA_TEST_DB_PASSWORD=password - $ export AURA_TEST_DB_HOSTNAME=url - -If you are running a neo4j database for the first time the test suite will set the password to 'test'. -If the database is already populated, the test suite will abort with an error message and ask you to re-run it with the -``--resetdb`` switch. This is a safeguard to ensure that the test suite does not accidentally wipe out a database if you happen -to not have restarted your Neo4j server to point to a (usually named) ``debug.db`` database. - -If you have ``docker-compose`` installed, you can run the test suite against all supported Python -interpreters and neo4j versions: :: - - # in the project's root folder: - $ sh ./tests-with-docker-compose.sh - diff --git a/pyproject.toml b/pyproject.toml index 1f57d699..c5cda7e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ maintainers = [ {name = "Marius Conjeaud", email = "marius.conjeaud@outlook.com"}, ] description = "An object mapper for the neo4j graph database." -readme = "README.rst" +readme = "README.md" requires-python = ">=3.7" keywords = ["graph", "neo4j", "ORM", "OGM", "mapper"] license = {text = "MIT"} From bf4025c717ef4e90bd7d1ee9693315bc72fbc5ae Mon Sep 17 00:00:00 2001 From: Marius Conjeaud Date: Thu, 2 Nov 2023 16:00:29 +0100 Subject: [PATCH 76/76] Improve README layout --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index abb09588..b9f87179 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,7 @@ GitHub repo found at . [django_neomodel](https://github.com/neo4j-contrib/django-neomodel) [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=neo4j-contrib_neomodel&metric=reliability_rating)](https://sonarcloud.io/summary/new_code?id=neo4j-contrib_neomodel) - [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=neo4j-contrib_neomodel&metric=security_rating)](https://sonarcloud.io/summary/new_code?id=neo4j-contrib_neomodel) - [![Documentation Status](https://readthedocs.org/projects/neomodel/badge/?version=latest)](https://neomodel.readthedocs.io/en/latest/?badge=latest) # Requirements