Arweave files (#1226)

* Basic support for arweave files.
* Move all simple flows to parameterised test in consume.
* Add arweave type to files factory.

ocean_lib/ocean/ocean_assets.py

@@ -4,7 +4,6 @@
 #
 
 """Ocean module."""
-import glob
 import json
 import logging
 import lzma
@@ -28,16 +27,15 @@
 from ocean_lib.models.compute_input import ComputeInput
 from ocean_lib.models.data_nft import DataNFT
 from ocean_lib.models.datatoken import Datatoken
-from ocean_lib.models.dispenser import Dispenser
-from ocean_lib.ocean.util import (
-    create_checksum,
-    get_address_of_type,
-    get_ocean_token_address,
-)
+from ocean_lib.ocean.util import create_checksum, get_ocean_token_address
 from ocean_lib.services.service import Service
 from ocean_lib.structures.algorithm_metadata import AlgorithmMetadata
-from ocean_lib.structures.file_objects import GraphqlQuery, SmartContractCall, UrlFile
-from ocean_lib.web3_internal.constants import ZERO_ADDRESS
+from ocean_lib.structures.file_objects import (
+    ArweaveFile,
+    GraphqlQuery,
+    SmartContractCall,
+    UrlFile,
+)
 from ocean_lib.web3_internal.utils import check_network
 
 logger = logging.getLogger("ocean")
@@ -187,6 +185,19 @@ def create_url_asset(
         files = [UrlFile(url)]
         return self._create_1dt(metadata, files, publisher_wallet, wait_for_aqua)
 
+    @enforce_types
+    def create_arweave_asset(
+        self,
+        name: str,
+        transaction_id: str,
+        publisher_wallet,
+        wait_for_aqua: bool = True,
+    ) -> tuple:
+        """Create asset of type "data", having UrlFiles, with good defaults"""
+        metadata = self._default_metadata(name, publisher_wallet)
+        files = [ArweaveFile(transaction_id)]
+        return self._create_1dt(metadata, files, publisher_wallet, wait_for_aqua)
+
     @enforce_types
     def create_graphql_asset(
         self,

ocean_lib/structures/file_objects.py

@@ -85,6 +85,17 @@ def to_dict(self) -> dict:
         }
 
 
+class ArweaveFile(FilesType):
+    @enforce_types
+    def __init__(self, transaction_id: str) -> None:
+        self.transaction_id = transaction_id
+        self.type = "arweave"
+
+    @enforce_types
+    def to_dict(self) -> dict:
+        return {"type": self.type, "transactionId": self.transaction_id}
+
+
 @enforce_types
 def FilesTypeFactory(file_obj: dict) -> FilesType:
     """Factory Method"""
@@ -94,6 +105,8 @@ def FilesTypeFactory(file_obj: dict) -> FilesType:
             method=file_obj.get("method", "GET"),
             headers=file_obj.get("headers"),
         )
+    elif file_obj["type"] == "arweave":
+        return ArweaveFile(file_obj["transactionId"])
     elif file_obj["type"] == "ipfs":
         return IpfsFile(file_obj["hash"])
     elif file_obj["type"] == "graphql":

tests/integration/ganache/test_consume_flow.py

@@ -10,28 +10,63 @@
 
 from ocean_lib.agreements.service_types import ServiceTypes
 from ocean_lib.data_provider.data_service_provider import DataServiceProvider
-from ocean_lib.ocean.ocean import Ocean
 from ocean_lib.ocean.ocean_assets import OceanAssets
+from ocean_lib.ocean.util import get_address_of_type
 from ocean_lib.web3_internal.constants import ZERO_ADDRESS
-from tests.resources.ddo_helpers import (
-    get_first_service_by_type,
-    get_registered_asset_with_access_service,
-)
+from tests.resources.ddo_helpers import get_first_service_by_type
+
+"""branin.arff dataset, permanently stored in Arweave"""
+ARWEAVE_TRANSACTION_ID = "a4qJoQZa1poIv5guEzkfgZYSAD0uYm7Vw4zm_tCswVQ"
 
 
 @pytest.mark.integration
-def test_consume_flow(
-    config: dict,
-    publisher_wallet,
-    consumer_wallet,
-):
-    ocean = Ocean(config)
-
-    # Publish a plain asset with one data token on chain
-    data_nft, dt, ddo = get_registered_asset_with_access_service(
-        ocean,
-        publisher_wallet,
-    )
+@pytest.mark.parametrize("asset_type", ["simple", "graphql", "onchain", "arweave"])
+def test_consume_asset(config: dict, publisher_wallet, consumer_wallet, asset_type):
+    data_provider = DataServiceProvider
+    ocean_assets = OceanAssets(config, data_provider)
+
+    if asset_type == "simple":
+        url = "https://raw.githubusercontent.com/trentmc/branin/main/branin.arff"
+        data_nft, dt, ddo = ocean_assets.create_url_asset(
+            "Data NFTs in Ocean", url, publisher_wallet
+        )
+    elif asset_type == "arweave":
+        data_nft, dt, ddo = ocean_assets.create_arweave_asset(
+            "Data NFTs in Ocean", ARWEAVE_TRANSACTION_ID, publisher_wallet
+        )
+    elif asset_type == "onchain":
+        abi = {
+            "inputs": [],
+            "name": "swapOceanFee",
+            "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}],
+            "stateMutability": "view",
+            "type": "function",
+        }
+        router_address = get_address_of_type(config, "Router")
+
+        data_nft, dt, ddo = ocean_assets.create_onchain_asset(
+            "Data NFTs in Ocean", router_address, abi, publisher_wallet
+        )
+    else:
+        url = "http://172.15.0.15:8000/subgraphs/name/oceanprotocol/ocean-subgraph"
+        query = """
+            query{
+                nfts(orderby: createdtimestamp,orderdirection:desc){
+                    id
+                    symbol
+                    createdtimestamp
+                }
+            }
+            """
+
+        data_nft, dt, ddo = ocean_assets.create_graphql_asset(
+            "Data NFTs in Ocean", url, query, publisher_wallet
+        )
+
+    assert ddo, "The ddo is not created."
+    assert ddo.nft["address"] == data_nft.address
+    assert ddo.nft["owner"] == publisher_wallet.address
+    assert ddo.datatokens[0]["name"] == "Data NFTs in Ocean: DT1"
 
     service = get_first_service_by_type(ddo, ServiceTypes.ASSET_ACCESS)
 
@@ -43,7 +78,7 @@ def test_consume_flow(
     )
 
     # Initialize service
-    response = DataServiceProvider.initialize(
+    response = data_provider.initialize(
         did=ddo.did, service=service, consumer_address=consumer_wallet.address
     )
     assert response
@@ -80,8 +115,12 @@ def test_consume_flow(
 
     assert len(os.listdir(destination)) == 0
 
-    ocean.assets.download_asset(
-        ddo, consumer_wallet, destination, receipt.txid, service
+    ocean_assets.download_asset(
+        ddo,
+        consumer_wallet,
+        destination,
+        receipt.txid,
+        service,
     )
 
     assert (

tests/integration/ganache/test_onchain.py

@@ -19,94 +19,6 @@
 from tests.resources.ddo_helpers import get_first_service_by_type
 
 
-@pytest.mark.integration
-def test_consume_simple_onchain_data(
-    config: dict,
-    publisher_wallet,
-    consumer_wallet,
-):
-    data_provider = DataServiceProvider
-    ocean_assets = OceanAssets(config, data_provider)
-    abi = {
-        "inputs": [],
-        "name": "swapOceanFee",
-        "outputs": [{"internalType": "uint256", "name": "", "type": "uint256"}],
-        "stateMutability": "view",
-        "type": "function",
-    }
-    router_address = get_address_of_type(config, "Router")
-
-    data_nft, dt, ddo = ocean_assets.create_onchain_asset(
-        "NFT", router_address, abi, publisher_wallet
-    )
-
-    assert ddo, "The ddo is not created."
-    assert ddo.nft["name"] == "NFT"
-    assert ddo.nft["address"] == data_nft.address
-    assert ddo.nft["owner"] == publisher_wallet.address
-    assert ddo.datatokens[0]["name"] == "NFT: DT1"
-    assert ddo.datatokens[0]["symbol"] == "DT1"
-
-    service = get_first_service_by_type(ddo, ServiceTypes.ASSET_ACCESS)
-    dt = Datatoken(config, ddo.datatokens[0]["address"])
-
-    # Mint 50 datatokens in consumer wallet from publisher. Max cap = 100
-    dt.mint(
-        consumer_wallet.address,
-        Web3.toWei("50", "ether"),
-        {"from": publisher_wallet},
-    )
-
-    # Initialize service
-    response = data_provider.initialize(
-        did=ddo.did, service=service, consumer_address=consumer_wallet.address
-    )
-    assert response
-    assert response.status_code == 200
-    assert response.json()["providerFee"]
-    provider_fees = response.json()["providerFee"]
-
-    # Start order for consumer
-    receipt = dt.start_order(
-        consumer=consumer_wallet.address,
-        service_index=ddo.get_index_of_service(service),
-        provider_fees=provider_fees,
-        consume_market_order_fee_address=ZERO_ADDRESS,
-        consume_market_order_fee_token=ZERO_ADDRESS,
-        consume_market_order_fee_amount=0,
-        transaction_parameters={"from": consumer_wallet},
-    )
-
-    # Download file
-    destination = config["DOWNLOADS_PATH"]
-    if not os.path.isabs(destination):
-        destination = os.path.abspath(destination)
-
-    if os.path.exists(destination) and len(os.listdir(destination)) > 0:
-        list(
-            map(
-                lambda d: shutil.rmtree(os.path.join(destination, d)),
-                os.listdir(destination),
-            )
-        )
-
-    if not os.path.exists(destination):
-        os.makedirs(destination)
-
-    assert len(os.listdir(destination)) == 0
-
-    ocean_assets.download_asset(
-        ddo,
-        consumer_wallet,
-        destination,
-        receipt.txid,
-        service,
-    )
-
-    dir_files = os.listdir(os.path.join(destination, os.listdir(destination)[0]))
-    assert len(dir_files) == 1, "The asset folder is empty."
-
-
 @pytest.mark.integration
 def test_consume_parametrized_onchain_data(
     config: dict,