From cd468714f62ea2c77e49fc27fb41afb94cdc2983 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 22 Jun 2022 16:07:34 +0200 Subject: [PATCH] Introduction of AttemptClass and PaymentClass (Issue #20) (#24) * add .gitignore * introduce register payment method on OracleLightningNetwork.py * adding settlement_payment in OracleLightningNetwork first stub of a method as described in #16 Still needs to be tested * rewriting setter for `actual_liquidity` using properties (python way) * adjust channels in both directions when payment is made * Settlement of onions with partial payments only after all onions were routed successfully * deletion of unused import * shell for a Payment Class * Revert "shell for a Payment Class" This reverts commit 8dffa28ea2de5c4781d558b8cb6634a272b65291. * shell for a Payment Class for issue #20 * shell for as Attempt Class for issue #20 * Indroduction of Payment and Attempt class, refactoring. * Introduction of PaymentClass and AttemptClass, refactoring. * error fixing, semantic changes and refinement of Payment ant Attempt class * adding types in methods for Attempt and Payment Class and amending relative paths on imports * initializes fee and probability w/ none (instead of -1) #20 * adjustment of relative paths for modules in import * extending .gitignore * correcting tipo for gitignore * init for documentation * configuration updated * correction in filter function for Attempt.Status #24 * changing imports back to initial call * fixing issues from partial review https://github.com/renepickhardt/pickhardtpayments/pull/24#pullrequestreview-1010067780_ * deleting docs folder and moving _estimate_payment_statistics into attempt class * deleting docs folder and moving _estimate_payment_statistics into attempt class * removed doc folder * adding CHANGELOG * Cleaning up after Attempt and Payment PR review * changed filter function in payment to generator * added setter to OracleChannel and fixed imports in init.py * Fix typos in documentation * rewriting setter for `actual_liquidity` using properties (python way) * Indroduction of Payment and Attempt class, refactoring. * Introduction of PaymentClass and AttemptClass, refactoring. * adjustment of relative paths for modules in import * changing imports back to initial call * another rebasing hassle ;) Co-authored-by: Sebastian Co-authored-by: Rene Pickhardt Co-authored-by: nassersaazi --- .gitignore | 11 + CHANGELOG.md | 28 ++ LICNENSE => LICENSE | 0 examples/basicexample.py | 2 +- pickhardtpayments/Attempt.py | 136 ++++++++ pickhardtpayments/Channel.py | 4 +- pickhardtpayments/ChannelGraph.py | 4 +- pickhardtpayments/OracleChannel.py | 18 +- pickhardtpayments/OracleLightningNetwork.py | 34 +- pickhardtpayments/Payment.py | 183 +++++++++++ .../SyncSimulatedPaymentSession.py | 299 ++++++++++-------- pickhardtpayments/UncertaintyChannel.py | 8 +- pickhardtpayments/UncertaintyNetwork.py | 30 +- pickhardtpayments/__init__.py | 2 +- 14 files changed, 588 insertions(+), 171 deletions(-) create mode 100644 .gitignore create mode 100644 CHANGELOG.md rename LICNENSE => LICENSE (100%) create mode 100644 pickhardtpayments/Attempt.py create mode 100644 pickhardtpayments/Payment.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..52cff06 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +.DS_Store +__pycache__ +.idea +pickhardtpayments/pickhardtpayments.egg-info +pickhardtpayments/__pycache__ + +# not including large file with Channel Graph +*.json +examples/.json + +pickhardt_pay.log \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..41d06cf --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,28 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + + +## [Unreleased] + +## [0.1.0] - 2022-06-21 +### Added + - introduction of an Attempt Class and a Payment Class ([#28]) + - introduction of an AttemptStatus to describe the state of the Attempt ([#28]) + - settle_payment in OracleLightingNetwork is added ([#28]) + - logging added in SyncSimulatedPaymentSession ([#28]) + +### Changed + - calculation of fees and probabilities is moved from SyncSimulatedPaymentSession to Attempt class ([#28]) + +### Deprecated + +### Removed + +### Fixed + +### EXPERIMENTAL diff --git a/LICNENSE b/LICENSE similarity index 100% rename from LICNENSE rename to LICENSE diff --git a/examples/basicexample.py b/examples/basicexample.py index 5085ce0..dcf6b7b 100644 --- a/examples/basicexample.py +++ b/examples/basicexample.py @@ -29,4 +29,4 @@ C_OTTO = "027ce055380348d7812d2ae7745701c9f93e70c1adeb2657f053f91df4f2843c71" tested_amount = 10_000_000 #10 million sats -payment_session.pickhardt_pay(RENE,C_OTTO, tested_amount,mu=0,base=0) +payment_session.pickhardt_pay(RENE, C_OTTO, tested_amount, mu=0, base=0) diff --git a/pickhardtpayments/Attempt.py b/pickhardtpayments/Attempt.py new file mode 100644 index 0000000..3906caa --- /dev/null +++ b/pickhardtpayments/Attempt.py @@ -0,0 +1,136 @@ +from enum import Enum + +from Channel import Channel +from pickhardtpayments import UncertaintyChannel + + +class AttemptStatus(Enum): + PLANNED = 1 + INFLIGHT = 2 + ARRIVED = 4 + FAILED = 8 + SETTLED = 16 + + +class Attempt: + """ + An Attempt describes a path (a list of channels) of an amount from sender to receiver. + + # TODO Describe life cycle of an Attempt + + When sending an amount of sats from sender to receiver, a payment is usually split up and sent across + several paths, to increase the probability of being successfully delivered. Each of these paths is referred to as an + Attempt. + An Attempt consists of a list of Channels (class:Channel) and the amount in sats to be sent through this path. + When an Attempt is instantiated, the given amount is allocated to the inflight amount in the channels of the + path and the AttemptStatus is set to PLANNED. + + :param path: a list of Channel objects from sender to receiver + :type path: list[Channel] + :param amount: the amount to be transferred from source to destination + :type amount: int + """ + + def __init__(self, path: list[UncertaintyChannel], amount: int = 0): + """Constructor method + """ + if amount >= 0: + self._amount = amount + else: + raise ValueError("amount for payment attempts needs to be positive") + + i = 1 + valid_path = True + while i < len(path): + valid_path = valid_path and (path[i - 1].dest == path[i].src) + i += 1 + if valid_path: + self._path = path + + channel: UncertaintyChannel + self._routing_fee = 0 + self._probability = 1 + for channel in path: + self._routing_fee += channel.routing_cost_msat(amount) + self._probability *= channel.success_probability(amount) + # When Attempt is created, all amounts are set inflight. Needs to be updated with AttemptStatus change! + # This is to correctly compute conditional probabilities of non-disjoint paths in the same set of paths + # channel.in_flight(amount) + channel.allocate_amount(amount) + self._status = AttemptStatus.PLANNED + + def __str__(self): + description = "Path with {} channels to deliver {} sats and status {}.".format(len(self._path), + self._amount, self._status.name) + if self._routing_fee and self._routing_fee > 0: + description += "\nsuccess probability of {:6.2f}% , fee of {:8.3f} sat and a ppm of {:5} ".format( + self._probability * 100, self._routing_fee/1000, int(self._routing_fee * 1000 / self._amount)) + return description + + @property + def path(self) -> list[Channel]: + """Returns the path of the attempt. + + :return: the list of Channels that the path consists of + :rtype: list[Channel] + """ + return self._path + + @property + def amount(self) -> int: + """Returns the amount of the attempt. + + :return: the amount that was tried to send in this Attempt + :rtype: int + """ + return self._amount + + @property + def status(self) -> AttemptStatus: + """Returns the status of the attempt. + + :return: returns the state of the attempt + :rtype: AttemptStatus + """ + return self._status + + @status.setter + def status(self, value: AttemptStatus): + """Sets the status of the attempt. + + A flag to describe if the path failed, succeeded or was used for settlement (see enum SettlementStatus) + + :param value: Current state of the Attempt + :type value: AttemptStatus + """ + if not self._status == value: + # remove allocated amounts when Attempt status changes from PLANNED + if self._status == AttemptStatus.PLANNED and not value == AttemptStatus.INFLIGHT: + for channel in self._path: + channel.allocate_amount(-self._amount) + + if self._status == AttemptStatus.INFLIGHT and value == AttemptStatus.ARRIVED: + # TODO write amount from inflight to min_liquidity/max_liquidity + # for channel in self._path: + # channel.allocate_amount(-self._amount) + pass + + self._status = value + + @property + def routing_fee(self) -> int: + """Returns the accrued routing fee in msat requested for this path. + + :return: accrued routing fees for this attempt in msat + :rtype: int + """ + return self._routing_fee + + @property + def probability(self) -> float: + """Returns estimated success probability before the attempt + + :return: estimated success probability before the attempt + :rtype: float + """ + return self._probability diff --git a/pickhardtpayments/Channel.py b/pickhardtpayments/Channel.py index eb67fcd..93eae38 100644 --- a/pickhardtpayments/Channel.py +++ b/pickhardtpayments/Channel.py @@ -1,4 +1,4 @@ -class ChannelFields(): +class ChannelFields: """ These are the values describing public data about channels that is either available via gossip or via the Bitcoin Blockchain. Their format is taken from the core lighting @@ -21,7 +21,7 @@ class ChannelFields(): SHORT_CHANNEL_ID = 'short_channel_id' -class Channel(): +class Channel: """ Stores the public available information of a channel. diff --git a/pickhardtpayments/ChannelGraph.py b/pickhardtpayments/ChannelGraph.py index 748a261..578b60b 100644 --- a/pickhardtpayments/ChannelGraph.py +++ b/pickhardtpayments/ChannelGraph.py @@ -3,7 +3,7 @@ from .Channel import Channel -class ChannelGraph(): +class ChannelGraph: """ Represents the public information about the Lightning Network that we see from Gossip and the Bitcoin Blockchain. @@ -15,7 +15,7 @@ class ChannelGraph(): def _get_channel_json(self, filename: str): """ - extracts the dictionary from the file that contains lightnig-cli listchannels json string + extracts the dictionary from the file that contains lightning-cli listchannels json string """ f = open(filename) return json.load(f)["channels"] diff --git a/pickhardtpayments/OracleChannel.py b/pickhardtpayments/OracleChannel.py index bb1d7a9..23c30ca 100644 --- a/pickhardtpayments/OracleChannel.py +++ b/pickhardtpayments/OracleChannel.py @@ -17,7 +17,7 @@ def __init__(self, channel: Channel, actual_liquidity: int = None): self._actual_liquidity = random.randint(0, self.capacity) def __str__(self): - return super().__str__()+" actual Liquidity: {}".format(self.actual_liquidity) + return super().__str__() + " actual Liquidity: {}".format(self.actual_liquidity) @property def actual_liquidity(self): @@ -30,10 +30,18 @@ def actual_liquidity(self): return self._actual_liquidity @actual_liquidity.setter - def actual_liquidity(self,amt): - self._actual_liquidity = amt + def actual_liquidity(self, amt: int): + """Sets the liquidity of a channel in the Oracle Network + + :param amt: amount to be assigned to channel liquidity + :type amt: int + """ + if 0 <= amt <= self.capacity: + self._actual_liquidity = amt + else: + raise ValueError(f"Liquidity for channel {self.short_channel_id} cannot be set. Amount {amt} is negative or higher than capacity") + - def can_forward(self, amt: int): """ check if the oracle channel can forward a certain amount @@ -41,4 +49,4 @@ def can_forward(self, amt: int): if amt <= self.actual_liquidity: return True else: - return False + return False \ No newline at end of file diff --git a/pickhardtpayments/OracleLightningNetwork.py b/pickhardtpayments/OracleLightningNetwork.py index 2be2969..eb8aacc 100644 --- a/pickhardtpayments/OracleLightningNetwork.py +++ b/pickhardtpayments/OracleLightningNetwork.py @@ -1,3 +1,5 @@ +from typing import List + from .ChannelGraph import ChannelGraph from .OracleChannel import OracleChannel import networkx as nx @@ -13,7 +15,7 @@ def __init__(self, channel_graph: ChannelGraph): for src, dest, short_channel_id, channel in channel_graph.network.edges(data="channel", keys=True): oracle_channel = None - # If Channel in oposite direction already exists with liquidity information match the channel + # If Channel in opposite direction already exists with liquidity information match the channel if self._network.has_edge(dest, src): if short_channel_id in self._network[dest][src]: capacity = channel.capacity @@ -35,14 +37,18 @@ def network(self): return self._network def send_onion(self, path, amt): + """ + + :rtype: object + """ for channel in path: oracle_channel = self.get_channel( channel.src, channel.dest, channel.short_channel_id) success_of_probe = oracle_channel.can_forward( - channel.in_flight+amt) + channel.in_flight + amt) # print(channel,amt,success_of_probe) channel.update_knowledge(amt, success_of_probe) - if success_of_probe == False: + if not success_of_probe: return False, channel return True, None @@ -55,7 +61,7 @@ def theoretical_maximum_payable_amount(self, source: str, destination: str, base """ test_network = nx.DiGraph() for src, dest, channel in self.network.edges(data="channel"): - #liqudity = 0 + # liquidity = 0 # for channel in channels: if channel.base_fee > base_fee: continue @@ -71,3 +77,23 @@ def theoretical_maximum_payable_amount(self, source: str, destination: str, base mincut, _ = nx.minimum_cut(test_network, source, destination) return mincut + + def settle_payment(self, path: List[OracleChannel], payment_amount: int): + """ + receives a List of channels and payment amount and adjusts the balances of the channels along the path. + + settle_payment should only be called after all send_onions for a payment terminated successfully! + # TODO testing + """ + for channel in path: + settlement_channel = self.get_channel(channel.src, channel.dest, channel.short_channel_id) + return_settlement_channel = self.get_channel(channel.dest, channel.src, channel.short_channel_id) + if settlement_channel.actual_liquidity > payment_amount: + # decrease channel balance in sending channel by amount + settlement_channel.actual_liquidity = settlement_channel.actual_liquidity - payment_amount + # increase channel balance in the other direction by amount + return_settlement_channel.actual_liquidity = return_settlement_channel.actual_liquidity + payment_amount + else: + raise Exception("""Channel liquidity on Channel {} is lower than payment amount. + \nPayment cannot settle.""".format(channel.short_channel_id)) + return 0 diff --git a/pickhardtpayments/Payment.py b/pickhardtpayments/Payment.py new file mode 100644 index 0000000..348c9ed --- /dev/null +++ b/pickhardtpayments/Payment.py @@ -0,0 +1,183 @@ +import time +from typing import List + +from .Attempt import Attempt, AttemptStatus + + +class Payment: + """ + Payment stores the information about an amount of sats to be delivered from source to destination. + + When sending an amount of sats from sender to receiver, a payment is usually split up and sent across + several paths, to increase the probability of being successfully delivered. + The PaymentClass holds all necessary information about a payment. + It also holds information that helps to calculate performance statistics about the payment. + + :param sender: sender address for the payment + :type sender: class:`str` + :param receiver: receiver address for the payment + :type receiver: class:`str` + :param total_amount: The total amount of sats to be delivered from source address to destination address. + :type total_amount: class:`int` + """ + + def __init__(self, sender, receiver, total_amount: int = 1): + """Constructor method + """ + self._successful = False + self._ppm = None + self._fee = None + self._start_time = time.time() + self._end_time = None + self._sender = sender + self._receiver = receiver + self._total_amount = total_amount + self._attempts = list() + + def __str__(self): + return "Payment with {} attempts to deliver {} sats from {} to {}".format(len(self._attempts), + self._total_amount, + self._sender[-8:], + self._receiver[-8:]) + + @property + def sender(self) -> str: + """Returns the address of the sender of the payment. + + :return: sender address for the payment + :rtype: str + """ + return self._sender + + @property + def receiver(self) -> str: + """Returns the address of the receiver of the payment. + + :return: receiver address for the payment + :rtype: str + """ + return self._receiver + + @property + def total_amount(self) -> int: + """Returns the amount to be sent with this payment. + + :return: The total amount of sats to be delivered from source address to destination address. + :rtype: int + """ + return self._total_amount + + @property + def start_time(self) -> float: + """Returns the time when Payment object was instantiated. + + :return: time in seconds from epoch to instantiation of Payment object. + :rtype: float + """ + return self._start_time + + @property + def end_time(self) -> float: + """Time when payment was finished, either by being aborted or by successful settlement + + Returns the time when all Attempts in Payment did settle or fail. + + :return: time in seconds from epoch to successful payment or failure of payment. + :rtype: float + """ + return self._end_time + + @end_time.setter + def end_time(self, timestamp): + """Set time when payment was finished, either by being aborted or by successful settlement + + Sets end_time time in seconds from epoch. Should be called when the Payment failed or when Payment + settled successfully. + + :param timestamp: time in seconds from epoch + :type timestamp: float + """ + self._end_time = timestamp + + @property + def attempts(self) -> List[Attempt]: + """Returns all onions that were built and are associated with this Payment. + + :return: A list of Attempts of this payment. + :rtype: list[Attempt] + """ + return self._attempts + + def add_attempts(self, attempts: List[Attempt]): + """Adds Attempts (onions) that have been made to settle the Payment to the Payment object. + + :param attempts: a list of attempts that belong to this Payment + :type: list[Attempt] + """ + self._attempts.extend(attempts) + + @property + def settlement_fees(self) -> float: + """Returns the fees that accrued for this payment. It's the sum of the routing fees of all settled onions. + + :return: fee in sats for successful attempts of Payment + :rtype: float + """ + settlement_fees = 0 + for attempt in self.filter_attempts(AttemptStatus.SETTLED): + settlement_fees += attempt.routing_fee + return settlement_fees + + @property + def arrived_fees(self) -> float: + """Returns the fees for all Attempts/onions for this payment, that arrived but have not yet been settled. + + It's the sum of the routing fees of all arrived attempts. + + :return: fee in sats for arrived attempts of Payment + :rtype: float + """ + planned_fees = 0 + for attempt in self.filter_attempts(AttemptStatus.ARRIVED): + planned_fees += attempt.routing_fee + return planned_fees + + @property + def ppm(self) -> float: + """Returns the fees that accrued for this payment. It's the sum of the routing fees of all settled onions. + + :return: fee in ppm for successful delivery and settlement of payment + :rtype: float + """ + return self.fee * 1000 / self.total_amount + + def filter_attempts(self, flag: AttemptStatus) -> List[Attempt]: + """Returns all onions with the given state. + + :param flag: the state of the attempts that should be filtered for + :type: Attempt.AttemptStatus + + :return: A list of successful Attempts of this Payment, which could be settled. + :rtype: list[Attempt] + """ + for attempt in self._attempts: + if attempt.status.value == flag.value: + yield attempt + + @property + def successful(self) -> bool: + """Returns True if the total_amount of the payment could be delivered successfully. + + :return: True if Payment settled successfully, else False. + :rtype: bool + """ + return self._successful + + @successful.setter + def successful(self, value): + """Sets flag if all inflight attempts of the payment could settle successfully. + + :param value: True if Payment settled successfully, else False. + :type: bool + """ + self._successful = value diff --git a/pickhardtpayments/SyncSimulatedPaymentSession.py b/pickhardtpayments/SyncSimulatedPaymentSession.py index 917e903..8eeab0d 100644 --- a/pickhardtpayments/SyncSimulatedPaymentSession.py +++ b/pickhardtpayments/SyncSimulatedPaymentSession.py @@ -1,20 +1,45 @@ +""" +SyncSimulatedPaymentSession.py +==================================== +The core module of the pickhardt payment project. +An example payment is executed and statistics are run. +""" + +import logging +import sys +from typing import List + +from .Attempt import Attempt, AttemptStatus +from .Payment import Payment from .UncertaintyNetwork import UncertaintyNetwork from .OracleLightningNetwork import OracleLightningNetwork from ortools.graph import pywrapgraph - -from typing import List import time import networkx as nx - DEFAULT_BASE_THRESHOLD = 0 -class SyncSimulatedPaymentSession(): +def set_logger(): + # Set Logger + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(asctime)s.%(msecs)03d | %(levelname)s | %(message)s', datefmt='%H:%M:%S') + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.setLevel(logging.DEBUG) + stdout_handler.setFormatter(formatter) + file_handler = logging.FileHandler('pickhardt_pay.log') + file_handler.setLevel(logging.DEBUG) + file_handler.setFormatter(formatter) + logger.addHandler(file_handler) + logger.addHandler(stdout_handler) + + +class SyncSimulatedPaymentSession: """ - A PaymentSesssion is used to create the min cost flow problem from the UncertaintyNetwork + A PaymentSession is used to create the min cost flow problem from the UncertaintyNetwork This happens by adding several parallel arcs coming from the piece wise linearization of the UncertaintyChannel to the min_cost_flow object. @@ -37,8 +62,7 @@ def _prepare_integer_indices_for_nodes(self): """ necessary for the OR-lib by google and the min cost flow solver - - let's initialize the look up tables for node_ids to integers from [0,...,#number of nodes] + let's initialize the look-up tables for node_ids to integers from [0,...,#number of nodes] this is necessary because of the API of the Google Operations Research min cost flow solver """ self._mcf_id = {} @@ -47,14 +71,16 @@ def _prepare_integer_indices_for_nodes(self): self._mcf_id[node_id] = k self._node_key[k] = node_id - def _prepare_mcf_solver(self, src, dest, amt: int = 1, mu: int = 100_000_000, base_fee: int = DEFAULT_BASE_THRESHOLD): + def _prepare_mcf_solver(self, src, dest, amt: int = 1, mu: int = 100_000_000, + base_fee: int = DEFAULT_BASE_THRESHOLD): """ computes the uncertainty network given our prior belief and prepares the min cost flow solver - This function can define a value for \mu to control how heavily we combine the uncertainty cost and fees - Also the function supports only taking channels into account that don't charge a base_fee higher or equal to `base` + This function can define a value for mu to control how heavily we combine the uncertainty cost and fees Also + the function supports only taking channels into account that don't charge a base_fee higher or equal to `base` - returns the instantiated min_cost_flow object from the google OR-lib that contains the piecewise linearized problem + returns the instantiated min_cost_flow object from the Google OR-lib that contains the piecewise linearized + problem """ self._min_cost_flow = pywrapgraph.SimpleMinCostFlow() self._arc_to_channel = {} @@ -64,7 +90,7 @@ def _prepare_mcf_solver(self, src, dest, amt: int = 1, mu: int = 100_000_000, ba if channel.base_fee > base_fee: continue # FIXME: Remove Magic Number for pruning - # Prune channels away thay have too low success probability! This is a huge runtime boost + # Prune channels away that have too low success probability! This is a huge runtime boost # However the pruning would be much better to work on quantiles of normalized cost # So as soon as we have better Scaling, Centralization and feature engineering we can # probably have a more focused pruning @@ -101,20 +127,20 @@ def _next_hop(self, path): The path is a list of node ids. Each call returns a tuple src, dest of an edge in the path """ for i in range(1, len(path)): - src = path[i-1] + src = path[i - 1] dest = path[i] - yield (src, dest) + yield src, dest def _make_channel_path(self, G: nx.MultiDiGraph, path: List[str]): """ - network x returns a path as a list of node_ids. However we need a list of `UncertaintyChannels` - Since the graph has parallel edges it is quite some work to get the actual channels that the + network x returns a path as a list of node_ids. However, we need a list of `UncertaintyChannels` + Since the graph has parallel edges it is quite some work to get the actual channels that the min cost flow solver produced """ channel_path = [] - bottleneck = 2**63 + bottleneck = 2 ** 63 for src, dest in self._next_hop(path): - w = 2**63 + w = 2 ** 63 c = None flow = 0 for sid in G[src][dest].keys(): @@ -129,9 +155,10 @@ def _make_channel_path(self, G: nx.MultiDiGraph, path: List[str]): return channel_path, bottleneck - def _disect_flow_to_paths(self, s, d): + def _dissect_flow_to_paths(self, s, d): """ - A standard algorithm to disect a flow into several paths. + A standard algorithm to dissect a flow into several paths. + FIXME: Note that this dissection while accurate is probably not optimal in practice. As noted in our Probabilistic payment delivery paper the payment process is a bernoulli trial @@ -139,9 +166,7 @@ def _disect_flow_to_paths(self, s, d): progress but this is a mere conjecture at this point. I expect quite a bit of research will be necessary to resolve this issue. """ - total_flow = {} - - # first collect all linearized edges which are assigned a non zero flow put them into a networkx graph + # first collect all linearized edges which are assigned a non-zero flow put them into a networkx graph G = nx.MultiDiGraph() for i in range(self._min_cost_flow.NumArcs()): flow = self._min_cost_flow.Flow(i) # *QUANTIZATION @@ -154,22 +179,21 @@ def _disect_flow_to_paths(self, s, d): G[src][dest][channel.short_channel_id]["flow"] += flow else: # FIXME: cost is not reflecting exactly the piecewise linearization - # Probably not such a big issue as we just disect flow + # Probably not such a big issue as we just dissect flow G.add_edge(src, dest, key=channel.short_channel_id, flow=flow, channel=channel, weight=channel.combined_linearized_unit_cost()) used_flow = 1 - channel_paths = [] + attempts = [] # allocate flow to shortest / cheapest paths from src to dest as long as this is possible - # decrease flow along those edges. This is a standard mechanism to disect a flow int paths + # decrease flow along those edges. This is a standard mechanism to dissect a flow into paths while used_flow > 0: - path = None try: path = nx.shortest_path(G, s, d) except: break channel_path, used_flow = self._make_channel_path(G, path) - channel_paths.append((channel_path, used_flow)) + attempts.append(Attempt(channel_path, used_flow)) # reduce the flow from the selected path for pos, hop in enumerate(self._next_hop(path)): @@ -178,25 +202,28 @@ def _disect_flow_to_paths(self, s, d): G[src][dest][channel.short_channel_id]["flow"] -= used_flow if G[src][dest][channel.short_channel_id]["flow"] == 0: G.remove_edge(src, dest, key=channel.short_channel_id) - return channel_paths + return attempts - def _generate_candidate_paths(self, src, dest, amt, mu: int = 100_000_000, base: int = DEFAULT_BASE_THRESHOLD): + def _generate_candidate_paths(self, src, dest, amt: int, mu: int = 100_000_000, + base: int = DEFAULT_BASE_THRESHOLD): """ - computes the optimal payment split to deliver `amt` from `src` to `dest` and updates our belief about the liquidity + computes the optimal payment split to deliver `amt` from `src` to `dest` and updates our belief about the + liquidity This is one step within the payment loop. Returns the residual amount of the `amt` that could ne be delivered and the paid fees (on a per channel base not including fees for downstream fees) for the delivered amount - the function also prints some results an statistics about the paths of the flow to stdout. + the function also prints some results on statistics about the paths of the flow to stdout. """ + # initialisation of List of Attempts for this round. + attempts_in_round = List[Attempt] # First we prepare the min cost flow by getting arcs from the uncertainty network self._prepare_mcf_solver(src, dest, amt, mu, base) - start = time.time() - #print("solving mcf...") + # print("solving mcf...") status = self._min_cost_flow.Solve() if status != self._min_cost_flow.OPTIMAL: @@ -204,53 +231,56 @@ def _generate_candidate_paths(self, src, dest, amt, mu: int = 100_000_000, base: print(f'Status: {status}') exit(1) - paths = self._disect_flow_to_paths(src, dest) + attempts_in_round = self._dissect_flow_to_paths(src, dest) end = time.time() - return paths, end-start + return attempts_in_round, end - start - def _estimate_payment_statistics(self, paths): + def _estimate_payment_statistics(self, attempts): """ estimates the success probability of paths and computes fees (without paying downstream fees) @returns the statistics in the `payments` dictionary """ - # FIXME: Decide if an `Payments` or `Attempt` class shall be used - payments = {} # compute fees and probabilities of candidate paths for evaluation - for i, onion in enumerate(paths): - path, amount = onion - fee, probability = self._uncertainty_network.get_features_of_candidate_path( - path, amount) - payments[i] = { - "routing_fee": fee, "probability": probability, "path": path, "amount": amount} + for attempt in attempts: + attempt.routing_fee, attempt.probability = self._uncertainty_network.get_features_of_candidate_path( + attempt.path, attempt.amount) + # logging.debug("fee: {attempt.routing_fee} msat, p = {attempt.probability:.4%}, amount: {attempt.amount}") - # to correctly compute conditional probabilities of non disjoint paths in the same set of paths - self._uncertainty_network.allocate_amount_on_path(path, amount) + # to correctly compute conditional probabilities of non-disjoint paths in the same set of paths + self._uncertainty_network.allocate_amount_on_path(attempt.path, attempt.amount) # remove allocated amounts for all planned onions before doing actual attempts - for key, attempt in payments.items(): + for attempt in attempts: self._uncertainty_network.allocate_amount_on_path( - attempt["path"], -attempt["amount"]) + attempt.path, -attempt.amount) - return payments - def _attempt_payments(self, payments): + def _attempt_payments(self, attempts: List[Attempt]): """ we attempt all planned payments and test the success against the oracle in particular this method changes - depending on the outcome of each payment - our belief about the uncertainty - in the UncertaintyNetwork + in the UncertaintyNetwork. + successful onions are collected to be transacted on the OracleNetwork if complete payment can be delivered """ # test actual payment attempts - for key, attempt in payments.items(): + for attempt in attempts: success, erring_channel = self._oracle.send_onion( - attempt["path"], attempt["amount"]) - payments[key]["success"] = success - payments[key]["erring_channel"] = erring_channel + attempt.path, attempt.amount) if success: + # TODO: let this happen in Payment class? Or in Attempt class - with status change as settlement + attempt.status = AttemptStatus.ARRIVED + # handling amounts on path happens in Attempt Class. self._uncertainty_network.allocate_amount_on_path( - attempt["path"], attempt["amount"]) + attempt.path, attempt.amount) + + + # unnecessary, because information is in attempt (Status INFLIGHT) + # settled_onions.append(payments[key]) + else: + attempt.status = AttemptStatus.FAILED - def _evaluate_attempts(self, payments): + def _evaluate_attempts(self, payment: Payment): """ helper function to collect statistics about attempts and print them @@ -259,63 +289,47 @@ def _evaluate_attempts(self, payments): total_fees = 0 paid_fees = 0 residual_amt = 0 - number_failed_paths = 0 expected_sats_to_deliver = 0 amt = 0 - print("\nStatistics about {} candidate onions:\n".format(len(payments))) - - has_failed_attempt = False + arrived_attempts = [] + failed_attempts = [] + print("\nStatistics about {} candidate onions:\n".format(len(payment.attempts))) print("successful attempts:") print("--------------------") - for attempt in payments.values(): - success = attempt["success"] - if success == False: - has_failed_attempt = True - continue - fee = attempt["routing_fee"] / 1000. - probability = attempt["probability"] - path = attempt["path"] - amount = attempt["amount"] - amt += amount - total_fees += fee - expected_sats_to_deliver += probability * amount + for arrived_attempt in payment.filter_attempts(AttemptStatus.ARRIVED): + amt += arrived_attempt.amount + total_fees += arrived_attempt.routing_fee / 1000. + expected_sats_to_deliver += arrived_attempt.probability * arrived_attempt.amount + print(" p = {:6.2f}% amt: {:9} sats hops: {} ppm: {:5}".format( + arrived_attempt.probability * 100, arrived_attempt.amount, len(arrived_attempt.path), + int(arrived_attempt.routing_fee * 1000 / arrived_attempt.amount))) + paid_fees += arrived_attempt.routing_fee + + print("\nfailed attempts:") + print("----------------") + for failed_attempt in payment.filter_attempts(AttemptStatus.FAILED): + amt += failed_attempt.amount + total_fees += failed_attempt.routing_fee / 1000. + expected_sats_to_deliver += failed_attempt.probability * failed_attempt.amount print(" p = {:6.2f}% amt: {:9} sats hops: {} ppm: {:5}".format( - probability*100, amount, len(path), int(fee*1000_000/amount))) - paid_fees += fee - - if has_failed_attempt: - print("\nfailed attempts:") - print("----------------") - for attempt in payments.values(): - success = attempt["success"] - if success: - continue - fee = attempt["routing_fee"] / 1000. - probability = attempt["probability"] - path = attempt["path"] - amount = attempt["amount"] - amt += amount - total_fees += fee - expected_sats_to_deliver += probability * amount - print(" p = {:6.2f}% amt: {:9} sats hops: {} ppm: {:5} ".format( - probability*100, amount, len(path), int(fee*1000_000/amount))) - number_failed_paths += 1 - residual_amt += amount + failed_attempt.probability * 100, failed_attempt.amount, len(failed_attempt.path), + int(failed_attempt.routing_fee * 1000 / failed_attempt.amount))) + residual_amt += failed_attempt.amount print("\nAttempt Summary:") print("=================") - print("\nTried to deliver {:10} sats".format(amt)) - fraction = expected_sats_to_deliver*100./amt + print("\nTried to deliver \t{:10} sats".format(amt)) + fraction = expected_sats_to_deliver * 100. / amt print("expected to deliver {:10} sats \t({:4.2f}%)".format( int(expected_sats_to_deliver), fraction)) - fraction = (amt-residual_amt)*100./(amt) - print("actually deliverd {:10} sats \t({:4.2f}%)".format( - amt-residual_amt, fraction)) - print("deviation: {:4.2f}".format( - (amt-residual_amt)/(expected_sats_to_deliver+1))) + fraction = (amt - residual_amt) * 100. / (amt) + print("actually delivered {:10} sats \t({:4.2f}%)".format( + amt - residual_amt, fraction)) + print("deviation: \t\t{:4.2f}".format( + (amt - residual_amt) / (expected_sats_to_deliver + 1))) print("planned_fee: {:8.3f} sat".format(total_fees)) print("paid fees: {:8.3f} sat".format(paid_fees)) - return residual_amt, paid_fees, len(payments), number_failed_paths + return residual_amt, paid_fees, len(payment.attempts), len(failed_attempts) def forget_information(self): """ @@ -330,60 +344,81 @@ def activate_network_wide_uncertainty_reduction(self, n): self._uncertainty_network.activate_network_wide_uncertainty_reduction( n, self._oracle) - def pickhardt_pay(self, src, dest, amt, mu=1, base=0): + def pickhardt_pay(self, src, dest, amt, mu=1, base=DEFAULT_BASE_THRESHOLD): """ conduct one experiment! might need to call oracle.reset_uncertainty_network() first - I could not put it here as some experiments require sharing of liqudity information + I could not put it here as some experiments require sharing of liquidity information """ + + set_logger() + logging.info('*** new pickhardt payment ***') + + # Setup entropy_start = self._uncertainty_network.entropy() - start = time.time() - full_amt = amt cnt = 0 total_fees = 0 - number_number_of_onions = 0 total_number_failed_paths = 0 + # Initialise Payment + # currently with underscore to not mix up with existing variable 'payment' + payment = Payment(src, dest, amt) + # This is the main payment loop. It is currently blocking and synchronous but may be - # implemented in a concurrent way. Also we stop after 10 rounds which is pretty arbitrary - # a better stop criteria would be if we compute infeasable flows or if the probabilities - # are to low or residual amounts decrease to slowly + # implemented in a concurrent way. Also, we stop after 10 rounds which is pretty arbitrary + # a better stop criteria would be if we compute infeasible flows or if the probabilities + # are too low or residual amounts decrease to slowly while amt > 0 and cnt < 10: - print("Round number: ", cnt+1) + print("Round number: ", cnt + 1) print("Try to deliver", amt, "satoshi:") - # transfer to a min cost flow problem and rund the solver - paths, runtime = self._generate_candidate_paths( - src, dest, amt, mu, base) - - # compute some statistics about candidate paths - payments = self._estimate_payment_statistics(paths) + sub_payment = Payment(payment.sender, payment.receiver, amt) + # transfer to a min cost flow problem and run the solver + # paths is the lists of channels, runtime the time it took to calculate all candidates in this round + paths, runtime = self._generate_candidate_paths(payment.sender, payment.receiver, amt, mu, base) + sub_payment.add_attempts(paths) - # matke attempts and update our information about the UncertaintyNetwork - self._attempt_payments(payments) + # make attempts, try to send onion and register if success or not + # update our information about the UncertaintyNetwork + self._attempt_payments(sub_payment.attempts) # run some simple statistics and depict them amt, paid_fees, num_paths, number_failed_paths = self._evaluate_attempts( - payments) + sub_payment) + print("Runtime of flow computation: {:4.2f} sec ".format(runtime)) print("\n================================================================\n") - number_number_of_onions += num_paths total_number_failed_paths += number_failed_paths total_fees += paid_fees cnt += 1 - end = time.time() + + # add attempts of sub_payment to payment + payment.add_attempts(sub_payment.attempts) + + # When residual amount is 0 / enough successful onions have been found, then settle payment. Else drop onions. + if amt == 0: + for onion in payment.filter_attempts(AttemptStatus.ARRIVED): + try: + self._oracle.settle_payment(onion.path, onion.amount) + onion.status = AttemptStatus.SETTLED + except Exception as e: + print(e) + return -1 + payment.successful = True + payment.end_time = time.time() + entropy_end = self._uncertainty_network.entropy() print("SUMMARY:") print("========") - print("Rounds of mcf-computations: ", cnt) - print("Number of onions sent: ", number_number_of_onions) - print("Number of failed onions: ", total_number_failed_paths) + print("Rounds of mcf-computations:\t", cnt) + print("Number of attempts made:\t", len(payment.attempts)) + print("Number of failed attempts:\t", len(list(payment.filter_attempts(AttemptStatus.FAILED)))) print("Failure rate: {:4.2f}% ".format( - total_number_failed_paths*100./number_number_of_onions)) - print("total runtime (including inefficient memory managment): {:4.3f} sec".format( - end-start)) - print("Learnt entropy: {:5.2f} bits".format(entropy_start-entropy_end)) - print("Fees for successfull delivery: {:8.3f} sat --> {} ppm".format( - total_fees, int(total_fees*1000*1000/full_amt))) + len(list(payment.filter_attempts(AttemptStatus.FAILED))) * 100. / len(payment.attempts))) + print("total Payment lifetime (including inefficient memory management): {:4.3f} sec".format( + payment.end_time - payment.start_time)) + print("Learnt entropy: {:5.2f} bits".format(entropy_start - entropy_end)) + print("fee for settlement of delivery: {:8.3f} sat --> {} ppm".format( + payment.settlement_fees/1000, int(payment.settlement_fees * 1000 / payment.total_amount))) print("used mu:", mu) diff --git a/pickhardtpayments/UncertaintyChannel.py b/pickhardtpayments/UncertaintyChannel.py index 1c75172..890fb03 100644 --- a/pickhardtpayments/UncertaintyChannel.py +++ b/pickhardtpayments/UncertaintyChannel.py @@ -21,11 +21,11 @@ class UncertaintyChannel(Channel): Most importantly the class stores our belief about the liquidity information of a channel. This is done by reducing the uncertainty interval from [0,`capacity`] to [`min_liquidity`, `max_liquidity`]. - Additionally we need to know how many sats we currently have allocated via outstanding onions + Additionally, we need to know how many sats we currently have allocated via outstanding onions to the channel which is stored in `inflight`. The most important API call is the `get_piecewise_linearized_costs` function that computes the - pieceweise linearized cost for a channel rising from uncertainty as well as routing fees. + piecewise linearized cost for a channel rising from uncertainty as well as routing fees. """ TOTAL_NUMBER_OF_SATS = 21_000_000 * 100_000_000 @@ -81,7 +81,7 @@ def conditional_capacity(self, respect_inflight=True): def allocate_amount(self, amt: int): """ - assign or remove ammount that is assigned to be `in_flight`. + assign or remove amount that is assigned to be `in_flight`. """ self.in_flight += amt if self.in_flight < 0: @@ -263,7 +263,7 @@ def update_knowledge(self, amt: int, success_of_probe): """ updates our knowledge about the channel if we tried to probe it for amount `amt` - This API works ony if we have an Oracle that allows to ask the actual liquidity of a channel + This API works only if we have an Oracle that allows to ask the actual liquidity of a channel In mainnet Lightning our oracle will not work on a per_channel level. This will change the data flow. Here for simplicity of the simulation we make use of the Oracle on a per channel level """ diff --git a/pickhardtpayments/UncertaintyNetwork.py b/pickhardtpayments/UncertaintyNetwork.py index 89c7971..dea4663 100644 --- a/pickhardtpayments/UncertaintyNetwork.py +++ b/pickhardtpayments/UncertaintyNetwork.py @@ -2,6 +2,7 @@ from .UncertaintyChannel import UncertaintyChannel from .OracleLightningNetwork import OracleLightningNetwork + from typing import List import networkx as nx @@ -10,7 +11,7 @@ class UncertaintyNetwork(ChannelGraph): """ - The UncertaintayNetwork is the main data structure to store our belief about the + The UncertaintyNetwork is the main data structure to store our belief about the Liquidity in the channels of the ChannelGraph. Most of its functionality comes from the UncertaintyChannel. Most notably the ability @@ -40,17 +41,6 @@ def entropy(self): """ return sum(channel.entropy() for src, dest, channel in self.network.edges(data="channel")) - def get_features_of_candidate_path(self, path: List[UncertaintyChannel], amt: int) -> (float, float): - """ - returns the routing fees and probability of a candidate path - """ - probability = 1 - routing_fees = 0 - for channel in path: - routing_fees += channel.routing_cost_msat(amt) - probability *= channel.success_probability(amt) - return routing_fees, probability - def allocate_amount_on_path(self, path: List[UncertaintyChannel], amt: int): """ allocates `amt` to all channels of the path of `UncertaintyChannels` @@ -70,7 +60,7 @@ def activate_network_wide_uncertainty_reduction(self, n, oracle: OracleLightning With the help of an `OracleLightningNetwork` probes all channels `n` times to reduce uncertainty. While one can do this on mainnet by probing we can do this very quickly in simulation - at virtually no cost. Thus this API call needs to be taken with caution when using a different + at virtually no cost. Thus, this API call needs to be taken with caution when using a different oracle. """ for src, dest, channel in self.network.edges(data="channel"): @@ -78,13 +68,13 @@ def activate_network_wide_uncertainty_reduction(self, n, oracle: OracleLightning # FIXME: refactor to new code base. The following call will break! def activate_foaf_uncertainty_reduction(self, src, dest): - ego_netwok = set() + ego_network = set() foaf_network = set() out_set = set() edges = self.__channel_graph.out_edges(self.__node_key_to_id[src]) for edge in edges: - ego_netwok.add("{}x{}".format(edge[0], edge[1])) + ego_network.add("{}x{}".format(edge[0], edge[1])) out_set.add(edge[1]) for node in out_set: @@ -95,7 +85,7 @@ def activate_foaf_uncertainty_reduction(self, src, dest): in_set = set() edges = self.__channel_graph.in_edges(self.__node_key_to_id[dest]) for edge in edges: - ego_netwok.add("{}x{}".format(edge[0], edge[1])) + ego_network.add("{}x{}".format(edge[0], edge[1])) in_set.add(edge[0]) for node in in_set: @@ -103,12 +93,12 @@ def activate_foaf_uncertainty_reduction(self, src, dest): for edge in edges: foaf_network.add("{}x{}".format(edge[0], edge[1])) - # print(len(ego_netwok)) + # print(len(ego_network)) for k, arc in self.__arcs.items(): # print(k) vals = k.split("x") key = "{}x{}".format(vals[0], vals[1]) - if key in ego_netwok: + if key in ego_network: l = arc.get_actual_liquidity() arc.update_knowledge(l-1) arc.update_knowledge(l+1) @@ -119,6 +109,6 @@ def activate_foaf_uncertainty_reduction(self, src, dest): l = arc.get_actual_liquidity() arc.update_knowledge(l-1) arc.update_knowledge(l+1) - #print(key, arc.entropy()) - print("channels with full knowlege: ", len(ego_netwok)) + # print(key, arc.entropy()) + print("channels with full knowledge: ", len(ego_network)) print("channels with 2 Bits of less entropy: ", len(foaf_network)) diff --git a/pickhardtpayments/__init__.py b/pickhardtpayments/__init__.py index 5e2c60f..95453c8 100644 --- a/pickhardtpayments/__init__.py +++ b/pickhardtpayments/__init__.py @@ -17,4 +17,4 @@ "OracleLightningNetwork", "ChannelGraph", "SyncSimulatedPaymentSession" -] +] \ No newline at end of file