|
|
@@ -1,152 +1,224 @@
|
|
|
-import asyncio
|
|
|
+from __future__ import annotations
|
|
|
+import os
|
|
|
import heapq
|
|
|
-from typing import Optional, List, Tuple, Dict, Iterator
|
|
|
-from rpcudp.protocol import RPCProtocol
|
|
|
+import asyncio
|
|
|
+import logging
|
|
|
+import urllib.parse
|
|
|
+from typing import Optional, List, Tuple, Dict, Iterator, Any, Sequence, Union
|
|
|
+from warnings import warn
|
|
|
+from .routing import RoutingTable, DHTID, BinaryDHTValue, DHTExpiration, get_dht_time
|
|
|
+from ..utils import Endpoint, compile_grpc
|
|
|
+import grpc, grpc.experimental.aio
|
|
|
|
|
|
-from .routing import RoutingTable, DHTID, DHTValue, DHTExpiration, BinaryDHTID, get_dht_time
|
|
|
-from ..utils import Endpoint
|
|
|
+with open(os.path.join(os.path.dirname(__file__), 'dht.proto'), 'r') as f_proto:
|
|
|
+ dht_pb2, dht_grpc = compile_grpc(f_proto.read())
|
|
|
|
|
|
|
|
|
-class KademliaProtocol(RPCProtocol):
|
|
|
- """
|
|
|
- A protocol that allows DHT nodes to request keys/neighbors from other DHT nodes.
|
|
|
- As a side-effect, KademliaProtocol also maintains a routing table as described in
|
|
|
- https://pdos.csail.mit.edu/~petar/papers/maymounkov-kademlia-lncs.pdf
|
|
|
+class DHTProtocol(dht_grpc.DHTServicer):
|
|
|
+ node_id: DHTID; port: int; bucket_size: int; num_replicas: int; wait_timeout: float; node_info: dht_pb2.NodeInfo
|
|
|
+ channel_options: Optional[Sequence[Tuple[str, Any]]]; server: grpc.experimental.aio.Server
|
|
|
+ storage: LocalStorage; cache: LocalStorage; routing_table: RoutingTable
|
|
|
|
|
|
- See DHTNode (node.py) for a more detailed description.
|
|
|
+ @classmethod
|
|
|
+ async def create(cls, node_id: DHTID, bucket_size: int, depth_modulo: int, num_replicas: int, wait_timeout: float,
|
|
|
+ cache_size: Optional[int] = None, listen=True, listen_on='0.0.0.0:*',
|
|
|
+ channel_options: Optional[Sequence[Tuple[str, Any]]] = None, **kwargs) -> DHTProtocol:
|
|
|
+ """
|
|
|
+ A protocol that allows DHT nodes to request keys/neighbors from other DHT nodes.
|
|
|
+ As a side-effect, DHTProtocol also maintains a routing table as described in
|
|
|
+ https://pdos.csail.mit.edu/~petar/papers/maymounkov-kademlia-lncs.pdf
|
|
|
|
|
|
- :note: the rpc_* methods defined in this class will be automatically exposed to other DHT nodes,
|
|
|
- for instance, def rpc_ping can be called as protocol.call_ping(addr, dht_id) from a remote machine
|
|
|
- Only the call_* methods are meant to be called publicly, e.g. from DHTNode
|
|
|
- Read more: https://github.com/bmuller/rpcudp/tree/master/rpcudp
|
|
|
- """
|
|
|
+ See DHTNode (node.py) for a more detailed description.
|
|
|
|
|
|
- def __init__(self, node_id: DHTID, bucket_size: int, depth_modulo: int, wait_timeout: float,
|
|
|
- max_concurrent_rpc: int, num_replicas: Optional[int] = None, cache_size: Optional[int] = None):
|
|
|
- super().__init__(wait_timeout)
|
|
|
- self.node_id, self.bucket_size, self.num_replicas = node_id, bucket_size, num_replicas or bucket_size
|
|
|
- self.rpc_semaphore = asyncio.BoundedSemaphore(value=max_concurrent_rpc)
|
|
|
+ :note: the rpc_* methods defined in this class will be automatically exposed to other DHT nodes,
|
|
|
+ for instance, def rpc_ping can be called as protocol.call_ping(addr, dht_id) from a remote machine
|
|
|
+ Only the call_* methods are meant to be called publicly, e.g. from DHTNode
|
|
|
+ Read more: https://github.com/bmuller/rpcudp/tree/master/rpcudp
|
|
|
+ """
|
|
|
+ self = cls(_initialized_with_create=True)
|
|
|
+ self.node_id, self.bucket_size, self.num_replicas = node_id, bucket_size, num_replicas
|
|
|
+ self.wait_timeout, self.channel_options = wait_timeout, channel_options
|
|
|
+ self.storage, self.cache = LocalStorage(), LocalStorage(maxsize=cache_size)
|
|
|
self.routing_table = RoutingTable(node_id, bucket_size, depth_modulo)
|
|
|
- self.storage = LocalStorage()
|
|
|
- self.cache = LocalStorage(maxsize=cache_size)
|
|
|
-
|
|
|
- def rpc_ping(self, sender: Endpoint, sender_id_bytes: BinaryDHTID) -> BinaryDHTID:
|
|
|
- """ Some dht node wants us to add it to our routing table. """
|
|
|
- asyncio.ensure_future(self.update_routing_table(DHTID.from_bytes(sender_id_bytes), sender))
|
|
|
- return bytes(self.node_id)
|
|
|
-
|
|
|
- async def call_ping(self, recipient: Endpoint) -> Optional[DHTID]:
|
|
|
- """ Get recipient's node id and add him to the routing table. If recipient doesn't respond, return None """
|
|
|
- async with self.rpc_semaphore:
|
|
|
- responded, response = await self.ping(recipient, bytes(self.node_id))
|
|
|
- recipient_node_id = DHTID.from_bytes(response) if responded else None
|
|
|
- asyncio.ensure_future(self.update_routing_table(recipient_node_id, recipient, responded=responded))
|
|
|
- return recipient_node_id
|
|
|
-
|
|
|
- def rpc_store(self, sender: Endpoint, sender_id_bytes: BinaryDHTID, key_bytes: BinaryDHTID,
|
|
|
- value: DHTValue, expiration_time: DHTExpiration, in_cache: bool) -> Tuple[bool, BinaryDHTID]:
|
|
|
- """ Some node wants us to store this (key, value) pair """
|
|
|
- asyncio.ensure_future(self.update_routing_table(DHTID.from_bytes(sender_id_bytes), sender))
|
|
|
- if in_cache:
|
|
|
- store_accepted = self.cache.store(DHTID.from_bytes(key_bytes), value, expiration_time)
|
|
|
+
|
|
|
+ if listen: # set up server to process incoming rpc requests
|
|
|
+ grpc.experimental.aio.init_grpc_aio()
|
|
|
+ self.server = grpc.experimental.aio.server(**kwargs)
|
|
|
+ dht_grpc.add_DHTServicer_to_server(self, self.server)
|
|
|
+
|
|
|
+ found_port = self.server.add_insecure_port(listen_on)
|
|
|
+ assert found_port != 0, f"Failed to listen to {listen_on}"
|
|
|
+ self.node_info = dht_pb2.NodeInfo(node_id=node_id.to_bytes(), rpc_port=found_port)
|
|
|
+ self.port = found_port
|
|
|
+ await self.server.start()
|
|
|
+ else: # not listening to incoming requests, client-only mode
|
|
|
+ # note: use empty node_info so peers wont add you to their routing tables
|
|
|
+ self.node_info, self.server, self.port = dht_pb2.NodeInfo(), None, None
|
|
|
+ if listen_on != '0.0.0.0:*' or len(kwargs) != 0:
|
|
|
+ warn(f"DHTProtocol has no server (due to listen=False), listen_on"
|
|
|
+ f"and kwargs have no effect (unused kwargs: {kwargs})")
|
|
|
+ return self
|
|
|
+
|
|
|
+ def __init__(self, *, _initialized_with_create=False):
|
|
|
+ """ Internal init method. Please use DHTProtocol.create coroutine to spawn new protocol instances """
|
|
|
+ assert _initialized_with_create, " Please use DHTProtocol.create coroutine to spawn new protocol instances "
|
|
|
+ super().__init__()
|
|
|
+
|
|
|
+ async def shutdown(self, timeout=None):
|
|
|
+ """ Process existing requests, close all connections and stop the server """
|
|
|
+ if self.server:
|
|
|
+ await self.server.stop(timeout)
|
|
|
else:
|
|
|
- store_accepted = self.storage.store(DHTID.from_bytes(key_bytes), value, expiration_time)
|
|
|
- return store_accepted, bytes(self.node_id)
|
|
|
+ warn("DHTProtocol has no server (due to listen=False), it doesn't need to be shut down")
|
|
|
|
|
|
- async def call_store(self, recipient: Endpoint, key: DHTID, value: DHTValue,
|
|
|
- expiration_time: DHTExpiration, in_cache: bool = False) -> Optional[bool]:
|
|
|
- """
|
|
|
- Ask a recipient to store (key, value) pair until expiration time or update their older value
|
|
|
+ def _get(self, peer: Endpoint) -> dht_grpc.DHTStub:
|
|
|
+ """ get a DHTStub that sends requests to a given peer """
|
|
|
+ channel = grpc.experimental.aio.insecure_channel(peer, options=self.channel_options)
|
|
|
+ return dht_grpc.DHTStub(channel)
|
|
|
|
|
|
- :returns: True if value was accepted, False if it was rejected (recipient has newer value), None if no response
|
|
|
+ async def call_ping(self, peer: Endpoint) -> Optional[DHTID]:
|
|
|
"""
|
|
|
- async with self.rpc_semaphore:
|
|
|
- responded, response = await self.store(recipient, bytes(self.node_id), bytes(key),
|
|
|
- value, expiration_time, in_cache)
|
|
|
- if responded:
|
|
|
- store_accepted, recipient_node_id = response[0], DHTID.from_bytes(response[1])
|
|
|
- asyncio.ensure_future(self.update_routing_table(recipient_node_id, recipient, responded=responded))
|
|
|
- return store_accepted
|
|
|
- return None
|
|
|
-
|
|
|
- def rpc_find_node(self, sender: Endpoint, sender_id_bytes: BinaryDHTID,
|
|
|
- query_id_bytes: BinaryDHTID) -> Tuple[List[Tuple[BinaryDHTID, Endpoint]], BinaryDHTID]:
|
|
|
+ Get peer's node id and add him to the routing table. If peer doesn't respond, return None
|
|
|
+ :param peer: string network address, e.g. 123.123.123.123:1337 or [2a21:6с8:b192:2105]:8888
|
|
|
+ :note: if DHTProtocol was created with listen=True, also request peer to add you to his routing table
|
|
|
+
|
|
|
+ :return: node's DHTID, if peer responded and decided to send his node_id
|
|
|
"""
|
|
|
- Someone wants to find :key_node: in the DHT. Give him k nearest neighbors from our routing table
|
|
|
+ try:
|
|
|
+ peer_info = await self._get(peer).rpc_ping(self.node_info, timeout=self.wait_timeout)
|
|
|
+ except grpc.experimental.aio.AioRpcError as error:
|
|
|
+ logging.info(f"DHTProtocol failed to ping {peer}: {error.code()}")
|
|
|
+ peer_info = None
|
|
|
+ responded = bool(peer_info and peer_info.node_id)
|
|
|
+ peer_id = DHTID.from_bytes(peer_info.node_id) if responded else None
|
|
|
+ asyncio.create_task(self.update_routing_table(peer_id, peer, responded=responded))
|
|
|
+ return peer_id
|
|
|
+
|
|
|
+ async def rpc_ping(self, peer_info: dht_pb2.NodeInfo, context: grpc.ServicerContext):
|
|
|
+ """ Some node wants us to add it to our routing table. """
|
|
|
+ if peer_info.node_id and peer_info.rpc_port:
|
|
|
+ sender_id = DHTID.from_bytes(peer_info.node_id)
|
|
|
+ peer_url = urllib.parse.urlparse(context.peer())
|
|
|
+ address = peer_url.path[:peer_url.path.rindex(':')]
|
|
|
+ asyncio.create_task(self.update_routing_table(sender_id, f"{address}:{peer_info.rpc_port}"))
|
|
|
+ return self.node_info
|
|
|
|
|
|
- :returns: a list of pairs (node_id, address) of :bucket_size: nearest to key_node according to XOR distance,
|
|
|
- also returns our own node id for routing table maintenance
|
|
|
+ async def call_store(self, peer: Endpoint, keys: Sequence[DHTID], values: Sequence[BinaryDHTValue],
|
|
|
+ expirations: Union[DHTExpiration, Sequence[DHTExpiration]],
|
|
|
+ in_cache: Optional[Union[bool, Sequence[bool]]] = None) -> Sequence[bool]:
|
|
|
"""
|
|
|
- query_id, sender_id = DHTID.from_bytes(query_id_bytes), DHTID.from_bytes(sender_id_bytes)
|
|
|
- asyncio.ensure_future(self.update_routing_table(sender_id, sender))
|
|
|
- peer_ids_and_addr = self.routing_table.get_nearest_neighbors(query_id, k=self.bucket_size, exclude=sender_id)
|
|
|
- return [(bytes(peer_id), peer_addr) for peer_id, peer_addr in peer_ids_and_addr], bytes(self.node_id)
|
|
|
+ Ask a recipient to store several (key, value : expiration) items or update their older value
|
|
|
|
|
|
- async def call_find_node(self, recipient: Endpoint, query_id: DHTID) -> Dict[DHTID, Endpoint]:
|
|
|
+ :param peer: request this peer to store the data
|
|
|
+ :param keys: a list of N keys digested by DHTID.generate(source=some_dict_key)
|
|
|
+ :param values: a list of N serialized values (bytes) for each respective key
|
|
|
+ :param expirations: a list of N expiration timestamps for each respective key-value pair (see get_dht_time())
|
|
|
+ :param in_cache: a list of booleans, True = store i-th key in cache, value = store i-th key locally
|
|
|
+ :note: the difference between storing normally and in cache is that normal storage is guaranteed to be stored
|
|
|
+ until expiration time (best-effort), whereas cached storage can be evicted early due to limited cache size
|
|
|
+
|
|
|
+ :return: list of [True / False] True = stored, False = failed (found newer value or no response)
|
|
|
+ if peer did not respond (e.g. due to timeout or congestion), returns None
|
|
|
"""
|
|
|
- Ask a recipient to give you nearest neighbors to key_node. If recipient knows key_node directly,
|
|
|
- it will be returned as first of the neighbors; if recipient does not respond, return empty dict.
|
|
|
+ in_cache = in_cache if in_cache is not None else [False] * len(keys) # default value (None)
|
|
|
+ in_cache = [in_cache] * len(keys) if isinstance(in_cache, bool) else in_cache # single bool
|
|
|
+ expirations = [expirations] * len(keys) if isinstance(expirations, DHTExpiration) else expirations
|
|
|
+ keys, values, expirations, in_cache = map(list, [keys, values, expirations, in_cache])
|
|
|
+ assert len(keys) == len(values) == len(expirations) == len(in_cache), "Data is not aligned"
|
|
|
+ store_request = dht_pb2.StoreRequest(keys=list(map(DHTID.to_bytes, keys)), values=values,
|
|
|
+ expiration=expirations, in_cache=in_cache, peer=self.node_info)
|
|
|
+ try:
|
|
|
+ response = await self._get(peer).rpc_store(store_request, timeout=self.wait_timeout)
|
|
|
+ if response.peer and response.peer.node_id:
|
|
|
+ peer_id = DHTID.from_bytes(response.peer.node_id)
|
|
|
+ asyncio.create_task(self.update_routing_table(peer_id, peer, responded=True))
|
|
|
+ return response.store_ok
|
|
|
+ except grpc.experimental.aio.AioRpcError as error:
|
|
|
+ logging.info(f"DHTProtocol failed to store at {peer}: {error.code()}")
|
|
|
+ asyncio.create_task(self.update_routing_table(self.routing_table.get_id(peer), peer, responded=False))
|
|
|
+ return [False] * len(keys)
|
|
|
+
|
|
|
+ async def rpc_store(self, request: dht_pb2.StoreRequest, context: grpc.ServicerContext) -> dht_pb2.StoreResponse:
|
|
|
+ """ Some node wants us to store this (key, value) pair """
|
|
|
+ if request.peer: # if requested, add peer to the routing table
|
|
|
+ asyncio.create_task(self.rpc_ping(request.peer, context))
|
|
|
+ assert len(request.keys) == len(request.values) == len(request.expiration) == len(request.in_cache)
|
|
|
+ response = dht_pb2.StoreResponse(store_ok=[], peer=self.node_info)
|
|
|
+ for key_bytes, value_bytes, expiration_time, in_cache in zip(
|
|
|
+ request.keys, request.values, request.expiration, request.in_cache):
|
|
|
+ local_memory = self.cache if in_cache else self.storage
|
|
|
+ response.store_ok.append(local_memory.store(DHTID.from_bytes(key_bytes), value_bytes, expiration_time))
|
|
|
+ return response
|
|
|
|
|
|
- :returns: a dicitionary[node id => address] as per Section 2.3 of the paper
|
|
|
+ async def call_find(self, peer: Endpoint, keys: Sequence[DHTID]) -> \
|
|
|
+ Optional[Dict[DHTID, Tuple[Optional[BinaryDHTValue], Optional[DHTExpiration], Dict[DHTID, Endpoint]]]]:
|
|
|
"""
|
|
|
- async with self.rpc_semaphore:
|
|
|
- responded, response = await self.find_node(recipient, bytes(self.node_id), bytes(query_id))
|
|
|
- if responded:
|
|
|
- peers = {DHTID.from_bytes(peer_id_bytes): tuple(addr) for peer_id_bytes, addr in response[0]}
|
|
|
- # Note: we convert addr from list to tuple here --^ because some msgpack versions convert tuples to lists
|
|
|
- recipient_node_id = DHTID.from_bytes(response[1])
|
|
|
- asyncio.ensure_future(self.update_routing_table(recipient_node_id, recipient, responded=responded))
|
|
|
- return peers
|
|
|
- return {}
|
|
|
-
|
|
|
- def rpc_find_value(self, sender: Endpoint, sender_id_bytes: BinaryDHTID, key_bytes: BinaryDHTID) -> \
|
|
|
- Tuple[Optional[DHTValue], Optional[DHTExpiration], List[Tuple[BinaryDHTID, Endpoint]], BinaryDHTID]:
|
|
|
+ Request keys from a peer. For each key, look for its (value, expiration time) locally and
|
|
|
+ k additional peers that are most likely to have this key (ranked by XOR distance)
|
|
|
+
|
|
|
+ :returns: A dict key => Tuple[optional value, optional expiration time, nearest neighbors]
|
|
|
+ value: value stored by the recipient with that key, or None if peer doesn't have this value
|
|
|
+ expiration time: expiration time of the returned value, None if no value was found
|
|
|
+ neighbors: a dictionary[node_id : endpoint] containing nearest neighbors from peer's routing table
|
|
|
+ If peer didn't respond, returns None
|
|
|
"""
|
|
|
- Someone wants to find value corresponding to key. If we have the value, return the value and its expiration time
|
|
|
- Either way, return :bucket_size: nearest neighbors to that node.
|
|
|
+ keys = list(keys)
|
|
|
+ find_request = dht_pb2.FindRequest(keys=list(map(DHTID.to_bytes, keys)), peer=self.node_info)
|
|
|
+ try:
|
|
|
+ response = await self._get(peer).rpc_find(find_request, timeout=self.wait_timeout)
|
|
|
+ if response.peer and response.peer.node_id:
|
|
|
+ peer_id = DHTID.from_bytes(response.peer.node_id)
|
|
|
+ asyncio.create_task(self.update_routing_table(peer_id, peer, responded=True))
|
|
|
+ assert len(response.values) == len(response.expiration) == len(response.nearest) == len(keys), \
|
|
|
+ "DHTProtocol: response is not aligned with keys"
|
|
|
+
|
|
|
+ output = {} # unpack data without special NOT_FOUND_* values
|
|
|
+ for key, value, expiration, nearest in zip(keys, response.values, response.expiration, response.nearest):
|
|
|
+ value = value if value != _NOT_FOUND_VALUE else None
|
|
|
+ expiration = expiration if expiration != _NOT_FOUND_EXPIRATION else None
|
|
|
+ nearest = dict(zip(map(DHTID.from_bytes, nearest.node_ids), nearest.endpoints))
|
|
|
+ output[key] = (value, expiration, nearest)
|
|
|
+ return output
|
|
|
+ except grpc.experimental.aio.AioRpcError as error:
|
|
|
+ logging.info(f"DHTProtocol failed to store at {peer}: {error.code()}")
|
|
|
+ asyncio.create_task(self.update_routing_table(self.routing_table.get_id(peer), peer, responded=False))
|
|
|
|
|
|
- :returns: (value or None if we have no value, nearest neighbors, our own dht id)
|
|
|
- :note: this is a deviation from Section 2.3 of the paper, original kademlia returner EITHER value OR neighbors
|
|
|
+ async def rpc_find(self, request: dht_pb2.FindRequest, context: grpc.ServicerContext) -> dht_pb2.FindResponse:
|
|
|
"""
|
|
|
- maybe_value, maybe_expiration = self.storage.get(DHTID.from_bytes(key_bytes))
|
|
|
- cached_value, cached_expiration = self.cache.get(DHTID.from_bytes(key_bytes))
|
|
|
- if (cached_expiration or -float('inf')) > (maybe_expiration or -float('inf')):
|
|
|
- maybe_value, maybe_expiration = cached_value, cached_expiration
|
|
|
- nearest_neighbors, my_id = self.rpc_find_node(sender, sender_id_bytes, key_bytes)
|
|
|
- return maybe_value, maybe_expiration, nearest_neighbors, my_id
|
|
|
-
|
|
|
- async def call_find_value(self, recipient: Endpoint, key: DHTID) -> \
|
|
|
- Tuple[Optional[DHTValue], Optional[DHTExpiration], Dict[DHTID, Endpoint]]:
|
|
|
+ Someone wants to find keys in the DHT. For all keys that we have locally, return value and expiration
|
|
|
+ Also return :bucket_size: nearest neighbors from our routing table for each key (whether or not we found value)
|
|
|
"""
|
|
|
- Ask a recipient to give you the value, if it has one, or nearest neighbors to your key.
|
|
|
+ if request.peer: # if requested, add peer to the routing table
|
|
|
+ asyncio.create_task(self.rpc_ping(request.peer, context))
|
|
|
|
|
|
- :returns: (optional value, optional expiration time, and neighbors)
|
|
|
- value: whatever was the latest value stored by the recipient with that key (see DHTNode contract)
|
|
|
- expiration time: expiration time of the returned value, None if no value was found
|
|
|
- neighbors: a dictionary[node id => address] as per Section 2.3 of the paper;
|
|
|
- :note: if no response, returns None, None, {}
|
|
|
- """
|
|
|
- async with self.rpc_semaphore:
|
|
|
- responded, response = await self.find_value(recipient, bytes(self.node_id), bytes(key))
|
|
|
- if responded:
|
|
|
- (value, expiration_time, peers_bytes), recipient_id = response[:-1], DHTID.from_bytes(response[-1])
|
|
|
- peers = {DHTID.from_bytes(peer_id_bytes): tuple(addr) for peer_id_bytes, addr in peers_bytes}
|
|
|
- asyncio.ensure_future(self.update_routing_table(recipient_id, recipient, responded=responded))
|
|
|
- return value, expiration_time, peers
|
|
|
- return None, None, {}
|
|
|
-
|
|
|
- async def update_routing_table(self, node_id: Optional[DHTID], addr: Endpoint, responded=True):
|
|
|
+ response = dht_pb2.FindResponse(values=[], expiration=[], nearest=[], peer=self.node_info)
|
|
|
+ for key_id in map(DHTID.from_bytes, request.keys):
|
|
|
+ maybe_value, maybe_expiration = self.storage.get(key_id)
|
|
|
+ cached_value, cached_expiration = self.cache.get(key_id)
|
|
|
+ if (cached_expiration or -float('inf')) > (maybe_expiration or -float('inf')):
|
|
|
+ maybe_value, maybe_expiration = cached_value, cached_expiration
|
|
|
+ peer_ids, endpoints = zip(*self.routing_table.get_nearest_neighbors(
|
|
|
+ key_id, k=self.bucket_size, exclude=DHTID.from_bytes(request.peer.node_id)))
|
|
|
+
|
|
|
+ response.values.append(maybe_value if maybe_value is not None else _NOT_FOUND_VALUE)
|
|
|
+ response.expiration.append(maybe_expiration if maybe_expiration is not None else _NOT_FOUND_EXPIRATION)
|
|
|
+ response.nearest.append(dht_pb2.Peers(node_ids=list(map(DHTID.to_bytes, peer_ids)), endpoints=endpoints))
|
|
|
+ return response
|
|
|
+
|
|
|
+ async def update_routing_table(self, node_id: Optional[DHTID], peer_endpoint: Endpoint, responded=True):
|
|
|
"""
|
|
|
This method is called on every incoming AND outgoing request to update the routing table
|
|
|
|
|
|
- :param addr: sender endpoint for incoming requests, recipient endpoint for outgoing requests
|
|
|
+ :param peer_endpoint: sender endpoint for incoming requests, recipient endpoint for outgoing requests
|
|
|
:param node_id: sender node id for incoming requests, recipient node id for outgoing requests
|
|
|
:param responded: for outgoing requests, this indicated whether recipient responded or not.
|
|
|
For incoming requests, this should always be True
|
|
|
"""
|
|
|
+ node_id = node_id if node_id is not None else self.routing_table.get_id(peer_endpoint)
|
|
|
if responded: # incoming request or outgoing request with response
|
|
|
if node_id not in self.routing_table:
|
|
|
# we just met a new node, maybe we know some values that it *should* store
|
|
|
+ data_to_send: List[Tuple[DHTID, BinaryDHTValue, DHTExpiration]] = []
|
|
|
for key, value, expiration in list(self.storage.items()):
|
|
|
neighbors = self.routing_table.get_nearest_neighbors(key, self.num_replicas, exclude=self.node_id)
|
|
|
if neighbors:
|
|
|
@@ -155,29 +227,26 @@ class KademliaProtocol(RPCProtocol):
|
|
|
new_node_should_store = node_id.xor_distance(key) < farthest_distance
|
|
|
this_node_is_responsible = self.node_id.xor_distance(key) < nearest_distance
|
|
|
if not neighbors or (new_node_should_store and this_node_is_responsible):
|
|
|
- asyncio.create_task(self.call_store(addr, key, value, expiration))
|
|
|
+ data_to_send.append((key, value, expiration))
|
|
|
+ if data_to_send:
|
|
|
+ asyncio.create_task(self.call_store(peer_endpoint, *zip(*data_to_send), in_cache=False))
|
|
|
|
|
|
- maybe_node_to_ping = self.routing_table.add_or_update_node(node_id, addr)
|
|
|
+ maybe_node_to_ping = self.routing_table.add_or_update_node(node_id, peer_endpoint)
|
|
|
if maybe_node_to_ping is not None:
|
|
|
# we couldn't add new node because the table was full. Check if existing peers are alive (Section 2.2)
|
|
|
# ping one least-recently updated peer: if it won't respond, remove it from the table, else update it
|
|
|
asyncio.create_task(self.call_ping(maybe_node_to_ping[1])) # [1]-th element is that node's endpoint
|
|
|
|
|
|
- else: # outgoing request and peer did not respond
|
|
|
+ else: # we sent outgoing request and peer did not respond
|
|
|
if node_id is not None and node_id in self.routing_table:
|
|
|
del self.routing_table[node_id]
|
|
|
|
|
|
- def _accept_response(self, msg_id, data, address):
|
|
|
- """ Override for RPCProtocol._accept_response to handle cancelled tasks """
|
|
|
- future, timeout = self._outstanding[msg_id]
|
|
|
- if future.cancelled():
|
|
|
- timeout.cancel()
|
|
|
- del self._outstanding[msg_id]
|
|
|
- else:
|
|
|
- super()._accept_response(msg_id, data, address)
|
|
|
+
|
|
|
+_NOT_FOUND_VALUE, _NOT_FOUND_EXPIRATION = b'', -float('inf') # internal values to represent that a value was not found
|
|
|
|
|
|
|
|
|
class LocalStorage:
|
|
|
+ """ Local dictionary that maintains up to :maxsize: tuples of (key, value, expiration) """
|
|
|
def __init__(self, maxsize: Optional[int] = None):
|
|
|
self.cache_size = maxsize or float("inf")
|
|
|
self.data = dict()
|
|
|
@@ -192,7 +261,7 @@ class LocalStorage:
|
|
|
if self.key_to_heap[key] == heap_entry:
|
|
|
del self.data[key], self.key_to_heap[key]
|
|
|
|
|
|
- def store(self, key: DHTID, value: DHTValue, expiration_time: DHTExpiration) -> bool:
|
|
|
+ def store(self, key: DHTID, value: BinaryDHTValue, expiration_time: DHTExpiration) -> bool:
|
|
|
"""
|
|
|
Store a (key, value) pair locally at least until expiration_time. See class docstring for details.
|
|
|
:returns: True if new value was stored, False it was rejected (current value is newer)
|
|
|
@@ -210,14 +279,14 @@ class LocalStorage:
|
|
|
self.remove_outdated()
|
|
|
return True
|
|
|
|
|
|
- def get(self, key: DHTID) -> (Optional[DHTValue], Optional[DHTExpiration]):
|
|
|
+ def get(self, key: DHTID) -> (Optional[BinaryDHTValue], Optional[DHTExpiration]):
|
|
|
""" Get a value corresponding to a key if that (key, value) pair was previously stored here. """
|
|
|
self.remove_outdated()
|
|
|
if key in self.data:
|
|
|
return self.data[key]
|
|
|
return None, None
|
|
|
|
|
|
- def items(self) -> Iterator[Tuple[DHTID, DHTValue, DHTExpiration]]:
|
|
|
+ def items(self) -> Iterator[Tuple[DHTID, BinaryDHTValue, DHTExpiration]]:
|
|
|
""" Iterate over (key, value, expiration_time) tuples stored in this storage """
|
|
|
self.remove_outdated()
|
|
|
return ((key, value, expiration) for key, (value, expiration) in self.data.items())
|
justheuristic
