Proof handling

.gitignore

@@ -1,5 +1,4 @@
 .DS_Store
 *.pyc
-t.py
+testutils
-t2.py
 TODO

RNS/Destination.py

@@ -13,7 +13,7 @@ class Callbacks:
 	def __init__(self):
 		self.link_established = None
 		self.packet = None
-		self.proof = None
+		self.proof_requested = None
 
 class Destination:
 	KEYSIZE    = RNS.Identity.KEYSIZE;
@@ -101,8 +101,8 @@ class Destination:
 	def packet_callback(self, callback):
 		self.callbacks.packet = callback
 
-	def proof_callback(self, callback):
+	def proof_requested_callback(self, callback):
-		self.callbacks.proof = callback
+		self.callbacks.proof_requested = callback
 
 	def setProofStrategy(self, proof_strategy):
 		if not proof_strategy in Destination.proof_strategies:

RNS/Identity.py

@@ -18,8 +18,10 @@ class Identity:
 	KEYSIZE     = 1024
 	DERKEYSIZE  = KEYSIZE+272
 
-	# Padding size, not configurable
+	# Non-configurable constants
-	PADDINGSIZE= 336
+	PADDINGSIZE = 336		# In bits
+	HASHLENGTH  = 256		# In bits
+	SIGLENGTH   = KEYSIZE
 
 	# Storage
 	known_destinations = {}
@@ -257,13 +259,21 @@ class Identity:
 					hashes.SHA256()
 				)
 				return True
-			except:
+			except Exception as e:
 				return False
 		else:
 			raise KeyError("Signature validation failed because identity does not hold a public key")
 
-	def prove(self, packet, destination):
+	def prove(self, packet, destination=None):
-		proof_data = packet.packet_hash + self.sign(packet.packet_hash)
+		signature = self.sign(packet.packet_hash)
+		if RNS.Reticulum.should_use_implicit_proof():
+			proof_data = signature
+		else:
+			proof_data = packet.packet_hash + signature
+		
+		if destination == None:
+			destination = packet.generateProofDestination()
+
 		proof = RNS.Packet(destination, proof_data, RNS.Packet.PROOF)
 		proof.send()
 

RNS/Interfaces/AX25KISSInterface.py

@@ -28,7 +28,6 @@ class AX25():
 	CRC_CORRECT     = chr(0xF0)+chr(0xB8)
 
 
-# TODO: THIS CLASS IS NOT YET IMPLEMENTED --- PLACEHOLDER ONLY ---
 class AX25KISSInterface(Interface):
 	MAX_CHUNK = 32768
 

RNS/Packet.py

@@ -2,6 +2,96 @@ import struct
 import time
 import RNS
 
+class PacketReceipt:
+	# Receipt status constants
+	FAILED    = 0x00
+	SENT	  = 0x01
+	DELIVERED = 0x02
+
+
+	EXPL_LENGTH = RNS.Identity.HASHLENGTH/8+RNS.Identity.SIGLENGTH/8
+	IMPL_LENGTH = RNS.Identity.SIGLENGTH/8
+
+	# Creates a new packet receipt from a sent packet
+	def __init__(self, packet):
+		self.hash    = packet.getHash()
+		self.sent    = True
+		self.sent_at = time.time()
+		self.timeout = Packet.TIMEOUT
+		self.proved  = False
+		self.status  = PacketReceipt.SENT
+		self.destination = packet.destination
+		self.callbacks   = PacketReceiptCallbacks()
+		self.concluded_at = None
+
+	# Validate a proof packet
+	def validateProofPacket(self, proof_packet):
+		return self.validateProof(proof_packet.data)
+
+	# Validate a raw proof
+	def validateProof(self, proof):
+		if len(proof) == PacketReceipt.EXPL_LENGTH:
+			# This is an explicit proof
+			proof_hash = proof[:RNS.Identity.HASHLENGTH/8]
+			signature = proof[RNS.Identity.HASHLENGTH/8:RNS.Identity.HASHLENGTH/8+RNS.Identity.SIGLENGTH/8]
+			if proof_hash == self.hash:
+				proof_valid = self.destination.identity.validate(signature, self.hash)
+				if proof_valid:
+					self.status = PacketReceipt.DELIVERED
+					self.proved = True
+					if self.callbacks.delivery != None:
+						self.callbacks.delivery(self)
+					return True
+				else:
+					return False
+			else:
+				return False
+		elif len(proof) == PacketReceipt.IMPL_LENGTH:
+			# This is an implicit proof
+			signature = proof[:RNS.Identity.SIGLENGTH/8]
+			proof_valid = self.destination.identity.validate(signature, self.hash)
+			if proof_valid:
+					self.status = PacketReceipt.DELIVERED
+					self.proved = True
+					if self.callbacks.delivery != None:
+						self.callbacks.delivery(self)
+					return True
+			else:
+				return False
+		else:
+			return False
+
+
+	def isTimedOut(self):
+		return (self.sent_at+self.timeout < time.time())
+
+	def checkTimeout(self):
+		if self.isTimedOut():
+			self.status = PacketReceipt.FAILED
+			self.concluded_at = time.time()
+			if self.callbacks.timeout:
+				self.callbacks.timeout(self)
+
+
+	# Set the timeout in seconds
+	def setTimeout(self, timeout):
+		self.timeout = float(timeout)
+
+	# Set a function that gets called when
+	# a successfull delivery has been proved
+	def delivery_callback(self, callback):
+		self.callbacks.delivery = callback
+
+	# Set a function that gets called if the
+	# delivery times out
+	def timeout_callback(self, callback):
+		self.callbacks.timeout = callback
+
+class PacketReceiptCallbacks:
+	def __init__(self):
+		self.delivery = None
+		self.timeout  = None
+
 class Packet:
 	# Constants
 	DATA         = 0x00;
@@ -16,6 +106,9 @@ class Packet:
 	HEADER_4     = 0x03;	# Reserved
 	header_types = [HEADER_1, HEADER_2, HEADER_3, HEADER_4]
 
+	# Defaults
+	TIMEOUT 	 = 3600.0
+
 	def __init__(self, destination, data, packet_type = DATA, transport_type = RNS.Transport.BROADCAST, header_type = HEADER_1, transport_id = None):
 		if destination != None:
 			if transport_type == None:
@@ -35,6 +128,7 @@ class Packet:
 			self.raw    		= None
 			self.packed 		= False
 			self.sent   		= False
+			self.receipt 		= None
 			self.fromPacked		= False
 		else:
 			self.raw            = data
@@ -67,6 +161,7 @@ class Packet:
 				self.ciphertext = self.destination.encrypt(self.data)
 			else:
 				self.ciphertext = self.data
+
 		if self.header_type == Packet.HEADER_3:
 			self.header += self.destination.link_id
 			self.ciphertext = self.data
@@ -103,10 +198,11 @@ class Packet:
 			if not self.packed:
 				self.pack()
 	
-			RNS.Transport.outbound(self)
+			if RNS.Transport.outbound(self):
-			self.packet_hash = RNS.Identity.fullHash(self.raw)
+				return self.receipt
-			self.sent_at = time.time()
+			else:
-			self.sent = True
+				# TODO: Don't raise error here, handle gracefully
+				raise IOError("Packet could not be sent! Do you have any outbound interfaces configured?")
 		else:
 			raise IOError("Packet was already sent")
 
@@ -116,21 +212,36 @@ class Packet:
 		else:
 			raise IOError("Packet was not sent yet")
 
-	def prove(self, destination):
+	def prove(self, destination=None):
 		if self.fromPacked and self.destination:
 			if self.destination.identity and self.destination.identity.prv:
 				self.destination.identity.prove(self, destination)
 
+	# Generates a special destination that allows Reticulum
+	# to direct the proof back to the proved packet's sender
+	def generateProofDestination(self):
+		return ProofDestination(self)
+
 	def validateProofPacket(self, proof_packet):
-		return self.validateProof(proof_packet.data)
+		return self.receipt.validateProofPacket(proof_packet)
 
 	def validateProof(self, proof):
-		proof_hash = proof[:32]
+		return self.receipt.validateProof(proof)
-		signature = proof[32:]
-		if proof_hash == self.packet_hash:
-			return self.destination.identity.validate(signature, proof_hash)
-		else:
-			return False
 
+	def updateHash(self):
+		self.packet_hash = self.getHash()
 
+	def getHash(self):
+		return RNS.Identity.fullHash(self.getHashablePart())
+
+	def getHashablePart(self):
+		return self.raw[0:1]+self.raw[2:]
+
+class ProofDestination:
+	def __init__(self, packet):
+		self.hash = packet.getHash()[:10];
+		self.type = RNS.Destination.SINGLE
+
+	def encrypt(self, plaintext):
+		return plaintext
 

RNS/Reticulum.py

@@ -29,6 +29,7 @@ class Reticulum:
 		Reticulum.cachepath = Reticulum.configdir+"/storage/cache"
 
 		Reticulum.__allow_unencrypted = False
+		Reticulum.__use_implicit_proof = False
 
 		if not os.path.isdir(Reticulum.storagepath):
 			os.makedirs(Reticulum.storagepath)
@@ -50,6 +51,8 @@ class Reticulum:
 		RNS.Identity.loadKnownDestinations()
 		Reticulum.router = self
 
+		RNS.Transport.scheduleJobs()
+
 		atexit.register(RNS.Identity.exitHandler)
 
 	def applyConfig(self):
@@ -66,6 +69,11 @@ class Reticulum:
 		if "reticulum" in self.config:
 			for option in self.config["reticulum"]:
 				value = self.config["reticulum"][option]
+				if option == "use_implicit_proof":
+					if value == "true":
+						Reticulum.__use_implicit_proof = True
+					if value == "false":
+						Reticulum.__use_implicit_proof = False
 				if option == "allow_unencrypted":
 					if value == "true":
 						RNS.log("", RNS.LOG_CRITICAL)
@@ -260,3 +268,7 @@ class Reticulum:
 	@staticmethod
 	def should_allow_unencrypted():
 		return Reticulum.__allow_unencrypted
+
+	@staticmethod
+	def should_use_implicit_proof():
+		return Reticulum.__use_implicit_proof

RNS/Transport.py

@@ -1,4 +1,7 @@
 import RNS
+import time
+import threading
+from time import sleep
 
 class Transport:
 	# Constants
@@ -8,15 +11,65 @@ class Transport:
 	TUNNEL       = 0x03;
 	types        = [BROADCAST, TRANSPORT, RELAY, TUNNEL]
 
-	interfaces	 	= []
+	interfaces	 	= []	# All active interfaces
-	destinations    = []
+	destinations    = []	# All active destinations
-	pending_links   = []
+	pending_links   = []	# Links that are being established
-	active_links	= []
+	active_links	= []	# Links that are active
-	packet_hashlist = []
+	packet_hashlist = []	# A list of packet hashes for duplicate detection
+	receipts		= []	# Receipts of all outgoing packets for proof processing
+
+	jobs_locked = False
+	jobs_running = False
+	job_interval = 0.250
+	receipts_last_checked   = 0.0
+	receipts_check_interval = 1.0
+	hashlist_maxsize        = 1000000
+
+	@staticmethod
+	def scheduleJobs():
+		thread = threading.Thread(target=Transport.jobloop)
+		thread.setDaemon(True)
+		thread.start()
+
+	@staticmethod
+	def jobloop():
+		while (True):
+			Transport.jobs()
+			sleep(Transport.job_interval)
+
+	@staticmethod
+	def jobs():
+		Transport.jobs_running = True
+		try:
+			if not Transport.jobs_locked:
+				# Process receipts list for timed-out packets
+				if Transport.receipts_last_checked+Transport.receipts_check_interval < time.time():
+					for receipt in Transport.receipts:
+						receipt.checkTimeout()
+						if receipt.status != RNS.PacketReceipt.SENT:
+							Transport.receipts.remove(receipt)
+
+					Transport.receipts_last_checked = time.time()
+
+				# Cull the packet hashlist if it has reached max size
+				while (len(Transport.packet_hashlist) > Transport.hashlist_maxsize):
+					Transport.packet_hashlist.pop(0)
+
+		except Exception as e:
+			RNS.log("An exception occurred while running Transport jobs.", RNS.LOG_ERROR)
+			RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
+
+		Transport.jobs_running = False
 
 	@staticmethod
 	def outbound(packet):
-		Transport.cacheRaw(packet.raw)
+		while (Transport.jobs_running):
+			sleep(0.1)
+
+		Transport.jobs_locked = True
+		packet.updateHash()
+		sent = False
+		
 		for interface in Transport.interfaces:
 			if interface.OUT:
 				should_transmit = True
@@ -27,19 +80,38 @@ class Transport:
 				if should_transmit:
 					RNS.log("Transmitting "+str(len(packet.raw))+" bytes via: "+str(interface), RNS.LOG_DEBUG)
 					interface.processOutgoing(packet.raw)
+					sent = True
+
+		if sent:
+			packet.sent = True
+			packet.sent_at = time.time()
+
+			if (packet.packet_type == RNS.Packet.DATA):
+				packet.receipt = RNS.PacketReceipt(packet)
+				Transport.receipts.append(packet.receipt)
+			
+			Transport.cache(packet)
+
+		Transport.jobs_locked = False
+		return sent
 
 	@staticmethod
 	def inbound(raw, interface=None):
-		packet_hash = RNS.Identity.fullHash(raw)
+		while (Transport.jobs_running):
-		RNS.log(str(interface)+" received packet with hash "+RNS.prettyhexrep(packet_hash), RNS.LOG_DEBUG)
+			sleep(0.1)
+			
+		Transport.jobs_locked = True
 		
-		if not packet_hash in Transport.packet_hashlist:
-			Transport.packet_hashlist.append(packet_hash)
 		packet = RNS.Packet(None, raw)
 		packet.unpack()
-			packet.packet_hash = packet_hash
+		packet.updateHash()
 		packet.receiving_interface = interface
 
+		RNS.log(str(interface)+" received packet with hash "+RNS.prettyhexrep(packet.packet_hash), RNS.LOG_DEBUG)
+
+		if not packet.packet_hash in Transport.packet_hashlist:
+			Transport.packet_hashlist.append(packet.packet_hash)
+			
 			if packet.packet_type == RNS.Packet.ANNOUNCE:
 				if RNS.Identity.validateAnnounce(packet):
 					Transport.cache(packet)
@@ -64,6 +136,13 @@ class Transport:
 							destination.receive(packet)
 							Transport.cache(packet)
 
+							if destination.proof_strategy == RNS.Destination.PROVE_ALL:
+								packet.prove()
+
+							if destination.proof_strategy == RNS.Destination.PROVE_APP:
+								if destination.callbacks.proof_requested:
+									destination.callbacks.proof_requested(packet)
+
 			if packet.packet_type == RNS.Packet.PROOF:
 				if packet.header_type == RNS.Packet.HEADER_3:
 					# This is a link request proof, forward
@@ -72,11 +151,27 @@ class Transport:
 						if link.link_id == packet.destination_hash:
 							link.validateProof(packet)
 				else:
-					for destination in Transport.destinations:
+					# TODO: Make sure everything uses new proof handling
-						if destination.hash == packet.destination_hash:
+					if len(packet.data) == RNS.PacketReceipt.EXPL_LENGTH:
-							if destination.proofcallback != None:
+						proof_hash = packet.data[:RNS.Identity.HASHLENGTH/8]
-								destination.proofcallback(packet)
+					else:
-							# TODO: add universal proof handling
+						proof_hash = None
+
+					for receipt in Transport.receipts:
+						receipt_validated = False
+						if proof_hash != None:
+							# Only test validation if hash matches
+							if receipt.hash == proof_hash:
+								receipt_validated = receipt.validateProofPacket(packet)
+						else:
+							# In case of an implicit proof, we have
+							# to check every single outstanding receipt
+							receipt_validated = receipt.validateProofPacket(packet)
+
+						if receipt_validated:
+							Transport.receipts.remove(receipt)
+
+		Transport.jobs_locked = False
 
 	@staticmethod
 	def registerDestination(destination):
@@ -112,15 +207,13 @@ class Transport:
 	@staticmethod
 	def cache(packet):
 		if RNS.Transport.shouldCache(packet):
-			RNS.Transport.cacheRaw(packet.raw)
-
-	@staticmethod
-	def cacheRaw(raw):
 			try:
-			file = open(RNS.Reticulum.cachepath+"/"+RNS.hexrep(RNS.Identity.fullHash(raw), delimit=False), "w")
+				packet_hash = RNS.hexrep(packet.getHash(), delimit=False)
-			file.write(raw)
+				file = open(RNS.Reticulum.cachepath+"/"+packet_hash, "w")
+				file.write(packet.raw)
 				file.close()
-			RNS.log("Wrote packet "+RNS.prettyhexrep(RNS.Identity.fullHash(raw))+" to cache", RNS.LOG_DEBUG)
+				RNS.log("Wrote packet "+packet_hash+" to cache", RNS.LOG_DEBUG)
 			except Exception as e:
 				RNS.log("Error writing packet to cache", RNS.LOG_ERROR)
 				RNS.log("The contained exception was: "+str(e))
+

RNS/__init__.py

@@ -9,6 +9,7 @@ from .Link import Link
 from .Transport import Transport
 from .Destination import Destination
 from .Packet import Packet
+from .Packet import PacketReceipt
 
 modules = glob.glob(os.path.dirname(__file__)+"/*.py")
 __all__ = [ os.path.basename(f)[:-3] for f in modules if not f.endswith('__init__.py')]

RNS/vendor/configobj.py

@@ -38,7 +38,7 @@ BOMS = {
     BOM_UTF16: ('utf_16', 'utf_16'),
     }
 # All legal variants of the BOM codecs.
-# TODO: the list of aliases is not meant to be exhaustive, is there a
+# The list of aliases is not meant to be exhaustive, is there a
 #   better way ?
 BOM_LIST = {
     'utf_16': 'utf_16',