Variable size for trampoline onion

The size of trampoline onions was fixed at 400 bytes. We remove this limit, allowing larger trampoline onions when necessary.
For backward compatibility, anything that used to fit in a 400 bytes onion still uses a 400 bytes onion (even if it could fit in less).
Payments with large metadata that would fail because of the 400 bytes limit are now possible and the onion uses exactly the size that's needed, nothing more, nothing less.
The size of the trampoline onion is still constrained by the size of the standard onion (1300 bytes).

eclair-core/src/main/scala/fr/acinq/eclair/crypto/Sphinx.scala

@@ -227,15 +227,17 @@ object Sphinx extends Logging {
    * Create an encrypted onion packet that contains payloads for all nodes in the list.
    *
    * @param sessionKey          session key.
-   * @param packetPayloadLength length of the packet's encrypted onion payload (e.g. 1300 for standard payment onions).
+   * @param minPacketLength     minimum length of the packet's encrypted onion payload (e.g. 1300 for standard payment onions).
+   * @param maxPacketLength     maximum length of the packet's encrypted onion payload (e.g. 1300 for standard payment onions).
    * @param publicKeys          node public keys (one per node).
    * @param payloads            payloads (one per node).
    * @param associatedData      associated data.
    * @return An onion packet with all shared secrets. The onion packet can be sent to the first node in the list, and
    *         the shared secrets (one per node) can be used to parse returned failure messages if needed.
    */
-  def create(sessionKey: PrivateKey, packetPayloadLength: Int, publicKeys: Seq[PublicKey], payloads: Seq[ByteVector], associatedData: Option[ByteVector32]): Try[PacketAndSecrets] = Try {
-    require(payloads.map(_.length + MacLength).sum <= packetPayloadLength, s"packet per-hop payloads cannot exceed $packetPayloadLength bytes")
+  def create(sessionKey: PrivateKey, minPacketLength: Int, maxPacketLength: Int, publicKeys: Seq[PublicKey], payloads: Seq[ByteVector], associatedData: Option[ByteVector32]): Try[PacketAndSecrets] = Try {
+    val packetPayloadLength = minPacketLength max payloads.map(_.length + MacLength).sum.toInt
+    require(packetPayloadLength <= maxPacketLength, s"packet per-hop payloads cannot exceed $maxPacketLength bytes")
     val (ephemeralPublicKeys, sharedsecrets) = computeEphemeralPublicKeysAndSharedSecrets(sessionKey, publicKeys)
     val filler = generateFiller("rho", packetPayloadLength, sharedsecrets.dropRight(1), payloads.dropRight(1))
 

eclair-core/src/main/scala/fr/acinq/eclair/message/OnionMessages.scala

@@ -157,7 +157,7 @@ object OnionMessages {
           payloadSize.toInt
         }
         // Since we are setting the packet size based on the payload, the onion creation should never fail (hence the `.get`).
-        val Sphinx.PacketAndSecrets(packet, _) = Sphinx.create(sessionKey, packetSize, route.blindedNodes.map(_.blindedPublicKey), payloads, None).get
+        val Sphinx.PacketAndSecrets(packet, _) = Sphinx.create(sessionKey, packetSize, packetSize, route.blindedNodes.map(_.blindedPublicKey), payloads, None).get
         Right((route.introductionNodeId, OnionMessage(route.blindingKey, packet)))
     }
   }

eclair-core/src/main/scala/fr/acinq/eclair/payment/PaymentPacket.scala

@@ -251,7 +251,7 @@ object OutgoingPaymentPacket {
   // @formatter:on
 
   /** Build an encrypted onion packet from onion payloads and node public keys. */
-  def buildOnion(packetPayloadLength: Int, payloads: Seq[NodePayload], associatedData: ByteVector32): Either[OutgoingPaymentError, Sphinx.PacketAndSecrets] = {
+  def buildOnion(minPacketLength: Int, maxPacketLength: Int, payloads: Seq[NodePayload], associatedData: ByteVector32): Either[OutgoingPaymentError, Sphinx.PacketAndSecrets] = {
     val sessionKey = randomKey()
     val nodeIds = payloads.map(_.nodeId)
     val payloadsBin = payloads
@@ -260,7 +260,7 @@ object OutgoingPaymentPacket {
         case Attempt.Successful(bits) => bits.bytes
         case Attempt.Failure(cause) => return Left(CannotCreateOnion(cause.message))
       }
-    Sphinx.create(sessionKey, packetPayloadLength, nodeIds, payloadsBin, Some(associatedData)) match {
+    Sphinx.create(sessionKey, minPacketLength, maxPacketLength, nodeIds, payloadsBin, Some(associatedData)) match {
       case Failure(f) => Left(CannotCreateOnion(f.getMessage))
       case Success(packet) => Right(packet)
     }
@@ -297,7 +297,7 @@ object OutgoingPaymentPacket {
     for {
       paymentTmp <- recipient.buildPayloads(paymentHash, route)
       outgoing <- getOutgoingChannel(privateKey, paymentTmp, route)
-      onion <- buildOnion(PaymentOnionCodecs.paymentOnionPayloadLength, outgoing.payment.payloads, paymentHash) // BOLT 2 requires that associatedData == paymentHash
+      onion <- buildOnion(PaymentOnionCodecs.paymentOnionPayloadLength, PaymentOnionCodecs.paymentOnionPayloadLength, outgoing.payment.payloads, paymentHash) // BOLT 2 requires that associatedData == paymentHash
     } yield {
       val cmd = CMD_ADD_HTLC(replyTo, outgoing.payment.amount, paymentHash, outgoing.payment.expiry, onion.packet, outgoing.nextBlinding_opt, Origin.Hot(replyTo, upstream), commit = true)
       OutgoingPaymentPacket(cmd, outgoing.shortChannelId, onion.sharedSecrets)

eclair-core/src/main/scala/fr/acinq/eclair/payment/send/Recipient.scala

@@ -23,7 +23,7 @@ import fr.acinq.eclair.payment.Invoice.ExtraEdge
 import fr.acinq.eclair.payment.OutgoingPaymentPacket._
 import fr.acinq.eclair.payment.{Bolt11Invoice, Bolt12Invoice, OutgoingPaymentPacket, PaymentBlindedRoute}
 import fr.acinq.eclair.router.Router._
-import fr.acinq.eclair.wire.protocol.PaymentOnion.{FinalPayload, IntermediatePayload, OutgoingBlindedPerHopPayload}
+import fr.acinq.eclair.wire.protocol.PaymentOnion.{EmptyPayload, FinalPayload, IntermediatePayload, OutgoingBlindedPerHopPayload}
 import fr.acinq.eclair.wire.protocol.{GenericTlv, OnionRoutingPacket, PaymentOnionCodecs}
 import fr.acinq.eclair.{CltvExpiry, Features, InvoiceFeature, MilliSatoshi, MilliSatoshiLong, ShortChannelId}
 import scodec.bits.ByteVector
@@ -230,14 +230,14 @@ case class ClearTrampolineRecipient(invoice: Bolt11Invoice,
       val finalPayload = NodePayload(nodeId, FinalPayload.Standard.createPayload(totalAmount, totalAmount, expiry, invoice.paymentSecret, invoice.paymentMetadata, customTlvs))
       val trampolinePayload = NodePayload(trampolineHop.nodeId, IntermediatePayload.NodeRelay.Standard(totalAmount, expiry, nodeId))
       val payloads = Seq(trampolinePayload, finalPayload)
-      OutgoingPaymentPacket.buildOnion(PaymentOnionCodecs.trampolineOnionPayloadLength, payloads, paymentHash)
+      OutgoingPaymentPacket.buildOnion(PaymentOnionCodecs.trampolineOnionPayloadLength, PaymentOnionCodecs.paymentOnionPayloadLength, payloads, paymentHash)
     } else {
       // The recipient doesn't support trampoline: the trampoline node will convert the payment to a non-trampoline payment.
       // The final payload will thus never reach the recipient, so we create the smallest payload possible to avoid overflowing the trampoline onion size.
-      val dummyFinalPayload = NodePayload(nodeId, IntermediatePayload.ChannelRelay.Standard(ShortChannelId(0), 0 msat, CltvExpiry(0)))
+      val dummyFinalPayload = NodePayload(nodeId, EmptyPayload)
       val trampolinePayload = NodePayload(trampolineHop.nodeId, IntermediatePayload.NodeRelay.Standard.createNodeRelayToNonTrampolinePayload(totalAmount, totalAmount, expiry, nodeId, invoice))
       val payloads = Seq(trampolinePayload, dummyFinalPayload)
-      OutgoingPaymentPacket.buildOnion(PaymentOnionCodecs.trampolineOnionPayloadLength, payloads, paymentHash)
+      OutgoingPaymentPacket.buildOnion(PaymentOnionCodecs.trampolineOnionPayloadLength, PaymentOnionCodecs.paymentOnionPayloadLength, payloads, paymentHash)
     }
   }
 }

eclair-core/src/main/scala/fr/acinq/eclair/wire/protocol/OnionRouting.scala

@@ -18,10 +18,11 @@ package fr.acinq.eclair.wire.protocol
 
 import fr.acinq.bitcoin.scalacompat.ByteVector32
 import fr.acinq.eclair.UInt64
-import fr.acinq.eclair.wire.protocol.CommonCodecs.bytes32
+import fr.acinq.eclair.wire.protocol.CommonCodecs.{bytes32, varintoverflow}
 import scodec.Codec
 import scodec.bits.ByteVector
 import scodec.codecs._
+import shapeless.HNil
 
 /**
  * Created by t-bast on 05/07/2019.
@@ -46,4 +47,17 @@ object OnionRoutingCodecs {
       ("onionPayload" | bytes(payloadLength)) ::
       ("hmac" | bytes32)).as[OnionRoutingPacket]
 
+
+  val variableSizeOnionRoutingPacketCodec: Codec[OnionRoutingPacket] = (
+    variableSizePrefixedBytesLong(varintoverflow.xmap(l => l - 66, l => l + 66),
+      ("version" | uint8) ::
+        ("publicKey" | bytes(33)),
+      ("onionPayload" | bytes)) ::
+      ("hmac" | bytes32)).xmap[OnionRoutingPacket](
+    {
+      case shapeless.::((shapeless.::(version, shapeless.::(publicKey, HNil)), payload), shapeless.::(hmac, HNil)) => OnionRoutingPacket(version, publicKey, payload, hmac)
+    }, {
+      case OnionRoutingPacket(version, publicKey, payload, hmac) => shapeless.::((shapeless.::(version, shapeless.::(publicKey, HNil)), payload), shapeless.::(hmac, HNil))
+    }
+  )
 }

eclair-core/src/main/scala/fr/acinq/eclair/wire/protocol/PaymentOnion.scala

@@ -20,7 +20,7 @@ import fr.acinq.bitcoin.scalacompat.ByteVector32
 import fr.acinq.bitcoin.scalacompat.Crypto.PublicKey
 import fr.acinq.eclair.payment.Bolt11Invoice
 import fr.acinq.eclair.wire.protocol.CommonCodecs._
-import fr.acinq.eclair.wire.protocol.OnionRoutingCodecs.{ForbiddenTlv, InvalidTlvPayload, MissingRequiredTlv}
+import fr.acinq.eclair.wire.protocol.OnionRoutingCodecs.{ForbiddenTlv, InvalidTlvPayload, MissingRequiredTlv, variableSizeOnionRoutingPacketCodec}
 import fr.acinq.eclair.wire.protocol.TlvCodecs._
 import fr.acinq.eclair.{CltvExpiry, Features, MilliSatoshi, MilliSatoshiLong, ShortChannelId, UInt64}
 import scodec.bits.{BitVector, ByteVector}
@@ -217,6 +217,10 @@ object PaymentOnion {
     def records: TlvStream[OnionPaymentPayloadTlv]
   }
 
+  case object EmptyPayload extends PerHopPayload {
+    override def records: TlvStream[OnionPaymentPayloadTlv] = TlvStream.empty
+  }
+
   /** Per-hop payload for an intermediate node. */
   sealed trait IntermediatePayload extends PerHopPayload
 
@@ -509,7 +513,7 @@ object PaymentOnionCodecs {
 
   private val invoiceRoutingInfo: Codec[InvoiceRoutingInfo] = tlvField(list(listOfN(uint8, Bolt11Invoice.Codecs.extraHopCodec)))
 
-  private val trampolineOnion: Codec[TrampolineOnion] = tlvField(trampolineOnionPacketCodec)
+  private val trampolineOnion: Codec[TrampolineOnion] = variableSizeOnionRoutingPacketCodec.as[TrampolineOnion]
 
   private val keySend: Codec[KeySend] = tlvField(bytes32)