From 0412e2bbdff880abf7ec0b18cfe14759f2fc8abc Mon Sep 17 00:00:00 2001
From: Yilong Li <yilongl@cs.stanford.edu>
Date: Sat, 27 Apr 2019 13:55:11 -0700
Subject: [PATCH] Fixed two bugs introduced in the previous commit (hash
 4319853)

The first bug is that packet buffers arriving out-of-order are not returned to
the driver properly; this will exhaust the driver's rx buffers eventually. The
second bug is some experimental junk left in the homa benchmark code.
---
 apps/ClusterPerf.cc   | 4 ++--
 src/BasicTransport.cc | 5 +++++
 src/DpdkDriver.cc     | 4 +---
 src/HomaTransport.cc  | 5 +++++
 4 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/apps/ClusterPerf.cc b/apps/ClusterPerf.cc
index 9f57fb2d3..b900df39b 100644
--- a/apps/ClusterPerf.cc
+++ b/apps/ClusterPerf.cc
@@ -3024,13 +3024,13 @@ echo_basic()
         double probability;
         inFile >> avgMessageSize;
         while (inFile >> size >> probability) {
-            outgoingSizes.push_back(30);
+            outgoingSizes.push_back(size);
             incomingSizes.push_back(size);
             ids.push_back(std::to_string(size));
         }
     } else {
         outgoingSizes = {30, 30, 30, 30, 30};
-        incomingSizes = {100, 1024, 10*1024, 100*1024, 1024*1024};
+        incomingSizes = {100, 1000, 10*1000, 100*1000, 1000000};
         ids = {"100", "1K", "10K", "100K", "1M"};
     }
 
diff --git a/src/BasicTransport.cc b/src/BasicTransport.cc
index c95ed6e06..e8b6880e4 100644
--- a/src/BasicTransport.cc
+++ b/src/BasicTransport.cc
@@ -1617,6 +1617,11 @@ BasicTransport::MessageAccumulator::addPacket(DataHeader *header,
             } else {
                 buffer->appendCopy(payload + sizeof32(DataHeader),
                         fragment.length);
+                if (fragment.header != header) {
+                    // This packet was retained earlier due to out-of-order
+                    // arrival and must be returned to driver now.
+                    t->driver->returnPacket(payload);
+                }
             }
             FragmentMap::iterator it = fragments.find(buffer->size());
             if (it == fragments.end()) {
diff --git a/src/DpdkDriver.cc b/src/DpdkDriver.cc
index cbee97bc7..39fec0559 100644
--- a/src/DpdkDriver.cc
+++ b/src/DpdkDriver.cc
@@ -411,12 +411,10 @@ DpdkDriver::sendPacket(const Address* addr,
     struct rte_mbuf* mbuf = rte_pktmbuf_alloc(mbufPool);
 #endif
     if (unlikely(NULL == mbuf)) {
-        RAMCLOUD_CLOG(WARNING,
-                "Failed to allocate a packet buffer; dropping packet; "
+        DIE("Failed to allocate a packet buffer; dropping packet; "
                 "%u mbufs available, %u mbufs in use",
                 rte_mempool_avail_count(mbufPool),
                 rte_mempool_in_use_count(mbufPool));
-        return;
     }
 
 #if TESTING
diff --git a/src/HomaTransport.cc b/src/HomaTransport.cc
index 6e2d7bae5..b87afcc4c 100644
--- a/src/HomaTransport.cc
+++ b/src/HomaTransport.cc
@@ -1818,6 +1818,11 @@ HomaTransport::MessageAccumulator::addPacket(DataHeader *header,
             } else {
                 buffer->appendCopy(payload + sizeof32(DataHeader),
                         fragment.length);
+                if (fragment.header != header) {
+                    // This packet was retained earlier due to out-of-order
+                    // arrival and must be returned to driver now.
+                    t->driver->returnPacket(payload);
+                }
             }
             FragmentMap::iterator it = fragments.find(buffer->size());
             if (it == fragments.end()) {