merge end of facebook branch into trunk, after copying the old trunk to the memcached-1.1.x branch

git-svn-id: http://code.sixapart.com/svn/memcached/trunk/server@320 b0b603af-a30f-0410-a34e-baf09ae79d0b

Author: bradfitz

ChangeLog

@@ -1,3 +1,18 @@
+2006-08-21
+	* Nathan Neulinger <[email protected]>: fix incompatabilities with
+	  unix domain socket support and the UDP code and clean up stale 
+	  sockets
+
+2006-08-20
+	* Nathan Neulinger <[email protected]>: unix domain socket support
+
+2006-05-03
+	* Steven Grimm <[email protected]>:  big bunch of changes:
+	  big CPU reduction work, UDP-based interface, increased memory
+	  efficiency.  (intertwined patch, committed all together)
+	  <http://lists.danga.com/pipermail/memcached/2006-May/002164.html>
+	  or see svn commit logs
+
 2006-04-30
 	* River Tarnell:  autoconf work for Solaris 10.  Brad:
 	merge and verify it works on Nexenta.

TODO

@@ -1,3 +1,20 @@
+* bug as shown with netcat (w/ small 16 byte object reproduces)
+
+>>I've done the following script to check that memcached has a key in it's
+>>inside, and thus know that it's working correctly:
+>>echo -e "get is_ok\r\nquit\r\n" | netcat $host $ip
+>>
+>>and I find that sometimes it returns the VALUE in it's inside, but other
+>>not.
+
+* namespaces
+
+* binary get protocol
+
+* refresh/touch command.
+
+* finer granularity of time for flush_all/delete, or generation number.
+
 * slab class reassignment still buggy and can crash.  once that's
   stable, server should re-assign pages every 60 seconds or so
   to keep all classes roughly equal.  [Update: fixed now?, but 

configure.ac

@@ -1,5 +1,5 @@
 AC_PREREQ(2.52)
-AC_INIT(memcached, 1.1.13-pre2, [email protected])
+AC_INIT(memcached, 1.2.0-rc1, [email protected])
 AC_CANONICAL_SYSTEM
 AC_CONFIG_SRCDIR(memcached.c)
 AM_INIT_AUTOMAKE(AC_PACKAGE_NAME, AC_PACKAGE_VERSION)

doc/memcached.1

@@ -43,7 +43,10 @@ caches, so consult the README and memcached homepage for configuration
 suggestions.
 .TP
 .B \-p <num> 
-Listen on port <num>, the default is port 11211.
+Listen on TCP port <num>, the default is port 11211.
+.TP
+.B \-U <num> 
+Listen on UDP port <num>, the default is port 11211.
 .TP
 .B \-M
 Disable automatic removal of items from the cache when out of memory.
@@ -52,6 +55,20 @@ Additions will not be possible until adequate space is freed up.
 .B \-r
 Raise the core file size limit to the maximum allowable.
 .TP
+.B \-f <factor>
+Use <factor> as the multiplier for computing the sizes of memory chunks that
+items are stored in. A lower value may result in less wasted memory depending
+on the total amount of memory available and the distribution of item sizes.
+The default is 1.25.
+.TP
+.B \-s <size>
+Allocate a minimum of <size> bytes for the item key, value, and flags. The
+default is 48. If you have a lot of small keys and values, you can get a
+significant memory efficiency gain with a lower value. If you use a high
+chunk growth factor (-f option), on the other hand, you may want to increase
+the size to allow a bigger percentage of your items to fit in the most densely
+packed (smallest) chunks.
+.TP
 .B \-h
 Show the version of memcached and a summary of options.
 .TP

doc/protocol.txt

@@ -1,8 +1,9 @@
 Protocol
 --------
 
-Clients of memcached communicate with server through TCP
-connections. A given running memcached server listens on some
+Clients of memcached communicate with server through TCP connections.
+(A UDP interface is also available; details are below under "UDP
+protocol.") A given running memcached server listens on some
 (configurable) port; clients connect to that port, send commands to
 the server, read responses, and eventually close the connection.
 
@@ -388,3 +389,45 @@ quit\r\n
 Upon receiving this command, the server closes the
 connection. However, the client may also simply close the connection
 when it no longer needs it, without issuing this command.
+
+
+UDP protocol
+------------
+
+For very large installations where the number of clients is high enough
+that the number of TCP connections causes scaling difficulties, there is
+also a UDP-based interface. The UDP interface does not provide guaranteed
+delivery, so should only be used for operations that aren't required to
+succeed; typically it is used for "get" requests where a missing or
+incomplete response can simply be treated as a cache miss.
+
+Each UDP datagram contains a simple frame header, followed by data in the
+same format as the TCP protocol described above. In the current
+implementation, requests must be contained in a single UDP datagram, but
+responses may span several datagrams. (The only common requests that would
+span multiple datagrams are huge multi-key "get" requests and "set"
+requests, both of which are more suitable to TCP transport for reliability
+reasons anyway.)
+
+The frame header is 8 bytes long, as follows (all values are 16-bit integers
+in network byte order, high byte first):
+
+0-1 Request ID
+2-3 Sequence number
+4-5 Total number of datagrams in this message
+6-7 Reserved for future use; must be 0
+
+The request ID is supplied by the client. Typically it will be a
+monotonically increasing value starting from a random seed, but the client
+is free to use whatever request IDs it likes. The server's response will
+contain the same ID as the incoming request. The client uses the request ID
+to differentiate between responses to outstanding requests if there are
+several pending from the same server; any datagrams with an unknown request
+ID are probably delayed responses to an earlier request and should be
+discarded.
+
+The sequence number ranges from 0 to n-1, where n is the total number of
+datagrams in the message. The client should concatenate the payloads of the
+datagrams for a given response in sequence number order; the resulting byte
+stream will contain a complete response in the same format as the TCP
+protocol (including terminating \r\n sequences).

items.c

@@ -21,12 +21,7 @@
 #include "memcached.h"
 
 
-/* 
- * NOTE: we assume here for simplicity that slab ids are <=32. That's true in 
- * the powers-of-2 implementation, but if that changes this should be changed too
- */
-
-#define LARGEST_ID 32
+#define LARGEST_ID 255
 static item *heads[LARGEST_ID];
 static item *tails[LARGEST_ID];
 unsigned int sizes[LARGEST_ID];
@@ -41,14 +36,31 @@ void item_init(void) {
 }
 
 
-item *item_alloc(char *key, int flags, time_t exptime, int nbytes) {
-    int ntotal, len;
+/*
+ * Generates the variable-sized part of the header for an object.
+ *
+ * suffix  - Buffer for the "VALUE" line suffix (flags, size).
+ * nsuffix - The length of the suffix is stored here.
+ * keylen  - The length of the key plus any padding required to word-align the
+ *           "VALUE" suffix (which is done to speed up copying.)
+ *
+ * Returns the total size of the header.
+ */
+int item_make_header(char *key, int flags, int nbytes,
+                     char *suffix, int *nsuffix, int *keylen) {
+    *keylen = strlen(key) + 1; if(*keylen % 4) *keylen += 4 - (*keylen % 4);
+    *nsuffix = sprintf(suffix, " %u %u\r\n", flags, nbytes - 2);
+    return sizeof(item) + *keylen + *nsuffix + nbytes;
+}
+ 
+item *item_alloc(char *key, int flags, rel_time_t exptime, int nbytes) {
+    int nsuffix, ntotal, len;
     item *it;
     unsigned int id;
+    char suffix[40];
 
-    len = strlen(key) + 1; if(len % 4) len += 4 - (len % 4);
-    ntotal = sizeof(item) + len + nbytes;
-    
+    ntotal = item_make_header(key, flags, nbytes, suffix, &nsuffix, &len);
+ 
     id = slabs_clsid(ntotal);
     if (id == 0)
         return 0;
@@ -97,7 +109,8 @@ item *item_alloc(char *key, int flags, time_t exptime, int nbytes) {
     it->nbytes = nbytes;
     strcpy(ITEM_key(it), key);
     it->exptime = exptime;
-    it->flags = flags;
+    memcpy(ITEM_suffix(it), suffix, nsuffix);
+    it->nsuffix = nsuffix;
     return it;
 }
 
@@ -114,6 +127,18 @@ void item_free(item *it) {
     slabs_free(it, ntotal);
 }
 
+/*
+ * Returns true if an item will fit in the cache (its size does not exceed
+ * the maximum for a cache entry.)
+ */
+int item_size_ok(char *key, int flags, int nbytes) {
+    char prefix[40];
+    int keylen, nsuffix;
+
+    return slabs_clsid(item_make_header(key, flags, nbytes,
+                                        prefix, &nsuffix, &keylen)) != 0;
+}
+
 void item_link_q(item *it) { /* item is the new head */
     item **head, **tail;
     assert(it->slabs_clsid <= LARGEST_ID);
@@ -159,7 +184,7 @@ int item_link(item *it) {
     assert((it->it_flags & (ITEM_LINKED|ITEM_SLABBED)) == 0);
     assert(it->nbytes < 1048576);
     it->it_flags |= ITEM_LINKED;
-    it->time = time(0);
+    it->time = current_time;
     assoc_insert(ITEM_key(it), it);
 
     stats.curr_bytes += ITEM_ntotal(it);
@@ -195,7 +220,7 @@ void item_update(item *it) {
     assert((it->it_flags & ITEM_SLABBED) == 0);
 
     item_unlink_q(it);
-    it->time = time(0);
+    it->time = current_time;
     item_link_q(it);
 }
 
@@ -224,7 +249,7 @@ char *item_cachedump(unsigned int slabs_clsid, unsigned int limit, unsigned int
     bufcurr = 0;
 
     while (it && (!limit || shown < limit)) {
-        len = sprintf(temp, "ITEM %s [%u b; %lu s]\r\n", ITEM_key(it), it->nbytes - 2, it->time);
+        len = sprintf(temp, "ITEM %s [%u b; %lu s]\r\n", ITEM_key(it), it->nbytes - 2, it->time + stats.started);
         if (bufcurr + len + 6 > memlimit)  /* 6 is END\r\n\0 */
             break;
         strcpy(buffer + bufcurr, temp);
@@ -243,7 +268,7 @@ char *item_cachedump(unsigned int slabs_clsid, unsigned int limit, unsigned int
 void item_stats(char *buffer, int buflen) {
     int i;
     char *bufcurr = buffer;
-    time_t now = time(0);
+    rel_time_t now = current_time;
 
     if (buflen < 4096) {
         strcpy(buffer, "SERVER_ERROR out of memory");
@@ -252,7 +277,7 @@ void item_stats(char *buffer, int buflen) {
 
     for (i=0; i<LARGEST_ID; i++) {
         if (tails[i])
-            bufcurr += sprintf(bufcurr, "STAT items:%u:number %u\r\nSTAT items:%u:age %lu\r\n", 
+            bufcurr += sprintf(bufcurr, "STAT items:%u:number %u\r\nSTAT items:%u:age %u\r\n", 
                                i, sizes[i], i, now - tails[i]->time);
     }
     strcpy(bufcurr, "END");
@@ -262,7 +287,7 @@ void item_stats(char *buffer, int buflen) {
 /* dumps out a list of objects of each size, with granularity of 32 bytes */
 char* item_stats_sizes(int *bytes) {
     int num_buckets = 32768;   /* max 1MB object, divided into 32 bytes size buckets */
-    unsigned int *histogram = (int*) malloc(num_buckets * sizeof(int));
+    unsigned int *histogram = (unsigned int*) malloc(num_buckets * sizeof(int));
     char *buf = (char*) malloc(1024*1024*2*sizeof(char));
     int i;