From ef10a9062b3aa4a3ee79a3763fa379b7c92f297f Mon Sep 17 00:00:00 2001 From: syz Date: Sat, 8 Aug 2020 09:57:48 +0800 Subject: [PATCH 1/2] feat: support command cas, stats $memc_value --- README.markdown | 33 ++++++++++++++++++- src/ngx_http_memc_handler.c | 43 ++++++++++++++++++++++-- src/ngx_http_memc_module.h | 4 ++- src/ngx_http_memc_request.c | 60 +++++++++++++++++++++++++++++++++ src/ngx_http_memc_request.h | 2 ++ src/ngx_http_memc_util.c | 4 +-- t/cmd.t | 66 +++++++++++++++++++++++++++++++++++++ t/stats.t | 13 ++++++++ 8 files changed, 217 insertions(+), 8 deletions(-) diff --git a/README.markdown b/README.markdown index 7f82bd1..51ff1a6 100644 --- a/README.markdown +++ b/README.markdown @@ -20,6 +20,7 @@ Table of Contents * [replace $memc_key $memc_flags $memc_exptime $memc_value](#replace-memc_key-memc_flags-memc_exptime-memc_value) * [append $memc_key $memc_flags $memc_exptime $memc_value](#append-memc_key-memc_flags-memc_exptime-memc_value) * [prepend $memc_key $memc_flags $memc_exptime $memc_value](#prepend-memc_key-memc_flags-memc_exptime-memc_value) + * [cas $memc_key $memc_flags $memc_exptime $memc_value $memc_unique_token](#cas-memc_key-memc_flags-memc_exptime-memc_value-memc_unique_token) * [delete $memc_key](#delete-memc_key) * [delete $memc_key $memc_exptime](#delete-memc_key-memc_exptime) * [incr $memc_key $memc_value](#incr-memc_key-memc_value) @@ -27,6 +28,7 @@ Table of Contents * [flush_all](#flush_all) * [flush_all $memc_exptime](#flush_all-memc_exptime) * [stats](#stats) + * [stats $memc_value](#stats-memc_value) * [version](#version) * [Directives](#directives) * [memc_pass](#memc_pass) @@ -324,6 +326,13 @@ Similar to the [append command](#append-memc_key-memc_flags-memc_exptime-memc_va [Back to TOC](#table-of-contents) +cas $memc_key $memc_flags $memc_exptime $memc_value $memc_unique_token +------------------------------------------------------- + +Similar to the [set command](#set-memc_key-memc_flags-memc_exptime-memc_value). + +[Back to TOC](#table-of-contents) + delete $memc_key ---------------- @@ -425,6 +434,28 @@ The raw `stats` command output from the upstream memcached server will be put in [Back to TOC](#table-of-contents) +stats $memc_value +----- + +Causes the memcached server to output general-purpose statistics and settings + +```nginx + + location /foo { + set $memc_cmd stats; + set $memc_value items; + memc_pass 127.0.0.1:11211; + } +``` + +Returns `200 OK` if the request succeeds, or 502 for `ERROR`, `CLIENT_ERROR`, or `SERVER_ERROR`. + +Possible `$memc_value` argument values are `items`, `sizes`, `slabs`, among others. + +The raw `stats` command output from the upstream memcached server will be put into the response body. + +[Back to TOC](#table-of-contents) + version ------- @@ -748,7 +779,7 @@ Some parts of the test suite requires modules [rewrite](http://nginx.org/en/docs TODO ==== -* add support for the memcached commands `cas`, `gets` and `stats $memc_value`. +* add support for the memcached commands `gets`. * add support for the `noreply` option. [Back to TOC](#table-of-contents) diff --git a/src/ngx_http_memc_handler.c b/src/ngx_http_memc_handler.c index 0236242..a86bf6e 100644 --- a/src/ngx_http_memc_handler.c +++ b/src/ngx_http_memc_handler.c @@ -36,6 +36,7 @@ static ngx_str_t ngx_http_memc_cmd = ngx_string("memc_cmd"); static ngx_str_t ngx_http_memc_value = ngx_string("memc_value"); static ngx_str_t ngx_http_memc_flags = ngx_string("memc_flags"); static ngx_str_t ngx_http_memc_exptime = ngx_string("memc_exptime"); +static ngx_str_t ngx_http_memc_unique_token = ngx_string("memc_unique_token"); static ngx_int_t ngx_http_memc_add_more_variables(ngx_conf_t *cf); @@ -68,6 +69,7 @@ ngx_http_memc_handler(ngx_http_request_t *r) ngx_http_variable_value_t *value_vv; ngx_http_variable_value_t *flags_vv; ngx_http_variable_value_t *exptime_vv; + ngx_http_variable_value_t *unique_token_vv; ngx_http_memc_cmd_t memc_cmd; ngx_flag_t is_storage_cmd = 0; @@ -257,9 +259,13 @@ ngx_http_memc_handler(ngx_http_request_t *r) u->input_filter_init = ngx_http_memc_empty_filter_init; u->input_filter = ngx_http_memc_empty_filter; - } else if (memc_cmd == ngx_http_memc_cmd_version - || memc_cmd == ngx_http_memc_cmd_stats) - { + } else if (memc_cmd == ngx_http_memc_cmd_stats) { + u->create_request = ngx_http_memc_create_stats_cmd_request; + u->process_header = ngx_http_memc_process_simple_header; + + u->input_filter_init = ngx_http_memc_empty_filter_init; + u->input_filter = ngx_http_memc_empty_filter; + } else if (memc_cmd == ngx_http_memc_cmd_version) { u->create_request = ngx_http_memc_create_noarg_cmd_request; u->process_header = ngx_http_memc_process_simple_header; @@ -344,6 +350,7 @@ ngx_http_memc_handler(ngx_http_request_t *r) if (is_storage_cmd || memc_cmd == ngx_http_memc_cmd_incr + || memc_cmd == ngx_http_memc_cmd_stats || memc_cmd == ngx_http_memc_cmd_decr) { value_vv = ngx_http_get_indexed_variable(r, mmcf->value_index); @@ -376,6 +383,31 @@ ngx_http_memc_handler(ngx_http_request_t *r) ctx->memc_value_vv = value_vv; } + if (memc_cmd == ngx_http_memc_cmd_cas) { + unique_token_vv = ngx_http_get_indexed_variable(r, mmcf->unique_token_index); + if (unique_token_vv == NULL) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + if (unique_token_vv->not_found || unique_token_vv->len == 0) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "the \"$memc_unique_token\" variable is required for " + "command \"%V\"", &ctx->cmd_str); + + return NGX_HTTP_BAD_REQUEST; + } + + if (!ngx_http_memc_valid_uint32_str(unique_token_vv->data, + unique_token_vv->len)) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "variable \"$memc_unique_token\" takes invalid value: %v", + unique_token_vv); + + return NGX_HTTP_BAD_REQUEST; + } + ctx->memc_unique_token_vv = unique_token_vv; + } + rc = ngx_http_read_client_request_body(r, ngx_http_upstream_init); if (rc == NGX_ERROR || rc > NGX_OK) { @@ -512,6 +544,11 @@ ngx_http_memc_init(ngx_conf_t *cf) return NGX_ERROR; } + mmcf->unique_token_index = ngx_http_memc_add_variable(cf, &ngx_http_memc_unique_token); + if (mmcf->unique_token_index == NGX_ERROR) { + return NGX_ERROR; + } + return ngx_http_memc_add_more_variables(cf); } diff --git a/src/ngx_http_memc_module.h b/src/ngx_http_memc_module.h index bea4578..d4abddf 100644 --- a/src/ngx_http_memc_module.h +++ b/src/ngx_http_memc_module.h @@ -19,7 +19,7 @@ typedef enum { ngx_http_memc_cmd_replace, ngx_http_memc_cmd_append, ngx_http_memc_cmd_prepend, - /* ngx_http_memc_cmd_cas, */ + ngx_http_memc_cmd_cas, ngx_http_memc_cmd_get, /* ngx_http_memc_cmd_gets, */ @@ -57,6 +57,7 @@ typedef struct { ngx_int_t value_index; ngx_int_t flags_index; ngx_int_t exptime_index; + ngx_int_t unique_token_index; ngx_int_t module_used; } ngx_http_memc_main_conf_t; @@ -78,6 +79,7 @@ typedef struct { ngx_http_variable_value_t *memc_key_vv; ngx_http_variable_value_t *memc_flags_vv; ngx_http_variable_value_t *memc_exptime_vv; + ngx_http_variable_value_t *memc_unique_token_vv; ngx_flag_t is_storage_cmd; diff --git a/src/ngx_http_memc_request.c b/src/ngx_http_memc_request.c index b02e322..0944827 100644 --- a/src/ngx_http_memc_request.c +++ b/src/ngx_http_memc_request.c @@ -31,6 +31,7 @@ ngx_http_memc_create_storage_cmd_request(ngx_http_request_t *r) ngx_http_variable_value_t *flags_vv; ngx_http_variable_value_t *exptime_vv; ngx_http_variable_value_t *memc_value_vv; + ngx_http_variable_value_t *unique_token_vv; u_char bytes_buf[NGX_UINT32_LEN]; @@ -124,6 +125,12 @@ ngx_http_memc_create_storage_cmd_request(ngx_http_request_t *r) + bytes_len + sizeof(CRLF) - 1; + + unique_token_vv = ctx->memc_unique_token_vv; + if (ctx->cmd == ngx_http_memc_cmd_cas) { + len += unique_token_vv->len + sizeof(" ") - 1; + } + b = ngx_create_temp_buf(r->pool, len); if (b == NULL) { return NGX_ERROR; @@ -182,6 +189,12 @@ ngx_http_memc_create_storage_cmd_request(ngx_http_request_t *r) b->last = ngx_copy(b->last, bytes_buf, bytes_len); + /* copy cas unique token */ + if (ctx->cmd == ngx_http_memc_cmd_cas) { + *b->last++ = ' '; + b->last = ngx_copy(b->last, unique_token_vv->data, unique_token_vv->len); + } + *b->last++ = CR; *b->last++ = LF; if (memc_value_vv) { @@ -325,6 +338,53 @@ ngx_http_memc_create_get_cmd_request(ngx_http_request_t *r) return NGX_OK; } + +ngx_int_t +ngx_http_memc_create_stats_cmd_request(ngx_http_request_t *r) +{ + size_t len; + ngx_buf_t *b; + ngx_http_memc_ctx_t *ctx; + ngx_chain_t *cl; + ngx_http_variable_value_t *value_vv; + + ctx = ngx_http_get_module_ctx(r, ngx_http_memc_module); + + len = ctx->cmd_str.len + sizeof(CRLF) - 1; + + value_vv = ctx->memc_value_vv; + if (value_vv && value_vv->not_found == 0 && value_vv->len) { + len += value_vv->len + sizeof(" ") - 1; + } + + b = ngx_create_temp_buf(r->pool, len); + if (b == NULL) { + return NGX_ERROR; + } + + cl = ngx_alloc_chain_link(r->pool); + if (cl == NULL) { + return NGX_ERROR; + } + + cl->buf = b; + cl->next = NULL; + + r->upstream->request_bufs = cl; + + b->last = ngx_copy(b->last, ctx->cmd_str.data, ctx->cmd_str.len); + + if (value_vv && value_vv->not_found == 0 && value_vv->len) { + *b->last++ = ' '; + b->last = ngx_copy(b->last, value_vv->data, value_vv->len); + } + + *b->last++ = CR; *b->last++ = LF; + + return NGX_OK; +} + + ngx_int_t ngx_http_memc_create_noarg_cmd_request(ngx_http_request_t *r) { diff --git a/src/ngx_http_memc_request.h b/src/ngx_http_memc_request.h index 10e74c6..999cd88 100644 --- a/src/ngx_http_memc_request.h +++ b/src/ngx_http_memc_request.h @@ -16,6 +16,8 @@ ngx_int_t ngx_http_memc_create_storage_cmd_request(ngx_http_request_t *r); ngx_int_t ngx_http_memc_create_noarg_cmd_request(ngx_http_request_t *r); +ngx_int_t ngx_http_memc_create_stats_cmd_request(ngx_http_request_t *r); + ngx_int_t ngx_http_memc_create_flush_all_cmd_request(ngx_http_request_t *r); ngx_int_t ngx_http_memc_create_delete_cmd_request(ngx_http_request_t *r); diff --git a/src/ngx_http_memc_util.c b/src/ngx_http_memc_util.c index dd19643..bf7b129 100644 --- a/src/ngx_http_memc_util.c +++ b/src/ngx_http_memc_util.c @@ -27,12 +27,10 @@ ngx_http_memc_parse_cmd(u_char *data, size_t len, ngx_flag_t *is_storage_cmd) return ngx_http_memc_cmd_add; } - /* - if (ngx_str3cmp(data, 'c', 'a', 's')) { + if (ngx_http_memc_strcmp_const(data, "cas") == 0) { *is_storage_cmd = 1; return ngx_http_memc_cmd_cas; } - */ if (ngx_http_memc_strcmp_const(data, "get") == 0) { return ngx_http_memc_cmd_get; diff --git a/t/cmd.t b/t/cmd.t index 327a0c7..13cf57e 100644 --- a/t/cmd.t +++ b/t/cmd.t @@ -142,3 +142,69 @@ STORED\r get big2 nice to meet you!" + + +=== TEST 9: cas cmd $memc_unique_token variable dont't set +--- config + location /foo { + set $memc_cmd "cas"; + set $memc_key "test"; + set $memc_value "value"; + memc_pass 127.0.0.1:$TEST_NGINX_MEMCACHED_PORT; + } +--- request + GET /foo +--- response_body_like: 400 Bad Request +--- error_code: 400 + + + +=== TEST 10: cas cmd key not exists. memc response NOT_FOUND +--- config + location /main { + echo 'flush all'; + echo_location '/foo?cmd=flush_all'; + + echo 'cas foo new value'; + echo_location '/foo?key=foo&cmd=cas&val=newvalue&token=123'; + } + location /foo { + set $memc_cmd $arg_cmd; + set $memc_key $arg_key; + set $memc_value $arg_val; + set $memc_unique_token $arg_token; + memc_pass 127.0.0.1:$TEST_NGINX_MEMCACHED_PORT; + } +--- request + GET /main +--- response_body_like +^flush all +OK\r +cas foo new value +.*?404 Not Found.*$ + + + +=== TEST 10: cas cmd key exists. memc response EXISTS +--- config + location /main { + echo 'set foo val ok'; + echo_location '/foo?cmd=set&val=value&key=foo'; + + echo 'cas foo new value'; + echo_location '/foo?key=foo&cmd=cas&val=newvalue&token=123'; + } + location /foo { + set $memc_cmd $arg_cmd; + set $memc_key $arg_key; + set $memc_value $arg_val; + set $memc_unique_token $arg_token; + memc_pass 127.0.0.1:$TEST_NGINX_MEMCACHED_PORT; + } +--- request + GET /main +--- response_body eval +"set foo val ok +STORED\r +cas foo new value +EXISTS\r\n" diff --git a/t/stats.t b/t/stats.t index 7dd5ea7..1d0ed36 100644 --- a/t/stats.t +++ b/t/stats.t @@ -43,3 +43,16 @@ __DATA__ --- timeout: 1 --- error_code: 504 + + +=== TEST 3: stats slabs +--- timeout: 5 +--- config + location /stats { + set $memc_cmd stats; + set $memc_value slabs; + memc_pass 127.0.0.1:$TEST_NGINX_MEMCACHED_PORT; + } +--- request + GET /stats +--- response_body_like: ^(?:STAT [^\r]*\r\n)*END\r\n$ From 2fbf8a7fa637018e74e470a6edc00f1571c9789f Mon Sep 17 00:00:00 2001 From: syz Date: Sat, 29 Aug 2020 14:21:18 +0800 Subject: [PATCH 2/2] style: change code style --- src/ngx_http_memc_handler.c | 5 ++++- src/ngx_http_memc_request.c | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ngx_http_memc_handler.c b/src/ngx_http_memc_handler.c index a86bf6e..7f4ab7e 100644 --- a/src/ngx_http_memc_handler.c +++ b/src/ngx_http_memc_handler.c @@ -265,6 +265,7 @@ ngx_http_memc_handler(ngx_http_request_t *r) u->input_filter_init = ngx_http_memc_empty_filter_init; u->input_filter = ngx_http_memc_empty_filter; + } else if (memc_cmd == ngx_http_memc_cmd_version) { u->create_request = ngx_http_memc_create_noarg_cmd_request; u->process_header = ngx_http_memc_process_simple_header; @@ -398,13 +399,15 @@ ngx_http_memc_handler(ngx_http_request_t *r) } if (!ngx_http_memc_valid_uint32_str(unique_token_vv->data, - unique_token_vv->len)) { + unique_token_vv->len)) + { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "variable \"$memc_unique_token\" takes invalid value: %v", unique_token_vv); return NGX_HTTP_BAD_REQUEST; } + ctx->memc_unique_token_vv = unique_token_vv; } diff --git a/src/ngx_http_memc_request.c b/src/ngx_http_memc_request.c index 0944827..b90bea5 100644 --- a/src/ngx_http_memc_request.c +++ b/src/ngx_http_memc_request.c @@ -125,7 +125,6 @@ ngx_http_memc_create_storage_cmd_request(ngx_http_request_t *r) + bytes_len + sizeof(CRLF) - 1; - unique_token_vv = ctx->memc_unique_token_vv; if (ctx->cmd == ngx_http_memc_cmd_cas) { len += unique_token_vv->len + sizeof(" ") - 1;