From 93d8b3d1424387f2a5ca264b9b9653c9da21a9a7 Mon Sep 17 00:00:00 2001 From: Zhidao HONG Date: Tue, 11 Jun 2024 17:59:00 +0800 Subject: [PATCH] http: Support chunked request bodies This is a temporary support for chunked request bodies by converting to Content-Length. This allows for processing of such requests until a more permanent solution is developed. A new configuration option "chunked_transform" has been added to enable this feature. The option can be set as follows: { "settings": { "chunked_transform": true } } By default, this option is set to false. Please note that this is an experimental implementation. --- src/nxt_conf_validation.c | 3 + src/nxt_h1proto.c | 184 ++++++++++++++++++++++++++++---------- src/nxt_http.h | 2 + src/nxt_http_request.c | 43 +++++++++ src/nxt_router.c | 7 ++ src/nxt_router.h | 1 + 6 files changed, 192 insertions(+), 48 deletions(-) diff --git a/src/nxt_conf_validation.c b/src/nxt_conf_validation.c index 4aaa1b9a8..f91fc8870 100644 --- a/src/nxt_conf_validation.c +++ b/src/nxt_conf_validation.c @@ -368,6 +368,9 @@ static nxt_conf_vldt_object_t nxt_conf_vldt_http_members[] = { }, { .name = nxt_string("server_version"), .type = NXT_CONF_VLDT_BOOLEAN, + }, { + .name = nxt_string("chunked_transform"), + .type = NXT_CONF_VLDT_BOOLEAN, }, NXT_CONF_VLDT_END diff --git a/src/nxt_h1proto.c b/src/nxt_h1proto.c index 76ba6eda2..8317ea914 100644 --- a/src/nxt_h1proto.c +++ b/src/nxt_h1proto.c @@ -839,7 +839,12 @@ nxt_h1p_transfer_encoding(void *ctx, nxt_http_field_t *field, uintptr_t data) if (field->value_length == 7 && memcmp(field->value, "chunked", 7) == 0) { + if (r->chunked_field != NULL) { + return NXT_HTTP_BAD_REQUEST; + } + te = NXT_HTTP_TE_CHUNKED; + r->chunked_field = field; } else { te = NXT_HTTP_TE_UNSUPPORTED; @@ -856,8 +861,7 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) { size_t size, body_length, body_buffer_size, body_rest; ssize_t res; - nxt_str_t *tmp_path, tmp_name; - nxt_buf_t *in, *b; + nxt_buf_t *in, *b, *out, *chunk; nxt_conn_t *c; nxt_h1proto_t *h1p; nxt_http_status_t status; @@ -869,22 +873,26 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) nxt_debug(task, "h1p request body read %O te:%d", r->content_length_n, h1p->transfer_encoding); - switch (h1p->transfer_encoding) { - - case NXT_HTTP_TE_CHUNKED: - status = NXT_HTTP_LENGTH_REQUIRED; - goto error; - - case NXT_HTTP_TE_UNSUPPORTED: + if (h1p->transfer_encoding == NXT_HTTP_TE_UNSUPPORTED) { status = NXT_HTTP_NOT_IMPLEMENTED; goto error; + } - default: - case NXT_HTTP_TE_NONE: - break; + if (r->chunked_field != NULL) { + if (r->content_length != NULL || !nxt_h1p_is_http11(h1p)) { + status = NXT_HTTP_BAD_REQUEST; + goto error; + } + + if (r->conf->socket_conf->chunked_transform) { + r->chunked = 1; + h1p->chunked_parse.mem_pool = r->mem_pool; + } } - if (r->content_length_n == -1 || r->content_length_n == 0) { + if (!r->chunked && + (r->content_length_n == -1 || r->content_length_n == 0)) + { goto ready; } @@ -893,7 +901,9 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) body_buffer_size = nxt_min(r->conf->socket_conf->body_buffer_size, body_length); - if (body_length > body_buffer_size) { + if (r->chunked || (body_length > body_buffer_size)) { + nxt_str_t *tmp_path, tmp_name; + tmp_path = &r->conf->socket_conf->body_temp_path; tmp_name.length = tmp_path->length + tmp_name_pattern.length; @@ -936,9 +946,6 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) unlink((char *) tmp_name.start); } else { - tmp_path = NULL; - tmp_name.length = 0; - b = nxt_buf_mem_alloc(r->mem_pool, body_buffer_size, 0); if (nxt_slow_path(b == NULL)) { status = NXT_HTTP_INTERNAL_SERVER_ERROR; @@ -948,31 +955,72 @@ nxt_h1p_request_body_read(nxt_task_t *task, nxt_http_request_t *r) r->body = b; - body_rest = body_length; + body_rest = r->chunked ? 1 : body_length; in = h1p->conn->read; size = nxt_buf_mem_used_size(&in->mem); if (size != 0) { - size = nxt_min(size, body_length); - if (nxt_buf_is_file(b)) { - res = nxt_fd_write(b->file->fd, in->mem.pos, size); - if (nxt_slow_path(res < (ssize_t) size)) { - status = NXT_HTTP_INTERNAL_SERVER_ERROR; - goto error; - } + if (r->chunked) { + out = nxt_http_chunk_parse(task, &h1p->chunked_parse, in); - b->file_end += size; + if (h1p->chunked_parse.error) { + status = NXT_HTTP_INTERNAL_SERVER_ERROR; + goto error; + } + + if (h1p->chunked_parse.chunk_error) { + status = NXT_HTTP_BAD_REQUEST; + goto error; + } + + for (chunk = out; chunk != NULL; chunk = chunk->next) { + size = nxt_buf_mem_used_size(&chunk->mem); + + res = nxt_fd_write(b->file->fd, chunk->mem.pos, size); + if (nxt_slow_path(res < (ssize_t) size)) { + status = NXT_HTTP_INTERNAL_SERVER_ERROR; + goto error; + } + + b->file_end += size; + + if ((size_t) b->file_end + > r->conf->socket_conf->max_body_size) + { + status = NXT_HTTP_PAYLOAD_TOO_LARGE; + goto error; + } + } + + if (h1p->chunked_parse.last) { + body_rest = 0; + } + + } else { + size = nxt_min(size, body_length); + res = nxt_fd_write(b->file->fd, in->mem.pos, size); + if (nxt_slow_path(res < (ssize_t) size)) { + status = NXT_HTTP_INTERNAL_SERVER_ERROR; + goto error; + } + + b->file_end += size; + + in->mem.pos += size; + body_rest -= size; + } } else { + size = nxt_min(size, body_length); size = nxt_min(body_buffer_size, size); b->mem.free = nxt_cpymem(b->mem.free, in->mem.pos, size); - } - in->mem.pos += size; - body_rest -= size; + in->mem.pos += size; + body_rest -= size; + } } nxt_debug(task, "h1p body rest: %uz", body_rest); @@ -1030,7 +1078,7 @@ nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data) { size_t size, body_rest; ssize_t res; - nxt_buf_t *b; + nxt_buf_t *b, *out, *chunk; nxt_conn_t *c; nxt_h1proto_t *h1p; nxt_http_request_t *r; @@ -1048,32 +1096,72 @@ nxt_h1p_conn_request_body_read(nxt_task_t *task, void *obj, void *data) b = c->read; if (nxt_buf_is_file(b)) { - body_rest = b->file->size - b->file_end; - size = nxt_buf_mem_used_size(&b->mem); - size = nxt_min(size, body_rest); + if (r->chunked) { + body_rest = 1; - res = nxt_fd_write(b->file->fd, b->mem.pos, size); - if (nxt_slow_path(res < (ssize_t) size)) { - nxt_h1p_request_error(task, h1p, r); - return; - } + out = nxt_http_chunk_parse(task, &h1p->chunked_parse, b); + + if (h1p->chunked_parse.error) { + nxt_h1p_request_error(task, h1p, r); + return; + } - b->file_end += size; - body_rest -= res; + if (h1p->chunked_parse.chunk_error) { + nxt_http_request_error(task, r, NXT_HTTP_BAD_REQUEST); + return; + } - b->mem.pos += size; + for (chunk = out; chunk != NULL; chunk = chunk->next) { + size = nxt_buf_mem_used_size(&chunk->mem); + res = nxt_fd_write(b->file->fd, chunk->mem.pos, size); + if (nxt_slow_path(res < (ssize_t) size)) { + nxt_h1p_request_error(task, h1p, r); + return; + } - if (b->mem.pos == b->mem.free) { - if (body_rest >= (size_t) nxt_buf_mem_size(&b->mem)) { - b->mem.free = b->mem.start; + b->file_end += size; - } else { - /* This required to avoid reading next request. */ - b->mem.free = b->mem.end - body_rest; + if ((size_t) b->file_end + > r->conf->socket_conf->max_body_size) + { + nxt_h1p_request_error(task, h1p, r); + return; + } + } + + if (h1p->chunked_parse.last) { + body_rest = 0; + } + + } else { + body_rest = b->file->size - b->file_end; + + size = nxt_buf_mem_used_size(&b->mem); + size = nxt_min(size, body_rest); + + res = nxt_fd_write(b->file->fd, b->mem.pos, size); + if (nxt_slow_path(res < (ssize_t) size)) { + nxt_h1p_request_error(task, h1p, r); + return; } - b->mem.pos = b->mem.free; + b->file_end += size; + body_rest -= res; + + b->mem.pos += size; + + if (b->mem.pos == b->mem.free) { + if (body_rest >= (size_t) nxt_buf_mem_size(&b->mem)) { + b->mem.free = b->mem.start; + + } else { + /* This required to avoid reading next request. */ + b->mem.free = b->mem.end - body_rest; + } + + b->mem.pos = b->mem.free; + } } } else { diff --git a/src/nxt_http.h b/src/nxt_http.h index 5fab5c672..fe5e72a87 100644 --- a/src/nxt_http.h +++ b/src/nxt_http.h @@ -157,6 +157,7 @@ struct nxt_http_request_s { nxt_list_t *fields; nxt_http_field_t *content_type; nxt_http_field_t *content_length; + nxt_http_field_t *chunked_field; nxt_http_field_t *cookie; nxt_http_field_t *referer; nxt_http_field_t *user_agent; @@ -204,6 +205,7 @@ struct nxt_http_request_s { uint8_t inconsistent; /* 1 bit */ uint8_t error; /* 1 bit */ uint8_t websocket_handshake; /* 1 bit */ + uint8_t chunked; /* 1 bit */ }; diff --git a/src/nxt_http_request.c b/src/nxt_http_request.c index 425a4607f..54d1bd27e 100644 --- a/src/nxt_http_request.c +++ b/src/nxt_http_request.c @@ -540,15 +540,58 @@ static const nxt_http_request_state_t nxt_http_request_body_state }; +static nxt_int_t +nxt_http_request_chunked_transform(nxt_http_request_t *r) +{ + size_t size; + u_char *p, *end; + nxt_http_field_t *f; + + r->chunked_field->skip = 1; + + size = r->body->file_end; + + f = nxt_list_zero_add(r->fields); + if (nxt_slow_path(f == NULL)) { + return NXT_ERROR; + } + + nxt_http_field_name_set(f, "Content-Length"); + + p = nxt_mp_nget(r->mem_pool, NXT_OFF_T_LEN); + if (nxt_slow_path(p == NULL)) { + return NXT_ERROR; + } + + f->value = p; + end = nxt_sprintf(p, p + NXT_OFF_T_LEN, "%uz", size); + f->value_length = end - p; + + r->content_length = f; + r->content_length_n = size; + + return NXT_OK; +} + + static void nxt_http_request_ready(nxt_task_t *task, void *obj, void *data) { + nxt_int_t ret; nxt_http_action_t *action; nxt_http_request_t *r; r = obj; action = r->conf->socket_conf->action; + if (r->chunked) { + ret = nxt_http_request_chunked_transform(r); + if (nxt_slow_path(ret != NXT_OK)) { + nxt_http_request_error(task, r, NXT_HTTP_INTERNAL_SERVER_ERROR); + return; + } + } + nxt_http_request_action(task, r, action); } diff --git a/src/nxt_router.c b/src/nxt_router.c index 48870d206..432094511 100644 --- a/src/nxt_router.c +++ b/src/nxt_router.c @@ -1575,6 +1575,12 @@ static nxt_conf_map_t nxt_router_http_conf[] = { NXT_CONF_MAP_INT8, offsetof(nxt_socket_conf_t, server_version), }, + + { + nxt_string("chunked_transform"), + NXT_CONF_MAP_INT8, + offsetof(nxt_socket_conf_t, chunked_transform), + }, }; @@ -1994,6 +2000,7 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf, skcf->proxy_read_timeout = 30 * 1000; skcf->server_version = 1; + skcf->chunked_transform = 0; skcf->websocket_conf.max_frame_size = 1024 * 1024; skcf->websocket_conf.read_timeout = 60 * 1000; diff --git a/src/nxt_router.h b/src/nxt_router.h index 3e523001c..cfc7258c5 100644 --- a/src/nxt_router.h +++ b/src/nxt_router.h @@ -208,6 +208,7 @@ typedef struct { uint8_t discard_unsafe_fields; /* 1 bit */ uint8_t server_version; /* 1 bit */ + uint8_t chunked_transform; /* 1 bit */ nxt_http_forward_t *forwarded; nxt_http_forward_t *client_ip;