From a95dd30f0ef82154532fdbfffd97b013289d3231 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Tue, 31 May 2022 12:22:05 +0900 Subject: [PATCH] webclient: Add tunneling support Add a primitive API for tunnel establishment. (WEBCLIENT_FLAG_TUNNEL and webclient_get_tunnel) I plan to use this to implement https proxy support. That is, the primary user will be webclient itself. --- include/netutils/webclient.h | 38 +++++++ netutils/webclient/webclient.c | 202 +++++++++++++++++++++++++-------- 2 files changed, 193 insertions(+), 47 deletions(-) diff --git a/include/netutils/webclient.h b/include/netutils/webclient.h index de0efdf91..8a15ac535 100644 --- a/include/netutils/webclient.h +++ b/include/netutils/webclient.h @@ -103,6 +103,22 @@ #define WEBCLIENT_FLAG_NON_BLOCKING 1U +/* WEBCLIENT_FLAG_TUNNEL: Establish a tunnel + * + * If WEBCLIENT_FLAG_TUNNEL is set, ctx->url is ignored and + * tunnel_target_host and tunnel_target_port members are used instead. + * + * Once a tunnel is established, webclient_perform returns success, + * keeping the tunneled connection open. + * + * After the successful (0-returning) call of webclient_perform, + * the user can use webclient_get_tunnel only once. + * webclient_get_tunnel effectively detaches the returned + * webclient_conn_s from the context. It's users' responsibility + * to dispose the connection. + */ +#define WEBCLIENT_FLAG_TUNNEL 2U + /* The following WEBCLIENT_FLAG_xxx constants are for * webclient_poll_info::flags. */ @@ -350,6 +366,11 @@ struct webclient_context size_t bodylen; unsigned int timeout_sec; + /* Parameters for WEBCLIENT_FLAG_TUNNEL */ + + FAR const char *tunnel_target_host; + uint16_t tunnel_target_port; + /* other parameters * * buffer - A user provided buffer to receive the file data @@ -408,6 +429,7 @@ struct webclient_context WEBCLIENT_CONTEXT_STATE_IN_PROGRESS, WEBCLIENT_CONTEXT_STATE_ABORTED, WEBCLIENT_CONTEXT_STATE_DONE, + WEBCLIENT_CONTEXT_STATE_TUNNEL_ESTABLISHED, } state; #endif }; @@ -420,6 +442,20 @@ struct webclient_poll_info unsigned int flags; /* OR'ed WEBCLIENT_POLL_INFO_xxx flags */ }; +struct webclient_conn_s +{ + bool tls; + + /* for !tls */ + + int sockfd; + unsigned int flags; + + /* for tls */ + + struct webclient_tls_connection *tls_conn; +}; + /**************************************************************************** * Public Function Prototypes ****************************************************************************/ @@ -478,6 +514,8 @@ void webclient_set_static_body(FAR struct webclient_context *ctx, size_t bodylen); int webclient_get_poll_info(FAR struct webclient_context *ctx, FAR struct webclient_poll_info *info); +int webclient_get_tunnel(FAR struct webclient_context *ctx, + FAR struct webclient_conn_s **connp); #undef EXTERN #ifdef __cplusplus diff --git a/netutils/webclient/webclient.c b/netutils/webclient/webclient.c index aeb6aad2d..6ffcd1671 100644 --- a/netutils/webclient/webclient.c +++ b/netutils/webclient/webclient.c @@ -161,22 +161,9 @@ enum webclient_state_e WEBCLIENT_STATE_WAIT_CLOSE, WEBCLIENT_STATE_CLOSE, WEBCLIENT_STATE_DONE, + WEBCLIENT_STATE_TUNNEL_ESTABLISHED, }; -struct conn_s -{ - bool tls; - - /* for !tls */ - - int sockfd; - unsigned int flags; - - /* for tls */ - - struct webclient_tls_connection *tls_conn; -}; - /* flags for wget_s::internal_flags */ #define WGET_FLAG_GOT_CONTENT_LENGTH 1U @@ -225,7 +212,7 @@ struct wget_s struct wget_target_s proxy; bool need_conn_close; - struct conn_s conn; + struct webclient_conn_s *conn; unsigned int nredirect; int redirected; @@ -270,11 +257,22 @@ static const char g_httpcache[] = "Cache-Control: no-cache"; * Private Functions ****************************************************************************/ +/**************************************************************************** + * Name: free_ws + ****************************************************************************/ + +static void free_ws(FAR struct wget_s *ws) +{ + free(ws->conn); + free(ws); +} + /**************************************************************************** * Name: conn_send ****************************************************************************/ -static ssize_t conn_send(struct webclient_context *ctx, struct conn_s *conn, +static ssize_t conn_send(struct webclient_context *ctx, + struct webclient_conn_s *conn, const void *buffer, size_t len) { if (conn->tls) @@ -308,7 +306,8 @@ static ssize_t conn_send(struct webclient_context *ctx, struct conn_s *conn, * Name: conn_recv ****************************************************************************/ -static ssize_t conn_recv(struct webclient_context *ctx, struct conn_s *conn, +static ssize_t conn_recv(struct webclient_context *ctx, + struct webclient_conn_s *conn, void *buffer, size_t len) { if (conn->tls) @@ -342,7 +341,8 @@ static ssize_t conn_recv(struct webclient_context *ctx, struct conn_s *conn, * Name: conn_close ****************************************************************************/ -static void conn_close(struct webclient_context *ctx, struct conn_s *conn) +static void conn_close(struct webclient_context *ctx, + struct webclient_conn_s *conn) { if (conn->tls) { @@ -677,7 +677,24 @@ static inline int wget_parseheaders(struct webclient_context *ctx, } else { - ws->state = WEBCLIENT_STATE_DATA; + if ((ctx->flags & WEBCLIENT_FLAG_TUNNEL) != 0) + { + if (ctx->http_status / 100 == 2) + { + ninfo("Tunnel established\n"); + ws->state = WEBCLIENT_STATE_TUNNEL_ESTABLISHED; + } + else + { + ninfo("HTTP error from tunnelling proxy: %u\n", + ctx->http_status); + ws->state = WEBCLIENT_STATE_DATA; + } + } + else + { + ws->state = WEBCLIENT_STATE_DATA; + } } goto exit; @@ -1111,7 +1128,7 @@ int webclient_perform(FAR struct webclient_context *ctx) struct timeval tv; char *dest; char *ep; - struct conn_s *conn; + struct webclient_conn_s *conn; FAR const struct webclient_tls_ops *tls_ops = ctx->tls_ops; FAR const char *method = ctx->method; FAR void *tls_ctx = ctx->tls_ctx; @@ -1145,6 +1162,14 @@ int webclient_perform(FAR struct webclient_context *ctx) return -errno; } + ws->conn = calloc(1, sizeof(struct webclient_conn_s)); + if (!ws->conn) + { + free_ws(ws); + _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE); + return -errno; + } + ws->buffer = ctx->buffer; ws->buflen = ctx->buflen; @@ -1152,13 +1177,16 @@ int webclient_perform(FAR struct webclient_context *ctx) * from the URL. */ - ret = parseurl(ctx->url, &ws->target, false); - if (ret != 0) + if ((ctx->flags & WEBCLIENT_FLAG_TUNNEL) == 0) { - nwarn("WARNING: Malformed URL: %s\n", ctx->url); - free(ws); - _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE); - return ret; + ret = parseurl(ctx->url, &ws->target, false); + if (ret != 0) + { + nwarn("WARNING: Malformed URL: %s\n", ctx->url); + free_ws(ws); + _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE); + return ret; + } } if (ctx->proxy != NULL) @@ -1173,7 +1201,7 @@ int webclient_perform(FAR struct webclient_context *ctx) if (ret != 0) { nerr("ERROR: Malformed proxy setting: %s\n", ctx->proxy); - free(ws); + free_ws(ws); _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE); return ret; } @@ -1182,7 +1210,7 @@ int webclient_perform(FAR struct webclient_context *ctx) strcmp(ws->proxy.filename, "/")) { nerr("ERROR: Unsupported proxy setting: %s\n", ctx->proxy); - free(ws); + free_ws(ws); _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE); return -ENOTSUP; } @@ -1199,12 +1227,16 @@ int webclient_perform(FAR struct webclient_context *ctx) /* The following sequence may repeat indefinitely if we are redirected */ - conn = &ws->conn; + conn = ws->conn; do { if (ws->state == WEBCLIENT_STATE_SOCKET) { - if (!strcmp(ws->target.scheme, "https") && tls_ops != NULL) + if ((ctx->flags & WEBCLIENT_FLAG_TUNNEL) != 0) + { + conn->tls = false; + } + else if (!strcmp(ws->target.scheme, "https") && tls_ops != NULL) { conn->tls = true; } @@ -1215,7 +1247,7 @@ int webclient_perform(FAR struct webclient_context *ctx) else { nerr("ERROR: unsupported scheme: %s\n", ws->target.scheme); - free(ws); + free_ws(ws); _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE); return -ENOTSUP; } @@ -1237,7 +1269,7 @@ int webclient_perform(FAR struct webclient_context *ctx) if (ctx->unix_socket_path != NULL) { nerr("ERROR: TLS on AF_LOCAL socket is not implemented\n"); - free(ws); + free_ws(ws); _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE); return -ENOTSUP; } @@ -1246,7 +1278,7 @@ int webclient_perform(FAR struct webclient_context *ctx) if (ctx->proxy != NULL) { nerr("ERROR: TLS over proxy is not implemented\n"); - free(ws); + free_ws(ws); _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE); return -ENOTSUP; } @@ -1316,7 +1348,7 @@ int webclient_perform(FAR struct webclient_context *ctx) if (ctx->unix_socket_path != NULL) { nerr("ERROR: TLS on AF_LOCAL socket is not implemented\n"); - free(ws); + free_ws(ws); _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE); return -ENOTSUP; } @@ -1378,7 +1410,7 @@ int webclient_perform(FAR struct webclient_context *ctx) /* Could not resolve host (or malformed IP address) */ nwarn("WARNING: Failed to resolve hostname\n"); - free(ws); + free_ws(ws); _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE); return -EHOSTUNREACH; } @@ -1432,7 +1464,23 @@ int webclient_perform(FAR struct webclient_context *ctx) dest = append(dest, ep, method); dest = append(dest, ep, " "); - if (ctx->proxy != NULL) + if ((ctx->flags & WEBCLIENT_FLAG_TUNNEL) != 0) + { + /* Use authority-form for a tunnel + * + * https://datatracker.ietf.org/doc/html/rfc7231#section-4.3.6 + * https://datatracker.ietf.org/doc/html/rfc7230#section-5.3.3 + */ + + char port_str[sizeof("65535")]; + + dest = append(dest, ep, ctx->tunnel_target_host); + dest = append(dest, ep, ":"); + snprintf(port_str, sizeof(port_str), "%u", + ctx->tunnel_target_port); + dest = append(dest, ep, port_str); + } + else if (ctx->proxy != NULL) { /* Use absolute-form for a proxy * @@ -1542,8 +1590,7 @@ int webclient_perform(FAR struct webclient_context *ctx) { ssize_t ssz; - ssz = conn_send(ctx, conn, - ws->buffer + ws->state_offset, + ssz = conn_send(ctx, conn, ws->buffer + ws->state_offset, ws->state_len); if (ssz < 0) { @@ -1659,10 +1706,22 @@ int webclient_perform(FAR struct webclient_context *ctx) { if (ws->datend - ws->offset == 0) { + size_t want = ws->buflen; ssize_t ssz; ninfo("Reading new data\n"); - ssz = conn_recv(ctx, conn, ws->buffer, ws->buflen); + if ((ctx->flags & WEBCLIENT_FLAG_TUNNEL) != 0) + { + /* When tunnelling, we want to avoid troubles + * with reading the starting payload of the tunnelled + * protocol here, in case it's a server-speaks-first + * protocol. + */ + + want = 1; + } + + ssz = conn_recv(ctx, conn, ws->buffer, want); if (ssz < 0) { ret = ssz; @@ -1863,6 +1922,11 @@ int webclient_perform(FAR struct webclient_context *ctx) break; } } + + if (ws->state == WEBCLIENT_STATE_TUNNEL_ESTABLISHED) + { + break; + } } } @@ -1880,10 +1944,19 @@ int webclient_perform(FAR struct webclient_context *ctx) } } } - while (ws->state != WEBCLIENT_STATE_DONE); + while (ws->state != WEBCLIENT_STATE_DONE && + ws->state != WEBCLIENT_STATE_TUNNEL_ESTABLISHED); + + if (ws->state == WEBCLIENT_STATE_DONE) + { + free_ws(ws); + _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE); + } + else + { + _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_TUNNEL_ESTABLISHED); + } - free(ws); - _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE); return OK; errout_with_errno: @@ -1904,7 +1977,7 @@ errout_with_errno: conn_close(ctx, conn); } - free(ws); + free_ws(ws); _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE); return ret; } @@ -1940,12 +2013,12 @@ void webclient_abort(FAR struct webclient_context *ctx) if (ws->need_conn_close) { - struct conn_s *conn = &ws->conn; + struct webclient_conn_s *conn = ws->conn; conn_close(ctx, conn); } - free(ws); + free_ws(ws); _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_ABORTED); } @@ -2184,7 +2257,7 @@ int webclient_get_poll_info(FAR struct webclient_context *ctx, FAR struct webclient_poll_info *info) { struct wget_s *ws; - struct conn_s *conn; + struct webclient_conn_s *conn; _CHECK_STATE(ctx, WEBCLIENT_CONTEXT_STATE_IN_PROGRESS); DEBUGASSERT((ctx->flags & WEBCLIENT_FLAG_NON_BLOCKING) != 0); @@ -2195,7 +2268,7 @@ int webclient_get_poll_info(FAR struct webclient_context *ctx, return -EINVAL; } - conn = &ws->conn; + conn = ws->conn; if (conn->tls) { return ctx->tls_ops->get_poll_info(ctx->tls_ctx, conn->tls_conn, info); @@ -2206,3 +2279,38 @@ int webclient_get_poll_info(FAR struct webclient_context *ctx, conn->flags &= ~(CONN_WANT_READ | CONN_WANT_WRITE); return 0; } + +/**************************************************************************** + * Name: webclient_get_tunnel + * + * Description: + * This function is used to get the webclient_conn_s, which describes + * the tunneled connection. + * + * This function should be used exactly once after a successful + * call of webclient_perform with WEBCLIENT_FLAG_TUNNEL. + * + * This function also disposes the given webclient_context. + * The context will be invalid after the successful call of this + * function. + * + ****************************************************************************/ + +int webclient_get_tunnel(FAR struct webclient_context *ctx, + FAR struct webclient_conn_s **connp) +{ + struct wget_s *ws; + struct webclient_conn_s *conn; + + _CHECK_STATE(ctx, WEBCLIENT_CONTEXT_STATE_TUNNEL_ESTABLISHED); + ws = ctx->ws; + DEBUGASSERT(ws != NULL); + conn = ws->conn; + DEBUGASSERT(conn != NULL); + *connp = conn; + ws->conn = NULL; + free_ws(ws); + _SET_STATE(ctx, WEBCLIENT_CONTEXT_STATE_DONE); + + return 0; +}