upstream: SFTP protocol extension to allow the server to expand

~-prefixed paths, in particular ~user ones. Allows scp in sftp mode to accept
these paths, like scp in rcp mode does.

prompted by and much discussion deraadt@
ok markus@

OpenBSD-Commit-ID: 7d794def9e4de348e1e777f6030fc9bafdfff392

PROTOCOL

@@ -525,6 +525,25 @@ limits.
 This extension is advertised in the SSH_FXP_VERSION hello with version
 "1".
 
+3.9. sftp: Extension request "expand-path@openssh.com"
+
+This request supports canonicalisation of relative paths and
+those that need tilde-expansion, i.e. "~", "~/..." and "~user/..."
+These paths are expanded using shell-like rules and the resultant
+path is canonicalised similarly to SSH2_FXP_REALPATH.
+
+It is implemented as a SSH_FXP_EXTENDED request with the following
+format:
+
+	uint32		id
+	string		"expand-path@openssh.com"
+	string		path
+
+Its reply is the same format as that of SSH2_FXP_REALPATH.
+
+This extension is advertised in the SSH_FXP_VERSION hello with version
+"1".
+
 4. Miscellaneous changes
 
 4.1 Public key format
@@ -556,4 +575,4 @@ OpenSSH's connection multiplexing uses messages as described in
 PROTOCOL.mux over a Unix domain socket for communications between a
 master instance and later clients.
 
-$OpenBSD: PROTOCOL,v 1.41 2021/02/18 02:49:35 djm Exp $
+$OpenBSD: PROTOCOL,v 1.42 2021/08/09 23:47:44 djm Exp $

misc.c

@@ -1,4 +1,4 @@
-/* $OpenBSD: misc.c,v 1.168 2021/07/12 06:22:57 dtucker Exp $ */
+/* $OpenBSD: misc.c,v 1.169 2021/08/09 23:47:44 djm Exp $ */
 /*
  * Copyright (c) 2000 Markus Friedl.  All rights reserved.
  * Copyright (c) 2005-2020 Damien Miller.  All rights reserved.
@@ -1115,29 +1115,37 @@ freeargs(arglist *args)
  * Expands tildes in the file name.  Returns data allocated by xmalloc.
  * Warning: this calls getpw*.
  */
-char *
+int
-tilde_expand_filename(const char *filename, uid_t uid)
+tilde_expand(const char *filename, uid_t uid, char **retp)
 {
 	const char *path, *sep;
 	char user[128], *ret;
 	struct passwd *pw;
 	u_int len, slash;
 
-	if (*filename != '~')
+	if (*filename != '~') {
-		return (xstrdup(filename));
+		*retp = xstrdup(filename);
+		return 0;
+	}
 	filename++;
 
 	path = strchr(filename, '/');
 	if (path != NULL && path > filename) {		/* ~user/path */
 		slash = path - filename;
-		if (slash > sizeof(user) - 1)
+		if (slash > sizeof(user) - 1) {
-			fatal("tilde_expand_filename: ~username too long");
+			error_f("~username too long");
+			return -1;
+		}
 		memcpy(user, filename, slash);
 		user[slash] = '\0';
-		if ((pw = getpwnam(user)) == NULL)
+		if ((pw = getpwnam(user)) == NULL) {
-			fatal("tilde_expand_filename: No such user %s", user);
+			error_f("No such user %s", user);
-	} else if ((pw = getpwuid(uid)) == NULL)	/* ~/path */
+			return -1;
-		fatal("tilde_expand_filename: No such uid %ld", (long)uid);
+		}
+	} else if ((pw = getpwuid(uid)) == NULL) {	/* ~/path */
+		error_f("No such uid %ld", (long)uid);
+		return -1;
+	}
 
 	/* Make sure directory has a trailing '/' */
 	len = strlen(pw->pw_dir);
@@ -1150,10 +1158,23 @@ tilde_expand_filename(const char *filename, uid_t uid)
 	if (path != NULL)
 		filename = path + 1;
 
-	if (xasprintf(&ret, "%s%s%s", pw->pw_dir, sep, filename) >= PATH_MAX)
+	if (xasprintf(&ret, "%s%s%s", pw->pw_dir, sep, filename) >= PATH_MAX) {
-		fatal("tilde_expand_filename: Path too long");
+		error_f("Path too long");
+		return -1;
+	}
 
-	return (ret);
+	*retp = ret;
+	return 0;
+}
+
+char *
+tilde_expand_filename(const char *filename, uid_t uid)
+{
+	char *ret;
+
+	if (tilde_expand(filename, uid, &ret) != 0)
+		cleanup_exit(255);
+	return ret;
 }
 
 /*

misc.h

@@ -1,4 +1,4 @@
-/* $OpenBSD: misc.h,v 1.97 2021/06/08 06:54:40 djm Exp $ */
+/* $OpenBSD: misc.h,v 1.98 2021/08/09 23:47:44 djm Exp $ */
 
 /*
  * Author: Tatu Ylonen <ylo@cs.hut.fi>
@@ -71,6 +71,7 @@ int	 parse_user_host_port(const char *, char **, char **, int *);
 int	 parse_uri(const char *, const char *, char **, char **, int *, char **);
 int	 convtime(const char *);
 const char *fmt_timeframe(time_t t);
+int	 tilde_expand(const char *, uid_t, char **);
 char	*tilde_expand_filename(const char *, uid_t);
 
 char	*dollar_expand(int *, const char *string, ...);

scp.c

@@ -1,4 +1,4 @@
-/* $OpenBSD: scp.c,v 1.226 2021/08/09 23:44:32 djm Exp $ */
+/* $OpenBSD: scp.c,v 1.227 2021/08/09 23:47:44 djm Exp $ */
 /*
  * scp - secure remote copy.  This is basically patched BSD rcp which
  * uses ssh to do the data transfer (instead of using rcmd).
@@ -1255,10 +1255,14 @@ tolocal(int argc, char **argv, enum scp_mode_e mode, char *sftp_direct)
 
 /* Canonicalise a remote path, handling ~ by assuming cwd is the homedir */
 static char *
-absolute_remote_path(const char *path, const char *remote_path)
+absolute_remote_path(struct sftp_conn *conn, const char *path,
+    const char *remote_path)
 {
 	char *ret;
 
+	if (can_expand_path(conn))
+		return do_expand_path(conn, path);
+
 	/* Handle ~ prefixed paths */
 	if (*path != '~')
 		ret = xstrdup(path);
@@ -1296,7 +1300,7 @@ source_sftp(int argc, char *src, char *targ,
 	 * No need to glob here - the local shell already took care of
 	 * the expansions
 	 */
-	if ((target = absolute_remote_path(targ, *remote_path)) == NULL)
+	if ((target = absolute_remote_path(conn, targ, *remote_path)) == NULL)
 		cleanup_exit(255);
 	target_is_dir = remote_is_dir(conn, target);
 	if (targetshouldbedirectory && !target_is_dir) {
@@ -1508,7 +1512,7 @@ sink_sftp(int argc, char *dst, const char *src, struct sftp_conn *conn)
 		goto out;
 	}
 
-	if ((abs_src = absolute_remote_path(src, remote_path)) == NULL) {
+	if ((abs_src = absolute_remote_path(conn, src, remote_path)) == NULL) {
 		err = -1;
 		goto out;
 	}
@@ -1920,8 +1924,9 @@ throughlocal_sftp(struct sftp_conn *from, struct sftp_conn *to,
 	if ((filename = basename(src)) == NULL)
 		fatal("basename %s: %s", src, strerror(errno));
 
-	if ((abs_src = absolute_remote_path(src, from_remote_path)) == NULL ||
+	if ((abs_src = absolute_remote_path(from, src,
-	    (target = absolute_remote_path(targ, *to_remote_path)) == NULL)
+	    from_remote_path)) == NULL ||
+	    (target = absolute_remote_path(to, targ, *to_remote_path)) == NULL)
 		cleanup_exit(255);
 	free(from_remote_path);
 	memset(&g, 0, sizeof(g));

sftp-client.c

@@ -1,4 +1,4 @@
-/* $OpenBSD: sftp-client.c,v 1.153 2021/08/09 07:16:09 djm Exp $ */
+/* $OpenBSD: sftp-client.c,v 1.154 2021/08/09 23:47:44 djm Exp $ */
 /*
  * Copyright (c) 2001-2004 Damien Miller <djm@openbsd.org>
  *
@@ -102,6 +102,7 @@ struct sftp_conn {
 #define SFTP_EXT_FSYNC		0x00000010
 #define SFTP_EXT_LSETSTAT	0x00000020
 #define SFTP_EXT_LIMITS		0x00000040
+#define SFTP_EXT_PATH_EXPAND	0x00000080
 	u_int exts;
 	u_int64_t limit_kbps;
 	struct bwlimit bwlimit_in, bwlimit_out;
@@ -529,6 +530,10 @@ do_init(int fd_in, int fd_out, u_int transfer_buflen, u_int num_requests,
 		    strcmp((char *)value, "1") == 0) {
 			ret->exts |= SFTP_EXT_LIMITS;
 			known = 1;
+		} else if (strcmp(name, "expand-path@openssh.com") == 0 &&
+		    strcmp((char *)value, "1") == 0) {
+			ret->exts |= SFTP_EXT_PATH_EXPAND;
+			known = 1;
 		}
 		if (known) {
 			debug2("Server supports extension \"%s\" revision %s",
@@ -964,8 +969,9 @@ do_fsetstat(struct sftp_conn *conn, const u_char *handle, u_int handle_len,
 	return status == SSH2_FX_OK ? 0 : -1;
 }
 
-char *
+/* Implements both the realpath and expand-path operations */
-do_realpath(struct sftp_conn *conn, const char *path)
+static char *
+do_realpath_expand(struct sftp_conn *conn, const char *path, int expand)
 {
 	struct sshbuf *msg;
 	u_int expected_id, count, id;
@@ -973,14 +979,26 @@ do_realpath(struct sftp_conn *conn, const char *path)
 	Attrib a;
 	u_char type;
 	int r;
+	const char *what = "SSH2_FXP_REALPATH";
 
-	expected_id = id = conn->msg_id++;
+	if (expand)
-	send_string_request(conn, id, SSH2_FXP_REALPATH, path,
+		what = "expand-path@openssh.com";
-	    strlen(path));
-
 	if ((msg = sshbuf_new()) == NULL)
 		fatal_f("sshbuf_new failed");
 
+	expected_id = id = conn->msg_id++;
+	if (expand) {
+		if ((r = sshbuf_put_u8(msg, SSH2_FXP_EXTENDED)) != 0 ||
+		    (r = sshbuf_put_u32(msg, id)) != 0 ||
+		    (r = sshbuf_put_cstring(msg,
+		    "expand-path@openssh.com")) != 0 ||
+		    (r = sshbuf_put_cstring(msg, path)) != 0)
+			fatal_fr(r, "compose %s", what);
+		send_msg(conn, msg);
+	} else {
+		send_string_request(conn, id, SSH2_FXP_REALPATH,
+		    path, strlen(path));
+	}
 	get_msg(conn, msg);
 	if ((r = sshbuf_get_u8(msg, &type)) != 0 ||
 	    (r = sshbuf_get_u32(msg, &id)) != 0)
@@ -1004,15 +1022,14 @@ do_realpath(struct sftp_conn *conn, const char *path)
 	if ((r = sshbuf_get_u32(msg, &count)) != 0)
 		fatal_fr(r, "parse count");
 	if (count != 1)
-		fatal("Got multiple names (%d) from SSH_FXP_REALPATH", count);
+		fatal("Got multiple names (%d) from %s", count, what);
 
 	if ((r = sshbuf_get_cstring(msg, &filename, NULL)) != 0 ||
 	    (r = sshbuf_get_cstring(msg, &longname, NULL)) != 0 ||
 	    (r = decode_attrib(msg, &a)) != 0)
 		fatal_fr(r, "parse filename/attrib");
 
-	debug3("SSH_FXP_REALPATH %s -> %s size %lu", path, filename,
+	debug3("%s %s -> %s", what, path, filename);
-	    (unsigned long)a.size);
 
 	free(longname);
 
@@ -1021,6 +1038,28 @@ do_realpath(struct sftp_conn *conn, const char *path)
 	return(filename);
 }
 
+char *
+do_realpath(struct sftp_conn *conn, const char *path)
+{
+	return do_realpath_expand(conn, path, 0);
+}
+
+int
+can_expand_path(struct sftp_conn *conn)
+{
+	return (conn->exts & SFTP_EXT_PATH_EXPAND) != 0;
+}
+
+char *
+do_expand_path(struct sftp_conn *conn, const char *path)
+{
+	if (!can_expand_path(conn)) {
+		debug3_f("no server support, fallback to realpath");
+		return do_realpath_expand(conn, path, 0);
+	}
+	return do_realpath_expand(conn, path, 1);
+}
+
 int
 do_rename(struct sftp_conn *conn, const char *oldpath, const char *newpath,
     int force_legacy)

sftp-client.h

@@ -1,4 +1,4 @@
-/* $OpenBSD: sftp-client.h,v 1.33 2021/08/07 00:12:09 djm Exp $ */
+/* $OpenBSD: sftp-client.h,v 1.34 2021/08/09 23:47:44 djm Exp $ */
 
 /*
  * Copyright (c) 2001-2004 Damien Miller <djm@openbsd.org>
@@ -113,11 +113,17 @@ int do_lsetstat(struct sftp_conn *conn, const char *path, Attrib *a);
 /* Canonicalise 'path' - caller must free result */
 char *do_realpath(struct sftp_conn *, const char *);
 
+/* Canonicalisation with tilde expansion (requires server extension) */
+char *do_expand_path(struct sftp_conn *, const char *);
+
+/* Returns non-zero if server can tilde-expand paths */
+int can_expand_path(struct sftp_conn *);
+
 /* Get statistics for filesystem hosting file at "path" */
 int do_statvfs(struct sftp_conn *, const char *, struct sftp_statvfs *, int);
 
 /* Rename 'oldpath' to 'newpath' */
-int do_rename(struct sftp_conn *, const char *, const char *, int force_legacy);
+int do_rename(struct sftp_conn *, const char *, const char *, int);
 
 /* Link 'oldpath' to 'newpath' */
 int do_hardlink(struct sftp_conn *, const char *, const char *);

sftp-server.c

@@ -1,4 +1,4 @@
-/* $OpenBSD: sftp-server.c,v 1.128 2021/06/06 03:15:39 djm Exp $ */
+/* $OpenBSD: sftp-server.c,v 1.129 2021/08/09 23:47:44 djm Exp $ */
 /*
  * Copyright (c) 2000-2004 Markus Friedl.  All rights reserved.
  *
@@ -115,6 +115,7 @@ static void process_extended_hardlink(u_int32_t id);
 static void process_extended_fsync(u_int32_t id);
 static void process_extended_lsetstat(u_int32_t id);
 static void process_extended_limits(u_int32_t id);
+static void process_extended_expand(u_int32_t id);
 static void process_extended(u_int32_t id);
 
 struct sftp_handler {
@@ -158,6 +159,8 @@ static const struct sftp_handler extended_handlers[] = {
 	{ "fsync", "fsync@openssh.com", 0, process_extended_fsync, 1 },
 	{ "lsetstat", "lsetstat@openssh.com", 0, process_extended_lsetstat, 1 },
 	{ "limits", "limits@openssh.com", 0, process_extended_limits, 0 },
+	{ "expand-path", "expand-path@openssh.com", 0,
+	    process_extended_expand, 0 },
 	{ NULL, NULL, 0, NULL, 0 }
 };
 
@@ -706,6 +709,7 @@ process_init(void)
 	compose_extension(msg, "fsync@openssh.com", "1");
 	compose_extension(msg, "lsetstat@openssh.com", "1");
 	compose_extension(msg, "limits@openssh.com", "1");
+	compose_extension(msg, "expand-path@openssh.com", "1");
 
 	send_msg(msg);
 	sshbuf_free(msg);
@@ -1519,6 +1523,63 @@ process_extended_limits(u_int32_t id)
 	sshbuf_free(msg);
 }
 
+static void
+process_extended_expand(u_int32_t id)
+{
+	char cwd[PATH_MAX], resolvedname[PATH_MAX];
+	char *path, *npath;
+	int r;
+	Stat s;
+
+	if ((r = sshbuf_get_cstring(iqueue, &path, NULL)) != 0)
+		fatal_fr(r, "parse");
+	if (getcwd(cwd, sizeof(cwd)) == NULL) {
+		send_status(id, errno_to_portable(errno));
+		goto out;
+	}
+
+	debug3("request %u: expand, original \"%s\"", id, path);
+	if (path[0] == '\0') {
+		/* empty path */
+		free(path);
+		path = xstrdup(".");
+	} else if (*path == '~') {
+		/* ~ expand path */
+		/* Special-case for "~" and "~/" to respect homedir flag */
+		if (strcmp(path, "~") == 0) {
+			free(path);
+			path = xstrdup(cwd);
+		} else if (strncmp(path, "~/", 2) == 0) {
+			npath = xstrdup(path + 2);
+			free(path);
+			xasprintf(&path, "%s/%s", cwd, npath);
+		} else {
+			/* ~user expansions */
+			if (tilde_expand(path, pw->pw_uid, &npath) != 0) {
+				send_status(id, errno_to_portable(EINVAL));
+				goto out;
+			}
+			free(path);
+			path = npath;
+		}
+	} else if (*path != '/') {
+		/* relative path */
+		xasprintf(&npath, "%s/%s", cwd, path);
+		free(path);
+		path = npath;
+	}
+	verbose("expand \"%s\"", path);
+	if (sftp_realpath(path, resolvedname) == NULL) {
+		send_status(id, errno_to_portable(errno));
+		goto out;
+	}
+	attrib_clear(&s.attrib);
+	s.name = s.long_name = resolvedname;
+	send_names(id, 1, &s);
+ out:
+	free(path);
+}
+
 static void
 process_extended(u_int32_t id)
 {