Merge branch 'ta/fast-import-parse-path-fix'

The way "git fast-import" handles paths described in its input has
been tightened up and more clearly documented.

* ta/fast-import-parse-path-fix:
  fast-import: make comments more precise
  fast-import: forbid escaped NUL in paths
  fast-import: document C-style escapes for paths
  fast-import: improve documentation for path quoting
  fast-import: remove dead strbuf
  fast-import: allow unquoted empty path for root
  fast-import: directly use strbufs for paths
  fast-import: tighten path unquoting

3 files changed, 560 insertions, 267 deletions

diff --git a/Documentation/git-fast-import.txt b/Documentation/git-fast-import.txt
index b2607366b9..8b6dde45f1 100644
--- a/Documentation/git-fast-import.txt
+++ b/Documentation/git-fast-import.txt
@@ -630,18 +630,28 @@ in octal.  Git only supports the following modes:
 In both formats `<path>` is the complete path of the file to be added
 (if not already existing) or modified (if already existing).
 
-A `<path>` string must use UNIX-style directory separators (forward
-slash `/`), may contain any byte other than `LF`, and must not
-start with double quote (`"`).
-
-A path can use C-style string quoting; this is accepted in all cases
-and mandatory if the filename starts with double quote or contains
-`LF`. In C-style quoting, the complete name should be surrounded with
-double quotes, and any `LF`, backslash, or double quote characters
-must be escaped by preceding them with a backslash (e.g.,
-`"path/with\n, \\ and \" in it"`).
-
-The value of `<path>` must be in canonical form. That is it must not:
+A `<path>` can be written as unquoted bytes or a C-style quoted string.
+
+When a `<path>` does not start with a double quote (`"`), it is an
+unquoted string and is parsed as literal bytes without any escape
+sequences. However, if the filename contains `LF` or starts with double
+quote, it cannot be represented as an unquoted string and must be
+quoted. Additionally, the source `<path>` in `filecopy` or `filerename`
+must be quoted if it contains SP.
+
+When a `<path>` starts with a double quote (`"`), it is a C-style quoted
+string, where the complete filename is enclosed in a pair of double
+quotes and escape sequences are used. Certain characters must be escaped
+by preceding them with a backslash: `LF` is written as `\n`, backslash
+as `\\`, and double quote as `\"`. Some characters may optionally be
+written with escape sequences: `\a` for bell, `\b` for backspace, `\f`
+for form feed, `\n` for line feed, `\r` for carriage return, `\t` for
+horizontal tab, and `\v` for vertical tab. Any byte can be written with
+3-digit octal codes (e.g., `\033`). All filenames can be represented as
+quoted strings.
+
+A `<path>` must use UNIX-style directory separators (forward slash `/`)
+and its value must be in canonical form. That is it must not:
 
 * contain an empty directory component (e.g. `foo//bar` is invalid),
 * end with a directory separator (e.g. `foo/` is invalid),
@@ -651,6 +661,7 @@ The value of `<path>` must be in canonical form. That is it must not:
 
 The root of the tree can be represented by an empty string as `<path>`.
 
+`<path>` cannot contain NUL, either literally or escaped as `\000`.
 It is recommended that `<path>` always be encoded using UTF-8.
 
 `filedelete`
diff --git a/builtin/fast-import.c b/builtin/fast-import.c
index 782bda007c..dc5a9d32dd 100644
--- a/builtin/fast-import.c
+++ b/builtin/fast-import.c
@@ -2210,7 +2210,7 @@ static int parse_mapped_oid_hex(const char *hex, struct object_id *oid, const ch
  *
  *   idnum ::= ':' bigint;
  *
- * Return the first character after the value in *endptr.
+ * Update *endptr to point to the first character after the value.
  *
  * Complain if the following character is not what is expected,
  * either a space or end of the string.
@@ -2243,8 +2243,8 @@ static uintmax_t parse_mark_ref_eol(const char *p)
 }
 
 /*
- * Parse the mark reference, demanding a trailing space.  Return a
- * pointer to the space.
+ * Parse the mark reference, demanding a trailing space. Update *p to
+ * point to the first character after the space.
  */
 static uintmax_t parse_mark_ref_space(const char **p)
 {
@@ -2258,10 +2258,62 @@ static uintmax_t parse_mark_ref_space(const char **p)
 	return mark;
 }
 
+/*
+ * Parse the path string into the strbuf. The path can either be quoted with
+ * escape sequences or unquoted without escape sequences. Unquoted strings may
+ * contain spaces only if `is_last_field` is nonzero; otherwise, it stops
+ * parsing at the first space.
+ */
+static void parse_path(struct strbuf *sb, const char *p, const char **endp,
+		int is_last_field, const char *field)
+{
+	if (*p == '"') {
+		if (unquote_c_style(sb, p, endp))
+			die("Invalid %s: %s", field, command_buf.buf);
+		if (strlen(sb->buf) != sb->len)
+			die("NUL in %s: %s", field, command_buf.buf);
+	} else {
+		/*
+		 * Unless we are parsing the last field of a line,
+		 * SP is the end of this field.
+		 */
+		*endp = is_last_field
+			? p + strlen(p)
+			: strchrnul(p, ' ');
+		strbuf_add(sb, p, *endp - p);
+	}
+}
+
+/*
+ * Parse the path string into the strbuf, and complain if this is not the end of
+ * the string. Unquoted strings may contain spaces.
+ */
+static void parse_path_eol(struct strbuf *sb, const char *p, const char *field)
+{
+	const char *end;
+
+	parse_path(sb, p, &end, 1, field);
+	if (*end)
+		die("Garbage after %s: %s", field, command_buf.buf);
+}
+
+/*
+ * Parse the path string into the strbuf, and ensure it is followed by a space.
+ * Unquoted strings may not contain spaces. Update *endp to point to the first
+ * character after the space.
+ */
+static void parse_path_space(struct strbuf *sb, const char *p,
+		const char **endp, const char *field)
+{
+	parse_path(sb, p, endp, 0, field);
+	if (**endp != ' ')
+		die("Missing space after %s: %s", field, command_buf.buf);
+	(*endp)++;
+}
+
 static void file_change_m(const char *p, struct branch *b)
 {
-	static struct strbuf uq = STRBUF_INIT;
-	const char *endp;
+	static struct strbuf path = STRBUF_INIT;
 	struct object_entry *oe;
 	struct object_id oid;
 	uint16_t mode, inline_data = 0;
@@ -2298,16 +2350,12 @@ static void file_change_m(const char *p, struct branch *b)
 			die("Missing space after SHA1: %s", command_buf.buf);
 	}
 
-	strbuf_reset(&uq);
-	if (!unquote_c_style(&uq, p, &endp)) {
-		if (*endp)
-			die("Garbage after path in: %s", command_buf.buf);
-		p = uq.buf;
-	}
+	strbuf_reset(&path);
+	parse_path_eol(&path, p, "path");
 
 	/* Git does not track empty, non-toplevel directories. */
-	if (S_ISDIR(mode) && is_empty_tree_oid(&oid) && *p) {
-		tree_content_remove(&b->branch_tree, p, NULL, 0);
+	if (S_ISDIR(mode) && is_empty_tree_oid(&oid) && *path.buf) {
+		tree_content_remove(&b->branch_tree, path.buf, NULL, 0);
 		return;
 	}
 
@@ -2328,10 +2376,6 @@ static void file_change_m(const char *p, struct branch *b)
 		if (S_ISDIR(mode))
 			die("Directories cannot be specified 'inline': %s",
 				command_buf.buf);
-		if (p != uq.buf) {
-			strbuf_addstr(&uq, p);
-			p = uq.buf;
-		}
 		while (read_next_command() != EOF) {
 			const char *v;
 			if (skip_prefix(command_buf.buf, "cat-blob ", &v))
@@ -2357,74 +2401,48 @@ static void file_change_m(const char *p, struct branch *b)
 				command_buf.buf);
 	}
 
-	if (!*p) {
+	if (!*path.buf) {
 		tree_content_replace(&b->branch_tree, &oid, mode, NULL);
 		return;
 	}
-	tree_content_set(&b->branch_tree, p, &oid, mode, NULL);
+	tree_content_set(&b->branch_tree, path.buf, &oid, mode, NULL);
 }
 
 static void file_change_d(const char *p, struct branch *b)
 {
-	static struct strbuf uq = STRBUF_INIT;
-	const char *endp;
+	static struct strbuf path = STRBUF_INIT;
 
-	strbuf_reset(&uq);
-	if (!unquote_c_style(&uq, p, &endp)) {
-		if (*endp)
-			die("Garbage after path in: %s", command_buf.buf);
-		p = uq.buf;
-	}
-	tree_content_remove(&b->branch_tree, p, NULL, 1);
+	strbuf_reset(&path);
+	parse_path_eol(&path, p, "path");
+	tree_content_remove(&b->branch_tree, path.buf, NULL, 1);
 }
 
-static void file_change_cr(const char *s, struct branch *b, int rename)
+static void file_change_cr(const char *p, struct branch *b, int rename)
 {
-	const char *d;
-	static struct strbuf s_uq = STRBUF_INIT;
-	static struct strbuf d_uq = STRBUF_INIT;
-	const char *endp;
+	static struct strbuf source = STRBUF_INIT;
+	static struct strbuf dest = STRBUF_INIT;
 	struct tree_entry leaf;
 
-	strbuf_reset(&s_uq);
-	if (!unquote_c_style(&s_uq, s, &endp)) {
-		if (*endp != ' ')
-			die("Missing space after source: %s", command_buf.buf);
-	} else {
-		endp = strchr(s, ' ');
-		if (!endp)
-			die("Missing space after source: %s", command_buf.buf);
-		strbuf_add(&s_uq, s, endp - s);
-	}
-	s = s_uq.buf;
-
-	endp++;
-	if (!*endp)
-		die("Missing dest: %s", command_buf.buf);
-
-	d = endp;
-	strbuf_reset(&d_uq);
-	if (!unquote_c_style(&d_uq, d, &endp)) {
-		if (*endp)
-			die("Garbage after dest in: %s", command_buf.buf);
-		d = d_uq.buf;
-	}
+	strbuf_reset(&source);
+	parse_path_space(&source, p, &p, "source");
+	strbuf_reset(&dest);
+	parse_path_eol(&dest, p, "dest");
 
 	memset(&leaf, 0, sizeof(leaf));
 	if (rename)
-		tree_content_remove(&b->branch_tree, s, &leaf, 1);
+		tree_content_remove(&b->branch_tree, source.buf, &leaf, 1);
 	else
-		tree_content_get(&b->branch_tree, s, &leaf, 1);
+		tree_content_get(&b->branch_tree, source.buf, &leaf, 1);
 	if (!leaf.versions[1].mode)
-		die("Path %s not in branch", s);
-	if (!*d) {	/* C "path/to/subdir" "" */
+		die("Path %s not in branch", source.buf);
+	if (!*dest.buf) {	/* C "path/to/subdir" "" */
 		tree_content_replace(&b->branch_tree,
 			&leaf.versions[1].oid,
 			leaf.versions[1].mode,
 			leaf.tree);
 		return;
 	}
-	tree_content_set(&b->branch_tree, d,
+	tree_content_set(&b->branch_tree, dest.buf,
 		&leaf.versions[1].oid,
 		leaf.versions[1].mode,
 		leaf.tree);
@@ -2432,7 +2450,6 @@ static void file_change_cr(const char *s, struct branch *b, int rename)
 
 static void note_change_n(const char *p, struct branch *b, unsigned char *old_fanout)
 {
-	static struct strbuf uq = STRBUF_INIT;
 	struct object_entry *oe;
 	struct branch *s;
 	struct object_id oid, commit_oid;
@@ -2497,10 +2514,6 @@ static void note_change_n(const char *p, struct branch *b, unsigned char *old_fa
 		die("Invalid ref name or SHA1 expression: %s", p);
 
 	if (inline_data) {
-		if (p != uq.buf) {
-			strbuf_addstr(&uq, p);
-			p = uq.buf;
-		}
 		read_next_command();
 		parse_and_store_blob(&last_blob, &oid, 0);
 	} else if (oe) {
@@ -3152,6 +3165,7 @@ static void print_ls(int mode, const unsigned char *hash, const char *path)
 
 static void parse_ls(const char *p, struct branch *b)
 {
+	static struct strbuf path = STRBUF_INIT;
 	struct tree_entry *root = NULL;
 	struct tree_entry leaf = {NULL};
 
@@ -3168,17 +3182,9 @@ static void parse_ls(const char *p, struct branch *b)
 			root->versions[1].mode = S_IFDIR;
 		load_tree(root);
 	}
-	if (*p == '"') {
-		static struct strbuf uq = STRBUF_INIT;
-		const char *endp;
-		strbuf_reset(&uq);
-		if (unquote_c_style(&uq, p, &endp))
-			die("Invalid path: %s", command_buf.buf);
-		if (*endp)
-			die("Garbage after path in: %s", command_buf.buf);
-		p = uq.buf;
-	}
-	tree_content_get(root, p, &leaf, 1);
+	strbuf_reset(&path);
+	parse_path_eol(&path, p, "path");
+	tree_content_get(root, path.buf, &leaf, 1);
 	/*
 	 * A directory in preparation would have a sha1 of zero
 	 * until it is saved.  Save, for simplicity.
@@ -3186,7 +3192,7 @@ static void parse_ls(const char *p, struct branch *b)
 	if (S_ISDIR(leaf.versions[1].mode))
 		store_tree(&leaf);
 
-	print_ls(leaf.versions[1].mode, leaf.versions[1].oid.hash, p);
+	print_ls(leaf.versions[1].mode, leaf.versions[1].oid.hash, path.buf);
 	if (leaf.tree)
 		release_tree_content_recursive(leaf.tree);
 	if (!b || root != &b->branch_tree)
diff --git a/t/t9300-fast-import.sh b/t/t9300-fast-import.sh
index 60e30fed3c..1e68426852 100755
--- a/t/t9300-fast-import.sh
+++ b/t/t9300-fast-import.sh
@@ -1059,30 +1059,33 @@ test_expect_success 'M: rename subdirectory to new subdirectory' '
 	compare_diff_raw expect actual
 '
 
-test_expect_success 'M: rename root to subdirectory' '
-	cat >input <<-INPUT_END &&
-	commit refs/heads/M4
-	committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
-	data <<COMMIT
-	rename root
-	COMMIT
+for root in '""' ''
+do
+	test_expect_success "M: rename root ($root) to subdirectory" '
+		cat >input <<-INPUT_END &&
+		commit refs/heads/M4
+		committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+		data <<COMMIT
+		rename root
+		COMMIT
 
-	from refs/heads/M2^0
-	R "" sub
+		from refs/heads/M2^0
+		R $root sub
 
-	INPUT_END
+		INPUT_END
 
-	cat >expect <<-EOF &&
-	:100644 100644 $oldf $oldf R100	file2/oldf	sub/file2/oldf
-	:100755 100755 $f4id $f4id R100	file4	sub/file4
-	:100755 100755 $newf $newf R100	i/am/new/to/you	sub/i/am/new/to/you
-	:100755 100755 $f6id $f6id R100	newdir/exec.sh	sub/newdir/exec.sh
-	:100644 100644 $f5id $f5id R100	newdir/interesting	sub/newdir/interesting
-	EOF
-	git fast-import <input &&
-	git diff-tree -M -r M4^ M4 >actual &&
-	compare_diff_raw expect actual
-'
+		cat >expect <<-EOF &&
+		:100644 100644 $oldf $oldf R100	file2/oldf	sub/file2/oldf
+		:100755 100755 $f4id $f4id R100	file4	sub/file4
+		:100755 100755 $newf $newf R100	i/am/new/to/you	sub/i/am/new/to/you
+		:100755 100755 $f6id $f6id R100	newdir/exec.sh	sub/newdir/exec.sh
+		:100644 100644 $f5id $f5id R100	newdir/interesting	sub/newdir/interesting
+		EOF
+		git fast-import <input &&
+		git diff-tree -M -r M4^ M4 >actual &&
+		compare_diff_raw expect actual
+	'
+done
 
 ###
 ### series N
@@ -1259,49 +1262,52 @@ test_expect_success PIPE 'N: empty directory reads as missing' '
 	test_cmp expect actual
 '
 
-test_expect_success 'N: copy root directory by tree hash' '
-	cat >expect <<-EOF &&
-	:100755 000000 $newf $zero D	file3/newf
-	:100644 000000 $oldf $zero D	file3/oldf
-	EOF
-	root=$(git rev-parse refs/heads/branch^0^{tree}) &&
-	cat >input <<-INPUT_END &&
-	commit refs/heads/N6
-	committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
-	data <<COMMIT
-	copy root directory by tree hash
-	COMMIT
-
-	from refs/heads/branch^0
-	M 040000 $root ""
-	INPUT_END
-	git fast-import <input &&
-	git diff-tree -C --find-copies-harder -r N4 N6 >actual &&
-	compare_diff_raw expect actual
-'
+for root in '""' ''
+do
+	test_expect_success "N: copy root ($root) by tree hash" '
+		cat >expect <<-EOF &&
+		:100755 000000 $newf $zero D	file3/newf
+		:100644 000000 $oldf $zero D	file3/oldf
+		EOF
+		root_tree=$(git rev-parse refs/heads/branch^0^{tree}) &&
+		cat >input <<-INPUT_END &&
+		commit refs/heads/N6
+		committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+		data <<COMMIT
+		copy root directory by tree hash
+		COMMIT
 
-test_expect_success 'N: copy root by path' '
-	cat >expect <<-EOF &&
-	:100755 100755 $newf $newf C100	file2/newf	oldroot/file2/newf
-	:100644 100644 $oldf $oldf C100	file2/oldf	oldroot/file2/oldf
-	:100755 100755 $f4id $f4id C100	file4	oldroot/file4
-	:100755 100755 $f6id $f6id C100	newdir/exec.sh	oldroot/newdir/exec.sh
-	:100644 100644 $f5id $f5id C100	newdir/interesting	oldroot/newdir/interesting
-	EOF
-	cat >input <<-INPUT_END &&
-	commit refs/heads/N-copy-root-path
-	committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
-	data <<COMMIT
-	copy root directory by (empty) path
-	COMMIT
+		from refs/heads/branch^0
+		M 040000 $root_tree $root
+		INPUT_END
+		git fast-import <input &&
+		git diff-tree -C --find-copies-harder -r N4 N6 >actual &&
+		compare_diff_raw expect actual
+	'
+
+	test_expect_success "N: copy root ($root) by path" '
+		cat >expect <<-EOF &&
+		:100755 100755 $newf $newf C100	file2/newf	oldroot/file2/newf
+		:100644 100644 $oldf $oldf C100	file2/oldf	oldroot/file2/oldf
+		:100755 100755 $f4id $f4id C100	file4	oldroot/file4
+		:100755 100755 $f6id $f6id C100	newdir/exec.sh	oldroot/newdir/exec.sh
+		:100644 100644 $f5id $f5id C100	newdir/interesting	oldroot/newdir/interesting
+		EOF
+		cat >input <<-INPUT_END &&
+		commit refs/heads/N-copy-root-path
+		committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+		data <<COMMIT
+		copy root directory by (empty) path
+		COMMIT
 
-	from refs/heads/branch^0
-	C "" oldroot
-	INPUT_END
-	git fast-import <input &&
-	git diff-tree -C --find-copies-harder -r branch N-copy-root-path >actual &&
-	compare_diff_raw expect actual
-'
+		from refs/heads/branch^0
+		C $root oldroot
+		INPUT_END
+		git fast-import <input &&
+		git diff-tree -C --find-copies-harder -r branch N-copy-root-path >actual &&
+		compare_diff_raw expect actual
+	'
+done
 
 test_expect_success 'N: delete directory by copying' '
 	cat >expect <<-\EOF &&
@@ -1431,98 +1437,102 @@ test_expect_success 'N: reject foo/ syntax in ls argument' '
 	INPUT_END
 '
 
-test_expect_success 'N: copy to root by id and modify' '
-	echo "hello, world" >expect.foo &&
-	echo hello >expect.bar &&
-	git fast-import <<-SETUP_END &&
-	commit refs/heads/N7
-	committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
-	data <<COMMIT
-	hello, tree
-	COMMIT
-
-	deleteall
-	M 644 inline foo/bar
-	data <<EOF
-	hello
-	EOF
-	SETUP_END
-
-	tree=$(git rev-parse --verify N7:) &&
-	git fast-import <<-INPUT_END &&
-	commit refs/heads/N8
-	committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
-	data <<COMMIT
-	copy to root by id and modify
-	COMMIT
+for root in '""' ''
+do
+	test_expect_success "N: copy to root ($root) by id and modify" '
+		echo "hello, world" >expect.foo &&
+		echo hello >expect.bar &&
+		git fast-import <<-SETUP_END &&
+		commit refs/heads/N7
+		committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+		data <<COMMIT
+		hello, tree
+		COMMIT
 
-	M 040000 $tree ""
-	M 644 inline foo/foo
-	data <<EOF
-	hello, world
-	EOF
-	INPUT_END
-	git show N8:foo/foo >actual.foo &&
-	git show N8:foo/bar >actual.bar &&
-	test_cmp expect.foo actual.foo &&
-	test_cmp expect.bar actual.bar
-'
+		deleteall
+		M 644 inline foo/bar
+		data <<EOF
+		hello
+		EOF
+		SETUP_END
 
-test_expect_success 'N: extract subtree' '
-	branch=$(git rev-parse --verify refs/heads/branch^{tree}) &&
-	cat >input <<-INPUT_END &&
-	commit refs/heads/N9
-	committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
-	data <<COMMIT
-	extract subtree branch:newdir
-	COMMIT
+		tree=$(git rev-parse --verify N7:) &&
+		git fast-import <<-INPUT_END &&
+		commit refs/heads/N8
+		committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+		data <<COMMIT
+		copy to root by id and modify
+		COMMIT
 
-	M 040000 $branch ""
-	C "newdir" ""
-	INPUT_END
-	git fast-import <input &&
-	git diff --exit-code branch:newdir N9
-'
+		M 040000 $tree $root
+		M 644 inline foo/foo
+		data <<EOF
+		hello, world
+		EOF
+		INPUT_END
+		git show N8:foo/foo >actual.foo &&
+		git show N8:foo/bar >actual.bar &&
+		test_cmp expect.foo actual.foo &&
+		test_cmp expect.bar actual.bar
+	'
+
+	test_expect_success "N: extract subtree to the root ($root)" '
+		branch=$(git rev-parse --verify refs/heads/branch^{tree}) &&
+		cat >input <<-INPUT_END &&
+		commit refs/heads/N9
+		committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+		data <<COMMIT
+		extract subtree branch:newdir
+		COMMIT
 
-test_expect_success 'N: modify subtree, extract it, and modify again' '
-	echo hello >expect.baz &&
-	echo hello, world >expect.qux &&
-	git fast-import <<-SETUP_END &&
-	commit refs/heads/N10
-	committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
-	data <<COMMIT
-	hello, tree
-	COMMIT
+		M 040000 $branch $root
+		C "newdir" $root
+		INPUT_END
+		git fast-import <input &&
+		git diff --exit-code branch:newdir N9
+	'
+
+	test_expect_success "N: modify subtree, extract it to the root ($root), and modify again" '
+		echo hello >expect.baz &&
+		echo hello, world >expect.qux &&
+		git fast-import <<-SETUP_END &&
+		commit refs/heads/N10
+		committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+		data <<COMMIT
+		hello, tree
+		COMMIT
 
-	deleteall
-	M 644 inline foo/bar/baz
-	data <<EOF
-	hello
-	EOF
-	SETUP_END
+		deleteall
+		M 644 inline foo/bar/baz
+		data <<EOF
+		hello
+		EOF
+		SETUP_END
 
-	tree=$(git rev-parse --verify N10:) &&
-	git fast-import <<-INPUT_END &&
-	commit refs/heads/N11
-	committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
-	data <<COMMIT
-	copy to root by id and modify
-	COMMIT
+		tree=$(git rev-parse --verify N10:) &&
+		git fast-import <<-INPUT_END &&
+		commit refs/heads/N11
+		committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+		data <<COMMIT
+		copy to root by id and modify
+		COMMIT
 
-	M 040000 $tree ""
-	M 100644 inline foo/bar/qux
-	data <<EOF
-	hello, world
-	EOF
-	R "foo" ""
-	C "bar/qux" "bar/quux"
-	INPUT_END
-	git show N11:bar/baz >actual.baz &&
-	git show N11:bar/qux >actual.qux &&
-	git show N11:bar/quux >actual.quux &&
-	test_cmp expect.baz actual.baz &&
-	test_cmp expect.qux actual.qux &&
-	test_cmp expect.qux actual.quux'
+		M 040000 $tree $root
+		M 100644 inline foo/bar/qux
+		data <<EOF
+		hello, world
+		EOF
+		R "foo" $root
+		C "bar/qux" "bar/quux"
+		INPUT_END
+		git show N11:bar/baz >actual.baz &&
+		git show N11:bar/qux >actual.qux &&
+		git show N11:bar/quux >actual.quux &&
+		test_cmp expect.baz actual.baz &&
+		test_cmp expect.qux actual.qux &&
+		test_cmp expect.qux actual.quux
+	'
+done
 
 ###
 ### series O
@@ -2142,6 +2152,7 @@ test_expect_success 'Q: deny note on empty branch' '
 	EOF
 	test_must_fail git fast-import <input
 '
+
 ###
 ### series R (feature and option)
 ###
@@ -2790,7 +2801,7 @@ test_expect_success 'R: blob appears only once' '
 '
 
 ###
-### series S
+### series S (mark and path parsing)
 ###
 #
 # Make sure missing spaces and EOLs after mark references
@@ -3060,21 +3071,283 @@ test_expect_success 'S: ls with garbage after sha1 must fail' '
 	test_grep "space after tree-ish" err
 '
 
+#
+# Path parsing
+#
+# There are two sorts of ways a path can be parsed, depending on whether it is
+# the last field on the line. Additionally, ls without a <dataref> has a special
+# case. Test every occurrence of <path> in the grammar against every error case.
+# Paths for the root (empty strings) are tested elsewhere.
+#
+
+#
+# Valid paths at the end of a line: filemodify, filedelete, filecopy (dest),
+# filerename (dest), and ls.
+#
+# commit :301 from root -- modify hello.c (for setup)
+# commit :302 from :301 -- modify $path
+# commit :303 from :302 -- delete $path
+# commit :304 from :301 -- copy hello.c $path
+# commit :305 from :301 -- rename hello.c $path
+# ls :305 $path
+#
+test_path_eol_success () {
+	local test="$1" path="$2" unquoted_path="$3"
+	test_expect_success "S: paths at EOL with $test must work" '
+		test_when_finished "git branch -D S-path-eol" &&
+
+		git fast-import --export-marks=marks.out <<-EOF >out 2>err &&
+		blob
+		mark :401
+		data <<BLOB
+		hello world
+		BLOB
+
+		blob
+		mark :402
+		data <<BLOB
+		hallo welt
+		BLOB
+
+		commit refs/heads/S-path-eol
+		mark :301
+		committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+		data <<COMMIT
+		initial commit
+		COMMIT
+		M 100644 :401 hello.c
+
+		commit refs/heads/S-path-eol
+		mark :302
+		committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+		data <<COMMIT
+		commit filemodify
+		COMMIT
+		from :301
+		M 100644 :402 $path
+
+		commit refs/heads/S-path-eol
+		mark :303
+		committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+		data <<COMMIT
+		commit filedelete
+		COMMIT
+		from :302
+		D $path
+
+		commit refs/heads/S-path-eol
+		mark :304
+		committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+		data <<COMMIT
+		commit filecopy dest
+		COMMIT
+		from :301
+		C hello.c $path
+
+		commit refs/heads/S-path-eol
+		mark :305
+		committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+		data <<COMMIT
+		commit filerename dest
+		COMMIT
+		from :301
+		R hello.c $path
+
+		ls :305 $path
+		EOF
+
+		commit_m=$(grep :302 marks.out | cut -d\  -f2) &&
+		commit_d=$(grep :303 marks.out | cut -d\  -f2) &&
+		commit_c=$(grep :304 marks.out | cut -d\  -f2) &&
+		commit_r=$(grep :305 marks.out | cut -d\  -f2) &&
+		blob1=$(grep :401 marks.out | cut -d\  -f2) &&
+		blob2=$(grep :402 marks.out | cut -d\  -f2) &&
+
+		(
+			printf "100644 blob $blob2\t$unquoted_path\n" &&
+			printf "100644 blob $blob1\thello.c\n"
+		) | sort >tree_m.exp &&
+		git ls-tree $commit_m | sort >tree_m.out &&
+		test_cmp tree_m.exp tree_m.out &&
+
+		printf "100644 blob $blob1\thello.c\n" >tree_d.exp &&
+		git ls-tree $commit_d >tree_d.out &&
+		test_cmp tree_d.exp tree_d.out &&
+
+		(
+			printf "100644 blob $blob1\t$unquoted_path\n" &&
+			printf "100644 blob $blob1\thello.c\n"
+		) | sort >tree_c.exp &&
+		git ls-tree $commit_c | sort >tree_c.out &&
+		test_cmp tree_c.exp tree_c.out &&
+
+		printf "100644 blob $blob1\t$unquoted_path\n" >tree_r.exp &&
+		git ls-tree $commit_r >tree_r.out &&
+		test_cmp tree_r.exp tree_r.out &&
+
+		test_cmp out tree_r.exp
+	'
+}
+
+test_path_eol_success 'quoted spaces'   '" hello world.c "'  ' hello world.c '
+test_path_eol_success 'unquoted spaces' ' hello world.c '    ' hello world.c '
+test_path_eol_success 'octal escapes'   '"\150\151\056\143"' 'hi.c'
+
+#
+# Valid paths before a space: filecopy (source) and filerename (source).
+#
+# commit :301 from root -- modify $path (for setup)
+# commit :302 from :301 -- copy $path hello2.c
+# commit :303 from :301 -- rename $path hello2.c
+#
+test_path_space_success () {
+	local test="$1" path="$2" unquoted_path="$3"
+	test_expect_success "S: paths before space with $test must work" '
+		test_when_finished "git branch -D S-path-space" &&
+
+		git fast-import --export-marks=marks.out <<-EOF 2>err &&
+		blob
+		mark :401
+		data <<BLOB
+		hello world
+		BLOB
+
+		commit refs/heads/S-path-space
+		mark :301
+		committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+		data <<COMMIT
+		initial commit
+		COMMIT
+		M 100644 :401 $path
+
+		commit refs/heads/S-path-space
+		mark :302
+		committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+		data <<COMMIT
+		commit filecopy source
+		COMMIT
+		from :301
+		C $path hello2.c
+
+		commit refs/heads/S-path-space
+		mark :303
+		committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+		data <<COMMIT
+		commit filerename source
+		COMMIT
+		from :301
+		R $path hello2.c
+
+		EOF
+
+		commit_c=$(grep :302 marks.out | cut -d\  -f2) &&
+		commit_r=$(grep :303 marks.out | cut -d\  -f2) &&
+		blob=$(grep :401 marks.out | cut -d\  -f2) &&
+
+		(
+			printf "100644 blob $blob\t$unquoted_path\n" &&
+			printf "100644 blob $blob\thello2.c\n"
+		) | sort >tree_c.exp &&
+		git ls-tree $commit_c | sort >tree_c.out &&
+		test_cmp tree_c.exp tree_c.out &&
+
+		printf "100644 blob $blob\thello2.c\n" >tree_r.exp &&
+		git ls-tree $commit_r >tree_r.out &&
+		test_cmp tree_r.exp tree_r.out
+	'
+}
+
+test_path_space_success 'quoted spaces'      '" hello world.c "'  ' hello world.c '
+test_path_space_success 'no unquoted spaces' 'hello_world.c'      'hello_world.c'
+test_path_space_success 'octal escapes'      '"\150\151\056\143"' 'hi.c'
+
+#
+# Test a single commit change with an invalid path. Run it with all occurrences
+# of <path> in the grammar against all error kinds.
+#
+test_path_fail () {
+	local change="$1" what="$2" prefix="$3" path="$4" suffix="$5" err_grep="$6"
+	test_expect_success "S: $change with $what must fail" '
+		test_must_fail git fast-import <<-EOF 2>err &&
+		blob
+		mark :1
+		data <<BLOB
+		hello world
+		BLOB
+
+		commit refs/heads/S-path-fail
+		mark :2
+		committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+		data <<COMMIT
+		commit setup
+		COMMIT
+		M 100644 :1 hello.c
+
+		commit refs/heads/S-path-fail
+		committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+		data <<COMMIT
+		commit with bad path
+		COMMIT
+		from :2
+		$prefix$path$suffix
+		EOF
+
+		test_grep "$err_grep" err
+	'
+}
+
+test_path_base_fail () {
+	local change="$1" prefix="$2" field="$3" suffix="$4"
+	test_path_fail "$change" 'unclosed " in '"$field"          "$prefix" '"hello.c'    "$suffix" "Invalid $field"
+	test_path_fail "$change" "invalid escape in quoted $field" "$prefix" '"hello\xff"' "$suffix" "Invalid $field"
+	test_path_fail "$change" "escaped NUL in quoted $field"    "$prefix" '"hello\000"' "$suffix" "NUL in $field"
+}
+test_path_eol_quoted_fail () {
+	local change="$1" prefix="$2" field="$3"
+	test_path_base_fail "$change" "$prefix" "$field" ''
+	test_path_fail "$change" "garbage after quoted $field" "$prefix" '"hello.c"' 'x' "Garbage after $field"
+	test_path_fail "$change" "space after quoted $field"   "$prefix" '"hello.c"' ' ' "Garbage after $field"
+}
+test_path_eol_fail () {
+	local change="$1" prefix="$2" field="$3"
+	test_path_eol_quoted_fail "$change" "$prefix" "$field"
+}
+test_path_space_fail () {
+	local change="$1" prefix="$2" field="$3"
+	test_path_base_fail "$change" "$prefix" "$field" ' world.c'
+	test_path_fail "$change" "missing space after quoted $field"   "$prefix" '"hello.c"' 'x world.c' "Missing space after $field"
+	test_path_fail "$change" "missing space after unquoted $field" "$prefix" 'hello.c'   ''          "Missing space after $field"
+}
+
+test_path_eol_fail   filemodify       'M 100644 :1 ' path
+test_path_eol_fail   filedelete       'D '           path
+test_path_space_fail filecopy         'C '           source
+test_path_eol_fail   filecopy         'C hello.c '   dest
+test_path_space_fail filerename       'R '           source
+test_path_eol_fail   filerename       'R hello.c '   dest
+test_path_eol_fail   'ls (in commit)' 'ls :2 '       path
+
+# When 'ls' has no <dataref>, the <path> must be quoted.
+test_path_eol_quoted_fail 'ls (without dataref in commit)' 'ls ' path
+
 ###
 ### series T (ls)
 ###
 # Setup is carried over from series S.
 
-test_expect_success 'T: ls root tree' '
-	sed -e "s/Z\$//" >expect <<-EOF &&
-	040000 tree $(git rev-parse S^{tree})	Z
-	EOF
-	sha1=$(git rev-parse --verify S) &&
-	git fast-import --import-marks=marks <<-EOF >actual &&
-	ls $sha1 ""
-	EOF
-	test_cmp expect actual
-'
+for root in '""' ''
+do
+	test_expect_success "T: ls root ($root) tree" '
+		sed -e "s/Z\$//" >expect <<-EOF &&
+		040000 tree $(git rev-parse S^{tree})	Z
+		EOF
+		sha1=$(git rev-parse --verify S) &&
+		git fast-import --import-marks=marks <<-EOF >actual &&
+		ls $sha1 $root
+		EOF
+		test_cmp expect actual
+	'
+done
 
 test_expect_success 'T: delete branch' '
 	git branch to-delete &&
@@ -3176,30 +3449,33 @@ test_expect_success 'U: validate directory delete result' '
 	compare_diff_raw expect actual
 '
 
-test_expect_success 'U: filedelete root succeeds' '
-	cat >input <<-INPUT_END &&
-	commit refs/heads/U
-	committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
-	data <<COMMIT
-	must succeed
-	COMMIT
-	from refs/heads/U^0
-	D ""
+for root in '""' ''
+do
+	test_expect_success "U: filedelete root ($root) succeeds" '
+		cat >input <<-INPUT_END &&
+		commit refs/heads/U-delete-root
+		committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+		data <<COMMIT
+		must succeed
+		COMMIT
+		from refs/heads/U^0
+		D $root
 
-	INPUT_END
+		INPUT_END
 
-	git fast-import <input
-'
+		git fast-import <input
+	'
 
-test_expect_success 'U: validate root delete result' '
-	cat >expect <<-EOF &&
-	:100644 000000 $f7id $ZERO_OID D	hello.c
-	EOF
+	test_expect_success "U: validate root ($root) delete result" '
+		cat >expect <<-EOF &&
+		:100644 000000 $f7id $ZERO_OID D	hello.c
+		EOF
 
-	git diff-tree -M -r U^1 U >actual &&
+		git diff-tree -M -r U U-delete-root >actual &&
 
-	compare_diff_raw expect actual
-'
+		compare_diff_raw expect actual
+	'
+done
 
 ###
 ### series V (checkpoint)