Add cli-tests framework

Add a framework for writing automated tests of the fscrypt command-line
tool.  See cli-tests/README.md for details.

6 files changed, 550 insertions, 7 deletions

diff --git a/.gitignore b/.gitignore
index 666fe16..ca3e70e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,7 @@ bin/staticcheck
 bin/gocovmerge
 bin/misspell
 bin/config
+cli-tests/*.out.actual
 *coverage.out
 .vscode
 tags
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index ddd456c..5968d0e 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -59,7 +59,11 @@ On every pull request, [Travis CI](https://travis-ci.org/google/fscrypt) runs
 unit tests, integration tests, code formatters, and linters. To pass these
 checks you should make sure that in your submission:
 - `make` properly builds `fscrypt` and `pam_fscrypt.so`.
-- All tests, including [integration tests](#running-integration-tests), should pass.
+- All tests, including [integration tests](#running-integration-tests) and
+  [command-line interface (CLI)
+  tests](https://github.com/google/fscrypt/blob/master/cli-tests/README.md),
+  should pass. If the CLI tests fail due to an expected change in output, you
+  can use `make cli-test-update`.
 - `make format` has been run.
 - If you made any changes to files ending in `.proto`, the corresponding
   `.pb.go` files should be regenerated with `make gen`.
@@ -74,17 +78,26 @@ Essentially, if you run:
 make test-setup
 make all
 make test-teardown
+make cli-test
 go mod tidy
 ```
 and everything succeeds, and no files are changed, you're good to submit.
 
-The `Makefile` should automatically download and build whatever it needs.
-The only exceptions to this rule are:
+The `Makefile` will automatically download and build any needed Go dependencies.
+However, you'll also need to install some non-Go dependencies:
   - `make format` requires
     [`clang-format`](https://clang.llvm.org/docs/ClangFormat.html).
-  - `make test-setup` requires
-    [`e2fsprogs`](https://en.wikipedia.org/wiki/E2fsprogs) version 1.43
-    or later (or any patched version that supports `-O encrypt`).
+  - `make test-setup` and `make cli-test` require
+    [`e2fsprogs`](https://en.wikipedia.org/wiki/E2fsprogs) version 1.43 or
+    later.
+  - `make cli-test` requires [`expect`](https://en.wikipedia.org/wiki/Expect)
+    and
+    [`keyutils`](https://manpages.debian.org/testing/keyutils/keyctl.1.en.html).
+
+On Ubuntu, the following command installs the needed packages:
+```
+sudo apt-get install clang-format e2fsprogs expect keyutils
+```
 
 ### Running Integration Tests
 
diff --git a/Makefile b/Makefile
index d110dd8..6e2eda2 100644
--- a/Makefile
+++ b/Makefile
@@ -114,7 +114,7 @@ lint: $(BIN)/golint $(BIN)/staticcheck $(BIN)/misspell
 clean:
 	rm -f $(BIN)/$(NAME) $(PAM_MODULE) $(TOOLS) coverage.out $(COVERAGE_FILES) $(PAM_CONFIG)
 
-###### Testing Commands (setup/teardown require sudo) ######
+###### Go tests ######
 .PHONY: test test-setup test-teardown
 
 # If MOUNT exists signal that we should run integration tests.
@@ -139,6 +139,15 @@ test-teardown:
 	rmdir $(MOUNT)
 	rm -f $(IMAGE)
 
+###### Command-line interface tests ######
+.PHONY: cli-test cli-test-update
+
+cli-test: $(BIN)/$(NAME)
+	sudo cli-tests/run.sh
+
+cli-test-update: $(BIN)/$(NAME)
+	sudo cli-tests/run.sh --update-output
+
 # Runs tests and generates coverage
 COVERAGE_FILES := $(addsuffix coverage.out,$(GO_DIRS))
 coverage.out: $(BIN)/gocovmerge $(COVERAGE_FILES)
diff --git a/cli-tests/README.md b/cli-tests/README.md
new file mode 100644
index 0000000..dfcc1d0
--- /dev/null
+++ b/cli-tests/README.md
@@ -0,0 +1,67 @@
+# fscrypt command-line interface tests
+
+## Usage
+
+To run the command-line interface (CLI) tests for `fscrypt`, ensure
+that your kernel is v5.4 or later and has `CONFIG_FS_ENCRYPTION=y`.
+Also ensure that you have the following packages installed:
+
+* e2fsprogs
+* expect
+* keyutils
+
+Then, run:
+
+```shell
+make cli-test
+```
+
+You'll need to enter your `sudo` password, as the tests require root.
+
+If you only want to run specific tests, run a command like:
+
+```shell
+make && sudo cli-tests/run.sh t_encrypt t_unlock
+```
+
+## Updating the expected output
+
+When the output of `fscrypt` has intentionally changed, the test
+`.out` files need to be updated.  This can be done automatically by
+the following command, but be sure to review the changes:
+
+```shell
+make cli-test-update
+```
+
+## Writing CLI tests
+
+The fscrypt CLI tests are `bash` scripts named like `t_*.sh`.
+
+The test scripts must be executable and begin by sourcing `common.sh`.
+They all run in bash "extra-strict mode" (`-e -u -o pipefail`).  They
+run as root and have access to the following environment:
+
+* `$DEV`, `$DEV_ROOT`: ext4 filesystem images with encryption enabled
+
+* `$MNT`, `$MNT_ROOT`: the mountpoints of the above filesystems.
+  Initially all filesystems are mounted and are setup for fscrypt.
+  Login protectors will be stored on `$MNT_ROOT`.
+
+* `$TMPDIR`: a temporary directory that the test may use
+
+* `$FSCRYPT_CONF`: location of the fscrypt.conf file.  Initially this
+  file exists and specifies to use v2 policies with the default
+  settings, except password hashing is configured to be extra fast.
+
+* `$TEST_USER`: a non-root user that the test may use.  Their password
+  is `TEST_USER_PASS`.
+
+Any output (stdout and stderr) the test prints is compared to the
+corresponding `.out` file.  If a difference is detected then the test
+is considered to have failed.  The output is first sent through some
+standard filters; see `run.sh`.
+
+The test is also failed if it exits with nonzero status.
+
+See `common.sh` for utility functions the tests may use.
diff --git a/cli-tests/common.sh b/cli-tests/common.sh
new file mode 100644
index 0000000..fcebfd6
--- /dev/null
+++ b/cli-tests/common.sh
@@ -0,0 +1,154 @@
+#!/bin/bash
+#
+# common.sh - helper functions for fscrypt command-line interface tests
+#
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+#
+
+# Use extra-strict mode.
+set -e -u -o pipefail
+
+# Don't allow running the test scripts directly.  They need to be run via
+# run.sh, to set up everything correctly.
+if [ -z "${MNT:-}" ] || [ -z "${MNT_ROOT:-}" ]; then
+	echo 1>&2 "ERROR: This script can only be run via run.sh, not on its own."
+	exit 1
+fi
+
+# Prints an error message, then fails the test by exiting with failure status.
+_fail()
+{
+	[ $# -ne 1 ] && _fail "wrong argument count to ${FUNCNAME[0]}"
+	echo 1>&2 "ERROR: $1"
+	exit 1
+}
+
+# Runs a shell command and expects that it fails.
+_expect_failure()
+{
+	[ $# -ne 1 ] && _fail "wrong argument count to ${FUNCNAME[0]}"
+	if eval "$1"; then
+		_fail "command unexpectedly succeeded: \"$1\""
+	fi
+}
+
+# Prints a message to mark the beginning of the next part of the test.
+_print_header()
+{
+	[ $# -ne 1 ] && _fail "wrong argument count to ${FUNCNAME[0]}"
+	echo
+	echo "# $1"
+}
+
+# Deletes all files on the test filesystems, including all policies and
+# protectors.  Leaves the fscrypt metadata directories themselves.
+_reset_filesystems()
+{
+	local mnt
+
+	[ $# -ne 0 ] && _fail "wrong argument count to ${FUNCNAME[0]}"
+
+	for mnt in "$MNT" "$MNT_ROOT"; do
+		rm -rf "${mnt:?}"/* "${mnt:?}"/.fscrypt/{policies,protectors}/*
+	done
+}
+
+# Prints the number of filesystems that have encryption support enabled.
+_get_enabled_fs_count()
+{
+	local count
+
+	[ $# -ne 0 ] && _fail "wrong argument count to ${FUNCNAME[0]}"
+
+	count=$(fscrypt status | awk '/filesystems supporting encryption/ { print $4 }')
+	if [ -z "$count" ]; then
+		_fail "encryption support status line not found"
+	fi
+	echo "$count"
+}
+
+# Prints the number of filesystems that have fscrypt metadata.
+_get_setup_fs_count()
+{
+	local count
+
+	[ $# -ne 0 ] && _fail "wrong argument count to ${FUNCNAME[0]}"
+
+	count=$(fscrypt status | awk '/filesystems with fscrypt metadata/ { print $5 }')
+	if [ -z "$count" ]; then
+		_fail "fscrypt metadata status line not found"
+	fi
+	echo "$count"
+}
+
+# Removes all fscrypt metadata from the given filesystem.
+_rm_metadata()
+{
+	[ $# -ne 1 ] && _fail "wrong argument count to ${FUNCNAME[0]}"
+
+	rm -r "${1:?}/.fscrypt"
+}
+
+# Runs a shell command, ignoring its output (stdout and stderr) if it succeeds.
+# If the command fails, prints its output and fails the test.
+_run_noisy_command()
+{
+	[ $# -ne 1 ] && _fail "wrong argument count to ${FUNCNAME[0]}"
+
+	if ! eval "$1" &> "$TMPDIR/out"; then
+		_fail "Command failed: '$1'.  Output was: $(cat "$TMPDIR/out")"
+	fi
+}
+
+# Runs the given shell command as the test user.
+_user_do()
+{
+	[ $# -ne 1 ] && _fail "wrong argument count to ${FUNCNAME[0]}"
+
+	su "$TEST_USER" --command="$1"
+}
+
+# Runs the given shell command as the test user and expects it to fail.
+_user_do_and_expect_failure()
+{
+	[ $# -ne 1 ] && _fail "wrong argument count to ${FUNCNAME[0]}"
+
+	_expect_failure "_user_do '$1'"
+}
+
+# Gives the test a new session keyring which contains the test user's keyring
+# but not root's keyring.  Also clears the test user's keyring.  This must be
+# called at the beginning of the test script as it may re-execute the script.
+_setup_session_keyring()
+{
+	[ $# -ne 0 ] && _fail "wrong argument count to ${FUNCNAME[0]}"
+
+	# This *should* just use 'keyctl new_session', but that doesn't work if
+	# the session keyring is owned by a user other than root.  So instead we
+	# have to use 'keyctl session' and re-execute the script.
+	if [ -z "${FSCRYPT_SESSION_KEYRING_SET:-}" ]; then
+		export FSCRYPT_SESSION_KEYRING_SET=1
+		set +e
+		keyctl session - "$0" |& grep -v '^Joined session keyring'
+		exit "${PIPESTATUS[0]}"
+	fi
+
+	# Link the test user's keyring into the new session keyring.
+	keyctl setperm @s 0x3f000000 # all possessor permissions
+	_user_do "keyctl link @u @s"
+
+	# Clear the test user's keyring.
+	_user_do "keyctl clear @u"
+}
diff --git a/cli-tests/run.sh b/cli-tests/run.sh
new file mode 100755
index 0000000..909b645
--- /dev/null
+++ b/cli-tests/run.sh
@@ -0,0 +1,299 @@
+#!/bin/bash
+#
+# run.sh - run the fscrypt command-line interface tests
+#
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+#
+
+# Use extra-strict mode.
+set -e -u -o pipefail
+
+# Ensure we're in the cli-tests/ directory.
+cd "$(dirname "$0")"
+
+# Names of the test devices.
+# Variables with these names are exported to the tests.
+DEVICES=(DEV DEV_ROOT)
+
+# Names of the mountpoint of each test device.
+# Variables with these names are exported to the tests.
+MOUNTS=(MNT MNT_ROOT)
+
+# Name of the test user.  This user will be created and deleted by this script.
+# This variable is exported to the tests.
+TEST_USER=fscrypt-test-user
+
+# The temporary directory to use.
+# This variable is exported to the tests.
+TMPDIR=$(mktemp -d /tmp/fscrypt.XXXXXX)
+
+# The loopback devices that correspond to each test device.
+LOOPS=()
+
+# Update the expected output files to match the actual output?
+UPDATE_OUTPUT=false
+
+LONGOPTS_ARRAY=(
+'help'
+'update-output'
+)
+LONGOPTS=$(echo "${LONGOPTS_ARRAY[*]}" | tr ' ' ,)
+
+cleanup()
+{
+	local mnt loop
+
+	# Unmount all the test filesystems.
+	for mnt in "${MOUNTS[@]}"; do
+		mnt="$TMPDIR/$mnt"
+		if mountpoint "$mnt" &> /dev/null; then
+			umount "$mnt"
+		fi
+	done
+
+	# Delete the loopback device of each test device.
+	for loop in "${LOOPS[@]}"; do
+		losetup -d "$loop"
+	done
+
+	# Delete all temporary files.
+	rm -rf "${TMPDIR:?}"/*
+}
+
+cleanup_full()
+{
+	cleanup
+	rm -rf "$TMPDIR"
+	userdel "$TEST_USER" || true
+}
+
+# Filters the output of the test script to make the output consistent on every
+# run of the test.  For example, references to the mountpoint like
+# /tmp/fscrypt.4OTb6y/MNT will be replaced with simply MNT, since the name of
+# the temporary directory is different every time.
+filter_test_output()
+{
+	local sedscript=""
+	local raw_output=$TMPDIR/raw-test-output
+	local i
+
+	cat > "$raw_output"
+
+	# Filter mountpoint and device names.
+	for i in "${!DEVICES[@]}"; do
+		sedscript+="s@$TMPDIR/${MOUNTS[$i]}@${MOUNTS[$i]}@g;"
+		sedscript+="s@${LOOPS[$i]}@${DEVICES[$i]}@g;"
+	done
+
+	# Filter the path to fscrypt.conf.
+	sedscript+="s@$FSCRYPT_CONF@FSCRYPT_CONF@g;"
+
+	# Filter policy and protector descriptors.
+	sedscript+=$(grep -E -o '\<([a-f0-9]{16})|([a-f0-9]{32})\>' \
+		     "$raw_output" \
+		     | awk '{ printf "s@\\<" $1 "\\>@desc" NR "@g;" }')
+
+	# Filter any other paths in TMPDIR.
+	sedscript+="s@$TMPDIR@TMPDIR@g;"
+
+	sed -e "$sedscript" "$raw_output"
+}
+
+# Prepares to run a test script.
+setup_for_test()
+{
+	local i dev_var mnt_var img mnt loop
+
+	# Start with a clean state.
+	cleanup
+
+	# ../bin/fscrypt might not be accessible to $TEST_USER.  Copy it into
+	# $TMPDIR so that $TEST_USER is guaranteed to have access to it.
+	mkdir "$TMPDIR/bin"
+	cp ../bin/fscrypt "$TMPDIR/bin/"
+	chmod 755 "$TMPDIR" "$TMPDIR/bin" "$TMPDIR/bin/fscrypt"
+
+	# Create the test filesystems and mountpoints.
+	LOOPS=()
+	for i in "${!DEVICES[@]}"; do
+		dev_var=${DEVICES[$i]}
+		mnt_var=${MOUNTS[$i]}
+		img="$TMPDIR/$dev_var"
+		if ! mkfs.ext4 -O encrypt -F -b 4096 -I 256 "$img" $((1<<15)) \
+			&> "$TMPDIR/mkfs.out"
+		then
+			cat 1>&2 "$TMPDIR/mkfs.out"
+			exit 1
+		fi
+		loop=$(losetup --find --show "$img")
+		LOOPS+=("$loop")
+		export "$dev_var=$loop"
+		mnt="$TMPDIR/$mnt_var"
+		export "$mnt_var=$mnt"
+		mkdir -p "$mnt"
+		mount "$loop" "$mnt"
+	done
+
+	# Give the tests their own "root" mount for storing login protectors, so
+	# they don't use the real "/".
+	export FSCRYPT_ROOT_MNT="$MNT_ROOT"
+
+	# Enable consistent output mode.
+	export FSCRYPT_CONSISTENT_OUTPUT="1"
+
+	# Give the tests their own fscrypt.conf.
+	export FSCRYPT_CONF="$TMPDIR/fscrypt.conf"
+	fscrypt setup --time=1ms > /dev/null
+
+	# The tests assume kernel support for v2 policies.
+	if ! grep -q '"policy_version": "2"' "$FSCRYPT_CONF"; then
+		cat 1>&2 << EOF
+ERROR: Can't run these tests because your kernel doesn't support v2 policies.
+You need kernel v5.4 or later.
+EOF
+		exit 1
+	fi
+
+	# Set up the test filesystems that aren't already set up.
+	fscrypt setup "$MNT" > /dev/null
+}
+
+run_test()
+{
+	local t=$1
+
+	# Run the test script.
+	set +e
+	"./$1.sh" |& filter_test_output > "$t.out.actual"
+	status=${PIPESTATUS[0]}
+	set -e
+
+	# Check for failure status.
+	if [ "$status" != 0 ]; then
+		echo 1>&2 "FAILED: $t [exited with failure status $status]"
+		if [ -s "$t.out.actual" ]; then
+			if (( $(wc -l "$t.out.actual" | cut -f1 -d' ') > 10 )); then
+				echo 1>&2 "Last 10 lines of test output:"
+				tail -n10 "$t.out.actual" | sed 1>&2 's/^/    /'
+				echo 1>&2
+				echo 1>&2 "See $t.out.actual for the full output."
+			else
+				echo 1>&2 "Test output:"
+				sed 1>&2 's/^/    /' < "$t.out.actual"
+			fi
+		fi
+		exit 1
+	fi
+
+	# Check for output mismatch.
+	if ! cmp "$t.out" "$t.out.actual" &> /dev/null; then
+		if $UPDATE_OUTPUT; then
+			cp "$t.out.actual" "$t.out"
+			echo "Updated $t.out"
+		else
+			echo 1>&2 "FAILED: $t [output mismatch]"
+			echo 1>&2 "Differences between $t.out and $t.out.actual:"
+			echo 1>&2
+			diff 1>&2 "$t.out" "$t.out.actual"
+			exit 1
+		fi
+	fi
+	rm -f "$t.out.actual"
+}
+
+usage()
+{
+	cat << EOF
+Usage: run.sh [--update-output] [TEST_SCRIPT_NAME]..."
+EOF
+	exit 1
+}
+
+if ! options=$(getopt -o "" -l "$LONGOPTS" -- "$@"); then
+	usage
+fi
+eval set -- "$options"
+while (( $# >= 1 )); do
+	case "$1" in
+	--update-output)
+		UPDATE_OUTPUT=true
+		;;
+	--)
+		shift
+		break
+		;;
+	--help|*)
+		usage
+		;;
+	esac
+	shift
+done
+
+if [ "$(id -u)" != 0 ]; then
+	echo 1>&2 "ERROR: You must be root to run these tests."
+	exit 1
+fi
+
+# Check for prerequisites.
+PREREQ_CMDS=(mkfs.ext4 expect keyctl)
+PREREQ_PKGS=(e2fsprogs expect keyutils)
+for i in ${!PREREQ_CMDS[*]}; do
+	if ! type -P "${PREREQ_CMDS[$i]}" > /dev/null; then
+		cat 1>&2 << EOF
+ERROR: You must install the '${PREREQ_PKGS[$i]}' package to run these tests.
+       Try a command like 'sudo apt-get install ${PREREQ_PKGS[$i]}'.
+EOF
+		exit 1
+	fi
+done
+
+# Use a consistent umask.
+umask 022
+
+# Use a consistent locale, to prevent output mismatches.
+export LANG=C
+export LC_ALL=C
+
+# Always cleanup fully on exit.
+trap cleanup_full EXIT
+
+# Create a test user, so that we can test non-root use of fscrypt.  Give them a
+# password, so that we can test creating login passphrase protected directories.
+userdel "$TEST_USER" &> /dev/null || true
+useradd "$TEST_USER"
+echo "$TEST_USER:TEST_USER_PASS" | chpasswd
+export TEST_USER
+
+# Let the tests use $TMPDIR if they need it.
+export TMPDIR
+
+# Make it so that running 'fscrypt' in the tests runs the correct binary.
+export PATH="$TMPDIR/bin:$PATH"
+
+if (( $# >= 1 )); then
+	# Tests specified on command line.
+	tests=("$@")
+else
+	# No tests specified on command line.  Just run everything.
+	tests=(t_*.sh)
+fi
+for t in "${tests[@]}"; do
+	t=${t%.sh}
+	echo "Running $t"
+	setup_for_test
+	run_test "$t"
+done
+
+echo "All tests passed!"