keyring: support filesystem keyring with v1 encryption policies

Linux v5.4 and later allows fscrypt keys to be added/removed directly
to/from the filesystem via the new ioctls FS_IOC_ADD_ENCRYPTION_KEY and
FS_IOC_REMOVE_ENCRYPTION_KEY.  Among other benefits, these fix the key
visibility problems that many users have been running into, where system
services and containers can't access encrypted files.

Allow the user to opt-in to using these new ioctls for their existing
encrypted directories by setting in their /etc/fscrypt.conf:

	"use_fs_keyring_for_v1_policies": true

Note that it can't really be on by default, since for v1 policies the
ioctls require root, whereas user keyrings don't.  I.e., setting this to
true means that users will need to use 'sudo fscrypt unlock', not
'fscrypt unlock'.  v2 policies won't have this restriction.

4 files changed, 358 insertions, 34 deletions

diff --git a/keyring/fs_keyring.go b/keyring/fs_keyring.go
new file mode 100644
index 0000000..bed23af
--- /dev/null
+++ b/keyring/fs_keyring.go
@@ -0,0 +1,211 @@
+/*
+ * fs_keyring.go - Add/remove encryption policy keys to/from filesystem
+ *
+ * Copyright 2019 Google LLC
+ * Author: Eric Biggers (ebiggers@google.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package keyring
+
+/*
+#include <string.h>
+*/
+import "C"
+
+import (
+	"encoding/hex"
+	"log"
+	"os"
+	"os/user"
+	"sync"
+	"unsafe"
+
+	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
+
+	"github.com/google/fscrypt/crypto"
+	"github.com/google/fscrypt/filesystem"
+)
+
+var (
+	fsKeyringSupported      bool
+	fsKeyringSupportedKnown bool
+	fsKeyringSupportedLock  sync.Mutex
+)
+
+func checkForFsKeyringSupport(mount *filesystem.Mount) bool {
+	dir, err := os.Open(mount.Path)
+	if err != nil {
+		log.Printf("Unexpected error opening %q. Assuming filesystem keyring is unsupported.",
+			mount.Path)
+		return false
+	}
+	defer dir.Close()
+
+	// FS_IOC_ADD_ENCRYPTION_KEY with a NULL argument will fail with ENOTTY
+	// if the ioctl isn't supported.  Otherwise it should fail with EFAULT.
+	//
+	// Note that there's no need to check for FS_IOC_REMOVE_ENCRYPTION_KEY
+	// support separately, since it's guaranteed to be available if
+	// FS_IOC_ADD_ENCRYPTION_KEY is.  There's also no need to check for
+	// support on every filesystem separately, since either the kernel
+	// supports the ioctls on all fscrypt-capable filesystems or it doesn't.
+	_, _, errno := unix.Syscall(unix.SYS_IOCTL, dir.Fd(), unix.FS_IOC_ADD_ENCRYPTION_KEY, 0)
+	if errno == unix.ENOTTY {
+		log.Printf("Kernel doesn't support filesystem keyring. Falling back to user keyring.")
+		return false
+	}
+	if errno == unix.EFAULT {
+		log.Printf("Detected support for filesystem keyring")
+	} else {
+		// EFAULT is expected, but as long as we didn't get ENOTTY the
+		// ioctl should be available.
+		log.Printf("Unexpected error from FS_IOC_ADD_ENCRYPTION_KEY(%q, NULL): %v", mount.Path, errno)
+	}
+	return true
+}
+
+// isFsKeyringSupported returns true if the kernel supports the ioctls to
+// add/remove fscrypt keys directly to/from the filesystem.  For support to be
+// detected, the given Mount must be for a filesystem that supports fscrypt.
+func isFsKeyringSupported(mount *filesystem.Mount) bool {
+	fsKeyringSupportedLock.Lock()
+	defer fsKeyringSupportedLock.Unlock()
+	if !fsKeyringSupportedKnown {
+		fsKeyringSupported = checkForFsKeyringSupport(mount)
+		fsKeyringSupportedKnown = true
+	}
+	return fsKeyringSupported
+}
+
+// buildKeySpecifier converts the key descriptor string to an FscryptKeySpecifier.
+func buildKeySpecifier(spec *unix.FscryptKeySpecifier, descriptor string) error {
+	descriptorBytes, err := hex.DecodeString(descriptor)
+	if err != nil {
+		return errors.Errorf("key descriptor %q is invalid", descriptor)
+	}
+	switch len(descriptorBytes) {
+	case unix.FSCRYPT_KEY_DESCRIPTOR_SIZE:
+		spec.Type = unix.FSCRYPT_KEY_SPEC_TYPE_DESCRIPTOR
+	default:
+		return errors.Errorf("key descriptor %q has unknown length", descriptor)
+	}
+	copy(spec.U[:], descriptorBytes)
+	return nil
+}
+
+// fsAddEncryptionKey adds the specified encryption key to the specified filesystem.
+func fsAddEncryptionKey(key *crypto.Key, descriptor string,
+	mount *filesystem.Mount, user *user.User) error {
+
+	dir, err := os.Open(mount.Path)
+	if err != nil {
+		return err
+	}
+	defer dir.Close()
+
+	argKey, err := crypto.NewBlankKey(int(unsafe.Sizeof(unix.FscryptAddKeyArg{})) + key.Len())
+	if err != nil {
+		return err
+	}
+	defer argKey.Wipe()
+	arg := (*unix.FscryptAddKeyArg)(argKey.UnsafePtr())
+
+	if err = buildKeySpecifier(&arg.Key_spec, descriptor); err != nil {
+		return err
+	}
+
+	raw := unsafe.Pointer(uintptr(argKey.UnsafePtr()) + unsafe.Sizeof(*arg))
+	arg.Raw_size = uint32(key.Len())
+	C.memcpy(raw, key.UnsafePtr(), C.size_t(key.Len()))
+
+	_, _, errno := unix.Syscall(unix.SYS_IOCTL, dir.Fd(),
+		unix.FS_IOC_ADD_ENCRYPTION_KEY, uintptr(argKey.UnsafePtr()))
+	log.Printf("FS_IOC_ADD_ENCRYPTION_KEY(%q, %s, <raw>) = %v", mount.Path, descriptor, errno)
+	if errno != 0 {
+		return errors.Wrap(ErrKeyAdd, errno.Error())
+	}
+	return nil
+}
+
+// fsRemoveEncryptionKey removes the specified encryption key from the specified
+// filesystem.
+func fsRemoveEncryptionKey(descriptor string, mount *filesystem.Mount,
+	user *user.User) error {
+
+	dir, err := os.Open(mount.Path)
+	if err != nil {
+		return err
+	}
+	defer dir.Close()
+
+	var arg unix.FscryptRemoveKeyArg
+	if err = buildKeySpecifier(&arg.Key_spec, descriptor); err != nil {
+		return err
+	}
+
+	_, _, errno := unix.Syscall(unix.SYS_IOCTL, dir.Fd(),
+		unix.FS_IOC_REMOVE_ENCRYPTION_KEY, uintptr(unsafe.Pointer(&arg)))
+	log.Printf("FS_IOC_REMOVE_ENCRYPTION_KEY(%q, %s) = %v, removal_status_flags=0x%x",
+		mount.Path, descriptor, errno, arg.Removal_status_flags)
+	switch errno {
+	case 0:
+		if arg.Removal_status_flags&unix.FSCRYPT_KEY_REMOVAL_STATUS_FLAG_FILES_BUSY != 0 {
+			return ErrKeyFilesOpen
+		}
+		return nil
+	case unix.ENOKEY:
+		return ErrKeyNotPresent
+	default:
+		return errors.Wrap(ErrKeyRemove, errno.Error())
+	}
+}
+
+// fsGetEncryptionKeyStatus gets the status of the specified encryption key on
+// the specified filesystem.
+func fsGetEncryptionKeyStatus(descriptor string, mount *filesystem.Mount,
+	user *user.User) (KeyStatus, error) {
+
+	dir, err := os.Open(mount.Path)
+	if err != nil {
+		return KeyStatusUnknown, err
+	}
+	defer dir.Close()
+
+	var arg unix.FscryptGetKeyStatusArg
+	err = buildKeySpecifier(&arg.Key_spec, descriptor)
+	if err != nil {
+		return KeyStatusUnknown, err
+	}
+
+	_, _, errno := unix.Syscall(unix.SYS_IOCTL, dir.Fd(),
+		unix.FS_IOC_GET_ENCRYPTION_KEY_STATUS, uintptr(unsafe.Pointer(&arg)))
+	log.Printf("FS_IOC_GET_ENCRYPTION_KEY_STATUS(%q, %s) = %v, status=%d, status_flags=0x%x",
+		mount.Path, descriptor, errno, arg.Status, arg.Status_flags)
+	if errno != 0 {
+		return KeyStatusUnknown, errors.Wrap(ErrKeySearch, errno.Error())
+	}
+	switch arg.Status {
+	case unix.FSCRYPT_KEY_STATUS_ABSENT:
+		return KeyAbsent, nil
+	case unix.FSCRYPT_KEY_STATUS_PRESENT:
+		return KeyPresent, nil
+	case unix.FSCRYPT_KEY_STATUS_INCOMPLETELY_REMOVED:
+		return KeyAbsentButFilesBusy, nil
+	default:
+		return KeyStatusUnknown,
+			errors.Wrapf(ErrKeySearch, "unknown key status (%d)", arg.Status)
+	}
+}
diff --git a/keyring/keyring.go b/keyring/keyring.go
index 7a5fd64..f0bd1b7 100644
--- a/keyring/keyring.go
+++ b/keyring/keyring.go
@@ -19,7 +19,11 @@
 
 // Package keyring manages adding, removing, and getting the status of
 // encryption policy keys to/from the kernel.  Most public functions are in
-// keyring.go, and they delegate to user_keyring.go.
+// keyring.go, and they delegate to either user_keyring.go or fs_keyring.go,
+// depending on whether a user keyring or a filesystem keyring is being used.
+//
+// v1 policies use the user keyring by default, but can be configured to use the
+// filesystem keyring instead (requires root and kernel v5.4+).
 package keyring
 
 import (
@@ -29,6 +33,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/google/fscrypt/crypto"
+	"github.com/google/fscrypt/filesystem"
 	"github.com/google/fscrypt/metadata"
 	"github.com/google/fscrypt/util"
 )
@@ -38,6 +43,7 @@ var (
 	ErrKeyAdd            = util.SystemError("could not add key to the keyring")
 	ErrKeyRemove         = util.SystemError("could not remove key from the keyring")
 	ErrKeyNotPresent     = errors.New("key not present or already removed")
+	ErrKeyFilesOpen      = errors.New("some files using the key are still open")
 	ErrKeySearch         = errors.New("could not find key with descriptor")
 	ErrSessionUserKeying = errors.New("user keyring not linked into session keyring")
 	ErrAccessUserKeyring = errors.New("could not access user keyring")
@@ -47,22 +53,47 @@ var (
 // Options are the options which specify *which* keyring the key should be
 // added/removed/gotten to, and how.
 type Options struct {
+	// Mount is the filesystem to which the key should be
+	// added/removed/gotten.
+	Mount *filesystem.Mount
 	// User is the user for whom the key should be added/removed/gotten.
 	User *user.User
-	// Service is the prefix to prepend to the description of the keys.
+	// Service is the prefix to prepend to the description of the keys in
+	// user keyrings.  Not relevant for filesystem keyrings.
 	Service string
+	// UseFsKeyringForV1Policies is true if keys for v1 encryption policies
+	// should be put in the filesystem's keyring (if supported) rather than
+	// in the user's keyring.  Note that this makes AddEncryptionKey and
+	// RemoveEncryptionKey require root privileges.
+	UseFsKeyringForV1Policies bool
 }
 
-// AddEncryptionKey adds an encryption policy key to a kernel keyring.
+func shouldUseFsKeyring(descriptor string, options *Options) bool {
+	// Use the filesystem keyring if use_fs_keyring_for_v1_policies is set
+	// in /etc/fscrypt.conf and the kernel supports it.
+	return options.UseFsKeyringForV1Policies && isFsKeyringSupported(options.Mount)
+}
+
+// AddEncryptionKey adds an encryption policy key to a kernel keyring.  It uses
+// either the filesystem keyring for the target Mount or the user keyring for
+// the target User.
 func AddEncryptionKey(key *crypto.Key, descriptor string, options *Options) error {
 	if err := util.CheckValidLength(metadata.PolicyKeyLen, key.Len()); err != nil {
 		return errors.Wrap(err, "policy key")
 	}
+	if shouldUseFsKeyring(descriptor, options) {
+		return fsAddEncryptionKey(key, descriptor, options.Mount, options.User)
+	}
 	return userAddKey(key, options.Service+descriptor, options.User)
 }
 
 // RemoveEncryptionKey removes an encryption policy key from a kernel keyring.
+// It uses either the filesystem keyring for the target Mount or the user
+// keyring for the target User.
 func RemoveEncryptionKey(descriptor string, options *Options) error {
+	if shouldUseFsKeyring(descriptor, options) {
+		return fsRemoveEncryptionKey(descriptor, options.Mount, options.User)
+	}
 	return userRemoveKey(options.Service+descriptor, options.User)
 }
 
@@ -73,6 +104,7 @@ type KeyStatus int
 const (
 	KeyStatusUnknown = 0 + iota
 	KeyAbsent
+	KeyAbsentButFilesBusy
 	KeyPresent
 )
 
@@ -82,6 +114,8 @@ func (status KeyStatus) String() string {
 		return "Unknown"
 	case KeyAbsent:
 		return "Absent"
+	case KeyAbsentButFilesBusy:
+		return "AbsentButFilesBusy"
 	case KeyPresent:
 		return "Present"
 	default:
@@ -90,8 +124,12 @@ func (status KeyStatus) String() string {
 }
 
 // GetEncryptionKeyStatus gets the status of an encryption policy key in a
-// kernel keyring.
+// kernel keyring.  It uses either the filesystem keyring for the target Mount
+// or the user keyring for the target User.
 func GetEncryptionKeyStatus(descriptor string, options *Options) (KeyStatus, error) {
+	if shouldUseFsKeyring(descriptor, options) {
+		return fsGetEncryptionKeyStatus(descriptor, options.Mount, options.User)
+	}
 	_, err := userFindKey(options.Service+descriptor, options.User)
 	if err != nil {
 		return KeyAbsent, nil
diff --git a/keyring/keyring_test.go b/keyring/keyring_test.go
index 10ff874..9a4570b 100644
--- a/keyring/keyring_test.go
+++ b/keyring/keyring_test.go
@@ -24,6 +24,7 @@ import (
 	"golang.org/x/sys/unix"
 
 	"github.com/google/fscrypt/crypto"
+	"github.com/google/fscrypt/filesystem"
 	"github.com/google/fscrypt/metadata"
 	"github.com/google/fscrypt/util"
 )
@@ -44,52 +45,125 @@ func makeKey(b byte, n int) (*crypto.Key, error) {
 }
 
 var (
-	fakeValidDescriptor     = "0123456789abcdef"
 	defaultService          = unix.FSCRYPT_KEY_DESC_PREFIX
 	testUser, _             = util.EffectiveUser()
 	fakeValidPolicyKey, _   = makeKey(42, metadata.PolicyKeyLen)
 	fakeInvalidPolicyKey, _ = makeKey(42, metadata.PolicyKeyLen-1)
+	fakeV1Descriptor        = "0123456789abcdef"
 )
 
-// Adds and removes a key with various services.
-func TestAddRemoveKeys(t *testing.T) {
-	for _, service := range []string{defaultService, "ext4:", "f2fs:"} {
-		options := &Options{
-			User:    testUser,
-			Service: service,
-		}
-		if err := AddEncryptionKey(fakeValidPolicyKey, fakeValidDescriptor, options); err != nil {
-			t.Error(err)
-		}
-		if err := RemoveEncryptionKey(fakeValidDescriptor, options); err != nil {
-			t.Error(err)
-		}
+func assertKeyStatus(t *testing.T, descriptor string, options *Options,
+	expectedStatus KeyStatus) {
+	status, err := GetEncryptionKeyStatus(descriptor, options)
+	if err != nil {
+		t.Error(err)
+	}
+	if status != expectedStatus {
+		t.Errorf("Expected key status %v but got key status %v", expectedStatus, status)
 	}
 }
 
-// Adds a key twice (both should succeed)
-func TestAddTwice(t *testing.T) {
-	options := &Options{
-		User:    testUser,
-		Service: defaultService,
+// getTestMount retrieves the Mount for a test filesystem, or skips the test if
+// no test filesystem is available.
+func getTestMount(t *testing.T) *filesystem.Mount {
+	root, err := util.TestRoot()
+	if err != nil {
+		t.Skip(err)
+	}
+	mount, err := filesystem.GetMount(root)
+	if err != nil {
+		t.Skip(err)
+	}
+	return mount
+}
+
+// getTestMountV2 is like getTestMount, but it also checks that the filesystem
+// keyring is supported.
+func getTestMountV2(t *testing.T) *filesystem.Mount {
+	mount := getTestMount(t)
+	if !isFsKeyringSupported(mount) {
+		t.Skip("No support for fs keyring, skipping test.")
+	}
+	return mount
+}
+
+func requireRoot(t *testing.T) {
+	if !util.IsUserRoot() {
+		t.Skip("Not root, skipping test.")
 	}
-	if err := AddEncryptionKey(fakeValidPolicyKey, fakeValidDescriptor, options); err != nil {
+}
+
+// testAddAndRemoveKey does the common tests for adding+removing keys that are
+// run in multiple configurations (v1 policies with user keyring and v1 policies
+// with fs keyring).
+func testAddAndRemoveKey(t *testing.T, descriptor string, options *Options) {
+
+	// Basic add, get status, and remove
+	if err := AddEncryptionKey(fakeValidPolicyKey, descriptor, options); err != nil {
 		t.Error(err)
 	}
-	if err := AddEncryptionKey(fakeValidPolicyKey, fakeValidDescriptor, options); err != nil {
+	assertKeyStatus(t, descriptor, options, KeyPresent)
+	if err := RemoveEncryptionKey(descriptor, options); err != nil {
+		t.Error(err)
+	}
+	assertKeyStatus(t, descriptor, options, KeyAbsent)
+	err := RemoveEncryptionKey(descriptor, options)
+	if err != ErrKeyNotPresent {
+		t.Error(err)
+	}
+
+	// Adding a key twice should succeed
+	if err := AddEncryptionKey(fakeValidPolicyKey, descriptor, options); err != nil {
+		t.Error(err)
+	}
+	if err := AddEncryptionKey(fakeValidPolicyKey, descriptor, options); err != nil {
 		t.Error("AddEncryptionKey should not fail if key already exists")
 	}
-	RemoveEncryptionKey(fakeValidDescriptor, options)
+	RemoveEncryptionKey(descriptor, options)
+	assertKeyStatus(t, descriptor, options, KeyAbsent)
+
+	// Adding a key with wrong length should fail
+	if err := AddEncryptionKey(fakeInvalidPolicyKey, descriptor, options); err == nil {
+		RemoveEncryptionKey(descriptor, options)
+		t.Error("AddEncryptionKey should fail with wrong-length key")
+	}
+	assertKeyStatus(t, descriptor, options, KeyAbsent)
 }
 
-// Makes sure trying to add a key of the wrong length fails
-func TestAddWrongLengthKey(t *testing.T) {
+func TestUserKeyringDefaultService(t *testing.T) {
 	options := &Options{
-		User:    testUser,
-		Service: defaultService,
+		User:                      testUser,
+		Service:                   defaultService,
+		UseFsKeyringForV1Policies: false,
 	}
-	if err := AddEncryptionKey(fakeInvalidPolicyKey, fakeValidDescriptor, options); err == nil {
-		RemoveEncryptionKey(fakeValidDescriptor, options)
-		t.Error("AddEncryptionKey should fail with wrong-length key")
+	testAddAndRemoveKey(t, fakeV1Descriptor, options)
+}
+
+func TestUserKeyringExt4Service(t *testing.T) {
+	options := &Options{
+		User:                      testUser,
+		Service:                   "ext4:",
+		UseFsKeyringForV1Policies: false,
+	}
+	testAddAndRemoveKey(t, fakeV1Descriptor, options)
+}
+
+func TestUserKeyringF2fsService(t *testing.T) {
+	options := &Options{
+		User:                      testUser,
+		Service:                   "f2fs:",
+		UseFsKeyringForV1Policies: false,
+	}
+	testAddAndRemoveKey(t, fakeV1Descriptor, options)
+}
+
+func TestFsKeyringV1PolicyKey(t *testing.T) {
+	requireRoot(t)
+	mount := getTestMountV2(t)
+	options := &Options{
+		Mount:                     mount,
+		User:                      testUser,
+		UseFsKeyringForV1Policies: true,
 	}
+	testAddAndRemoveKey(t, fakeV1Descriptor, options)
 }
diff --git a/keyring/user_keyring.go b/keyring/user_keyring.go
index adac71a..71f519d 100644
--- a/keyring/user_keyring.go
+++ b/keyring/user_keyring.go
@@ -1,5 +1,6 @@
 /*
- * user_keyring.go - Add/remove encryption policy keys to/from user keyrings
+ * user_keyring.go - Add/remove encryption policy keys to/from user keyrings.
+ * This is the deprecated mechanism; see fs_keyring.go for the new mechanism.
  *
  * Copyright 2017 Google Inc.
  * Author: Joe Richey (joerichey@google.com)