diff options
Diffstat (limited to 'fs/ceph/io.c')
| -rw-r--r-- | fs/ceph/io.c | 100 |
1 files changed, 75 insertions, 25 deletions
diff --git a/fs/ceph/io.c b/fs/ceph/io.c index c456509b31c3..2d10f49c93a9 100644 --- a/fs/ceph/io.c +++ b/fs/ceph/io.c @@ -21,14 +21,23 @@ /* Call with exclusively locked inode->i_rwsem */ static void ceph_block_o_direct(struct ceph_inode_info *ci, struct inode *inode) { + bool is_odirect; + lockdep_assert_held_write(&inode->i_rwsem); - if (READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT) { - spin_lock(&ci->i_ceph_lock); - ci->i_ceph_flags &= ~CEPH_I_ODIRECT; - spin_unlock(&ci->i_ceph_lock); - inode_dio_wait(inode); + spin_lock(&ci->i_ceph_lock); + /* ensure that bit state is consistent */ + smp_mb__before_atomic(); + is_odirect = READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT; + if (is_odirect) { + clear_bit(CEPH_I_ODIRECT_BIT, &ci->i_ceph_flags); + /* ensure modified bit is visible */ + smp_mb__after_atomic(); } + spin_unlock(&ci->i_ceph_lock); + + if (is_odirect) + inode_dio_wait(inode); } /** @@ -47,20 +56,35 @@ static void ceph_block_o_direct(struct ceph_inode_info *ci, struct inode *inode) * Note that buffered writes and truncates both take a write lock on * inode->i_rwsem, meaning that those are serialised w.r.t. the reads. */ -void -ceph_start_io_read(struct inode *inode) +int ceph_start_io_read(struct inode *inode) { struct ceph_inode_info *ci = ceph_inode(inode); + bool is_odirect; + int err; /* Be an optimist! */ - down_read(&inode->i_rwsem); - if (!(READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT)) - return; + err = down_read_killable(&inode->i_rwsem); + if (err) + return err; + + spin_lock(&ci->i_ceph_lock); + /* ensure that bit state is consistent */ + smp_mb__before_atomic(); + is_odirect = READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT; + spin_unlock(&ci->i_ceph_lock); + if (!is_odirect) + return 0; up_read(&inode->i_rwsem); + /* Slow path.... */ - down_write(&inode->i_rwsem); + err = down_write_killable(&inode->i_rwsem); + if (err) + return err; + ceph_block_o_direct(ci, inode); downgrade_write(&inode->i_rwsem); + + return 0; } /** @@ -83,11 +107,12 @@ ceph_end_io_read(struct inode *inode) * Declare that a buffered write operation is about to start, and ensure * that we block all direct I/O. */ -void -ceph_start_io_write(struct inode *inode) +int ceph_start_io_write(struct inode *inode) { - down_write(&inode->i_rwsem); - ceph_block_o_direct(ceph_inode(inode), inode); + int err = down_write_killable(&inode->i_rwsem); + if (!err) + ceph_block_o_direct(ceph_inode(inode), inode); + return err; } /** @@ -106,12 +131,22 @@ ceph_end_io_write(struct inode *inode) /* Call with exclusively locked inode->i_rwsem */ static void ceph_block_buffered(struct ceph_inode_info *ci, struct inode *inode) { + bool is_odirect; + lockdep_assert_held_write(&inode->i_rwsem); - if (!(READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT)) { - spin_lock(&ci->i_ceph_lock); - ci->i_ceph_flags |= CEPH_I_ODIRECT; - spin_unlock(&ci->i_ceph_lock); + spin_lock(&ci->i_ceph_lock); + /* ensure that bit state is consistent */ + smp_mb__before_atomic(); + is_odirect = READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT; + if (!is_odirect) { + set_bit(CEPH_I_ODIRECT_BIT, &ci->i_ceph_flags); + /* ensure modified bit is visible */ + smp_mb__after_atomic(); + } + spin_unlock(&ci->i_ceph_lock); + + if (!is_odirect) { /* FIXME: unmap_mapping_range? */ filemap_write_and_wait(inode->i_mapping); } @@ -133,20 +168,35 @@ static void ceph_block_buffered(struct ceph_inode_info *ci, struct inode *inode) * Note that buffered writes and truncates both take a write lock on * inode->i_rwsem, meaning that those are serialised w.r.t. O_DIRECT. */ -void -ceph_start_io_direct(struct inode *inode) +int ceph_start_io_direct(struct inode *inode) { struct ceph_inode_info *ci = ceph_inode(inode); + bool is_odirect; + int err; /* Be an optimist! */ - down_read(&inode->i_rwsem); - if (READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT) - return; + err = down_read_killable(&inode->i_rwsem); + if (err) + return err; + + spin_lock(&ci->i_ceph_lock); + /* ensure that bit state is consistent */ + smp_mb__before_atomic(); + is_odirect = READ_ONCE(ci->i_ceph_flags) & CEPH_I_ODIRECT; + spin_unlock(&ci->i_ceph_lock); + if (is_odirect) + return 0; up_read(&inode->i_rwsem); + /* Slow path.... */ - down_write(&inode->i_rwsem); + err = down_write_killable(&inode->i_rwsem); + if (err) + return err; + ceph_block_buffered(ci, inode); downgrade_write(&inode->i_rwsem); + + return 0; } /** |