Skip to content

Commit e7768e6

Browse files
torvaldstorvalds
committed
Merge tag 'for-6.8/dm-fixes-2' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm
Pull device mapper fixes from Mike Snitzer: - Stable fixes for 3 DM targets (integrity, verity and crypt) to address systemic failure that can occur if user provided pages map to the same block. - Fix DM crypt to not allow modifying data that being encrypted for authenticated encryption. - Fix DM crypt and verity targets to align their respective bvec_iter struct members to avoid the need for byte level access (due to __packed attribute) that is costly on some arches (like RISC). * tag 'for-6.8/dm-fixes-2' of git://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm: dm-crypt, dm-integrity, dm-verity: bump target version dm-verity, dm-crypt: align "struct bvec_iter" correctly dm-crypt: recheck the integrity tag after a failure dm-crypt: don't modify the data when using authenticated encryption dm-verity: recheck the hash after a failure dm-integrity: recheck the integrity tag after a failure
2 parents 06b7ef7 + 0e0c50e commit e7768e6

4 files changed

Lines changed: 256 additions & 38 deletions

File tree

‎drivers/md/dm-crypt.c‎

Lines changed: 82 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -53,15 +53,17 @@
5353
struct convert_context {
5454
struct completion restart;
5555
struct bio *bio_in;
56-
struct bio *bio_out;
5756
struct bvec_iter iter_in;
57+
struct bio *bio_out;
5858
struct bvec_iter iter_out;
59-
u64 cc_sector;
6059
atomic_t cc_pending;
60+
u64 cc_sector;
6161
union {
6262
struct skcipher_request *req;
6363
struct aead_request *req_aead;
6464
} r;
65+
bool aead_recheck;
66+
bool aead_failed;
6567

6668
};
6769

@@ -82,6 +84,8 @@ struct dm_crypt_io {
8284
blk_status_t error;
8385
sector_t sector;
8486

87+
struct bvec_iter saved_bi_iter;
88+
8589
struct rb_node rb_node;
8690
} CRYPTO_MINALIGN_ATTR;
8791

@@ -1370,10 +1374,13 @@ static int crypt_convert_block_aead(struct crypt_config *cc,
13701374
if (r == -EBADMSG) {
13711375
sector_t s = le64_to_cpu(*sector);
13721376

1373-
DMERR_LIMIT("%pg: INTEGRITY AEAD ERROR, sector %llu",
1374-
ctx->bio_in->bi_bdev, s);
1375-
dm_audit_log_bio(DM_MSG_PREFIX, "integrity-aead",
1376-
ctx->bio_in, s, 0);
1377+
ctx->aead_failed = true;
1378+
if (ctx->aead_recheck) {
1379+
DMERR_LIMIT("%pg: INTEGRITY AEAD ERROR, sector %llu",
1380+
ctx->bio_in->bi_bdev, s);
1381+
dm_audit_log_bio(DM_MSG_PREFIX, "integrity-aead",
1382+
ctx->bio_in, s, 0);
1383+
}
13771384
}
13781385

13791386
if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
@@ -1757,6 +1764,8 @@ static void crypt_io_init(struct dm_crypt_io *io, struct crypt_config *cc,
17571764
io->base_bio = bio;
17581765
io->sector = sector;
17591766
io->error = 0;
1767+
io->ctx.aead_recheck = false;
1768+
io->ctx.aead_failed = false;
17601769
io->ctx.r.req = NULL;
17611770
io->integrity_metadata = NULL;
17621771
io->integrity_metadata_from_pool = false;
@@ -1768,6 +1777,8 @@ static void crypt_inc_pending(struct dm_crypt_io *io)
17681777
atomic_inc(&io->io_pending);
17691778
}
17701779

1780+
static void kcryptd_queue_read(struct dm_crypt_io *io);
1781+
17711782
/*
17721783
* One of the bios was finished. Check for completion of
17731784
* the whole request and correctly clean up the buffer.
@@ -1781,6 +1792,15 @@ static void crypt_dec_pending(struct dm_crypt_io *io)
17811792
if (!atomic_dec_and_test(&io->io_pending))
17821793
return;
17831794

1795+
if (likely(!io->ctx.aead_recheck) && unlikely(io->ctx.aead_failed) &&
1796+
cc->on_disk_tag_size && bio_data_dir(base_bio) == READ) {
1797+
io->ctx.aead_recheck = true;
1798+
io->ctx.aead_failed = false;
1799+
io->error = 0;
1800+
kcryptd_queue_read(io);
1801+
return;
1802+
}
1803+
17841804
if (io->ctx.r.req)
17851805
crypt_free_req(cc, io->ctx.r.req, base_bio);
17861806

@@ -1816,15 +1836,19 @@ static void crypt_endio(struct bio *clone)
18161836
struct dm_crypt_io *io = clone->bi_private;
18171837
struct crypt_config *cc = io->cc;
18181838
unsigned int rw = bio_data_dir(clone);
1819-
blk_status_t error;
1839+
blk_status_t error = clone->bi_status;
1840+
1841+
if (io->ctx.aead_recheck && !error) {
1842+
kcryptd_queue_crypt(io);
1843+
return;
1844+
}
18201845

18211846
/*
18221847
* free the processed pages
18231848
*/
1824-
if (rw == WRITE)
1849+
if (rw == WRITE || io->ctx.aead_recheck)
18251850
crypt_free_buffer_pages(cc, clone);
18261851

1827-
error = clone->bi_status;
18281852
bio_put(clone);
18291853

18301854
if (rw == READ && !error) {
@@ -1845,6 +1869,22 @@ static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp)
18451869
struct crypt_config *cc = io->cc;
18461870
struct bio *clone;
18471871

1872+
if (io->ctx.aead_recheck) {
1873+
if (!(gfp & __GFP_DIRECT_RECLAIM))
1874+
return 1;
1875+
crypt_inc_pending(io);
1876+
clone = crypt_alloc_buffer(io, io->base_bio->bi_iter.bi_size);
1877+
if (unlikely(!clone)) {
1878+
crypt_dec_pending(io);
1879+
return 1;
1880+
}
1881+
clone->bi_iter.bi_sector = cc->start + io->sector;
1882+
crypt_convert_init(cc, &io->ctx, clone, clone, io->sector);
1883+
io->saved_bi_iter = clone->bi_iter;
1884+
dm_submit_bio_remap(io->base_bio, clone);
1885+
return 0;
1886+
}
1887+
18481888
/*
18491889
* We need the original biovec array in order to decrypt the whole bio
18501890
* data *afterwards* -- thanks to immutable biovecs we don't need to
@@ -2071,6 +2111,12 @@ static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)
20712111
io->ctx.bio_out = clone;
20722112
io->ctx.iter_out = clone->bi_iter;
20732113

2114+
if (crypt_integrity_aead(cc)) {
2115+
bio_copy_data(clone, io->base_bio);
2116+
io->ctx.bio_in = clone;
2117+
io->ctx.iter_in = clone->bi_iter;
2118+
}
2119+
20742120
sector += bio_sectors(clone);
20752121

20762122
crypt_inc_pending(io);
@@ -2107,6 +2153,14 @@ static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)
21072153

21082154
static void kcryptd_crypt_read_done(struct dm_crypt_io *io)
21092155
{
2156+
if (io->ctx.aead_recheck) {
2157+
if (!io->error) {
2158+
io->ctx.bio_in->bi_iter = io->saved_bi_iter;
2159+
bio_copy_data(io->base_bio, io->ctx.bio_in);
2160+
}
2161+
crypt_free_buffer_pages(io->cc, io->ctx.bio_in);
2162+
bio_put(io->ctx.bio_in);
2163+
}
21102164
crypt_dec_pending(io);
21112165
}
21122166

@@ -2136,11 +2190,17 @@ static void kcryptd_crypt_read_convert(struct dm_crypt_io *io)
21362190

21372191
crypt_inc_pending(io);
21382192

2139-
crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio,
2140-
io->sector);
2193+
if (io->ctx.aead_recheck) {
2194+
io->ctx.cc_sector = io->sector + cc->iv_offset;
2195+
r = crypt_convert(cc, &io->ctx,
2196+
test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags), true);
2197+
} else {
2198+
crypt_convert_init(cc, &io->ctx, io->base_bio, io->base_bio,
2199+
io->sector);
21412200

2142-
r = crypt_convert(cc, &io->ctx,
2143-
test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags), true);
2201+
r = crypt_convert(cc, &io->ctx,
2202+
test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags), true);
2203+
}
21442204
/*
21452205
* Crypto API backlogged the request, because its queue was full
21462206
* and we're in softirq context, so continue from a workqueue
@@ -2182,10 +2242,13 @@ static void kcryptd_async_done(void *data, int error)
21822242
if (error == -EBADMSG) {
21832243
sector_t s = le64_to_cpu(*org_sector_of_dmreq(cc, dmreq));
21842244

2185-
DMERR_LIMIT("%pg: INTEGRITY AEAD ERROR, sector %llu",
2186-
ctx->bio_in->bi_bdev, s);
2187-
dm_audit_log_bio(DM_MSG_PREFIX, "integrity-aead",
2188-
ctx->bio_in, s, 0);
2245+
ctx->aead_failed = true;
2246+
if (ctx->aead_recheck) {
2247+
DMERR_LIMIT("%pg: INTEGRITY AEAD ERROR, sector %llu",
2248+
ctx->bio_in->bi_bdev, s);
2249+
dm_audit_log_bio(DM_MSG_PREFIX, "integrity-aead",
2250+
ctx->bio_in, s, 0);
2251+
}
21892252
io->error = BLK_STS_PROTECTION;
21902253
} else if (error < 0)
21912254
io->error = BLK_STS_IOERR;
@@ -3110,7 +3173,7 @@ static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **ar
31103173
sval = strchr(opt_string + strlen("integrity:"), ':') + 1;
31113174
if (!strcasecmp(sval, "aead")) {
31123175
set_bit(CRYPT_MODE_INTEGRITY_AEAD, &cc->cipher_flags);
3113-
} else if (strcasecmp(sval, "none")) {
3176+
} else if (strcasecmp(sval, "none")) {
31143177
ti->error = "Unknown integrity profile";
31153178
return -EINVAL;
31163179
}
@@ -3639,7 +3702,7 @@ static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits)
36393702

36403703
static struct target_type crypt_target = {
36413704
.name = "crypt",
3642-
.version = {1, 24, 0},
3705+
.version = {1, 25, 0},
36433706
.module = THIS_MODULE,
36443707
.ctr = crypt_ctr,
36453708
.dtr = crypt_dtr,

‎drivers/md/dm-integrity.c‎

Lines changed: 85 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -278,6 +278,8 @@ struct dm_integrity_c {
278278

279279
atomic64_t number_of_mismatches;
280280

281+
mempool_t recheck_pool;
282+
281283
struct notifier_block reboot_notifier;
282284
};
283285

@@ -1689,6 +1691,79 @@ static void integrity_sector_checksum(struct dm_integrity_c *ic, sector_t sector
16891691
get_random_bytes(result, ic->tag_size);
16901692
}
16911693

1694+
static void integrity_recheck(struct dm_integrity_io *dio)
1695+
{
1696+
struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1697+
struct dm_integrity_c *ic = dio->ic;
1698+
struct bvec_iter iter;
1699+
struct bio_vec bv;
1700+
sector_t sector, logical_sector, area, offset;
1701+
char checksum_onstack[max_t(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)];
1702+
struct page *page;
1703+
void *buffer;
1704+
1705+
get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);
1706+
dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset,
1707+
&dio->metadata_offset);
1708+
sector = get_data_sector(ic, area, offset);
1709+
logical_sector = dio->range.logical_sector;
1710+
1711+
page = mempool_alloc(&ic->recheck_pool, GFP_NOIO);
1712+
buffer = page_to_virt(page);
1713+
1714+
__bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) {
1715+
unsigned pos = 0;
1716+
1717+
do {
1718+
char *mem;
1719+
int r;
1720+
struct dm_io_request io_req;
1721+
struct dm_io_region io_loc;
1722+
io_req.bi_opf = REQ_OP_READ;
1723+
io_req.mem.type = DM_IO_KMEM;
1724+
io_req.mem.ptr.addr = buffer;
1725+
io_req.notify.fn = NULL;
1726+
io_req.client = ic->io;
1727+
io_loc.bdev = ic->dev->bdev;
1728+
io_loc.sector = sector;
1729+
io_loc.count = ic->sectors_per_block;
1730+
1731+
r = dm_io(&io_req, 1, &io_loc, NULL);
1732+
if (unlikely(r)) {
1733+
dio->bi_status = errno_to_blk_status(r);
1734+
goto free_ret;
1735+
}
1736+
1737+
integrity_sector_checksum(ic, logical_sector, buffer,
1738+
checksum_onstack);
1739+
r = dm_integrity_rw_tag(ic, checksum_onstack, &dio->metadata_block,
1740+
&dio->metadata_offset, ic->tag_size, TAG_CMP);
1741+
if (r) {
1742+
if (r > 0) {
1743+
DMERR_LIMIT("%pg: Checksum failed at sector 0x%llx",
1744+
bio->bi_bdev, logical_sector);
1745+
atomic64_inc(&ic->number_of_mismatches);
1746+
dm_audit_log_bio(DM_MSG_PREFIX, "integrity-checksum",
1747+
bio, logical_sector, 0);
1748+
r = -EILSEQ;
1749+
}
1750+
dio->bi_status = errno_to_blk_status(r);
1751+
goto free_ret;
1752+
}
1753+
1754+
mem = bvec_kmap_local(&bv);
1755+
memcpy(mem + pos, buffer, ic->sectors_per_block << SECTOR_SHIFT);
1756+
kunmap_local(mem);
1757+
1758+
pos += ic->sectors_per_block << SECTOR_SHIFT;
1759+
sector += ic->sectors_per_block;
1760+
logical_sector += ic->sectors_per_block;
1761+
} while (pos < bv.bv_len);
1762+
}
1763+
free_ret:
1764+
mempool_free(page, &ic->recheck_pool);
1765+
}
1766+
16921767
static void integrity_metadata(struct work_struct *w)
16931768
{
16941769
struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);
@@ -1776,15 +1851,8 @@ static void integrity_metadata(struct work_struct *w)
17761851
checksums_ptr - checksums, dio->op == REQ_OP_READ ? TAG_CMP : TAG_WRITE);
17771852
if (unlikely(r)) {
17781853
if (r > 0) {
1779-
sector_t s;
1780-
1781-
s = sector - ((r + ic->tag_size - 1) / ic->tag_size);
1782-
DMERR_LIMIT("%pg: Checksum failed at sector 0x%llx",
1783-
bio->bi_bdev, s);
1784-
r = -EILSEQ;
1785-
atomic64_inc(&ic->number_of_mismatches);
1786-
dm_audit_log_bio(DM_MSG_PREFIX, "integrity-checksum",
1787-
bio, s, 0);
1854+
integrity_recheck(dio);
1855+
goto skip_io;
17881856
}
17891857
if (likely(checksums != checksums_onstack))
17901858
kfree(checksums);
@@ -4261,6 +4329,12 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv
42614329
goto bad;
42624330
}
42634331

4332+
r = mempool_init_page_pool(&ic->recheck_pool, 1, 0);
4333+
if (r) {
4334+
ti->error = "Cannot allocate mempool";
4335+
goto bad;
4336+
}
4337+
42644338
ic->metadata_wq = alloc_workqueue("dm-integrity-metadata",
42654339
WQ_MEM_RECLAIM, METADATA_WORKQUEUE_MAX_ACTIVE);
42664340
if (!ic->metadata_wq) {
@@ -4609,6 +4683,7 @@ static void dm_integrity_dtr(struct dm_target *ti)
46094683
kvfree(ic->bbs);
46104684
if (ic->bufio)
46114685
dm_bufio_client_destroy(ic->bufio);
4686+
mempool_exit(&ic->recheck_pool);
46124687
mempool_exit(&ic->journal_io_mempool);
46134688
if (ic->io)
46144689
dm_io_client_destroy(ic->io);
@@ -4661,7 +4736,7 @@ static void dm_integrity_dtr(struct dm_target *ti)
46614736

46624737
static struct target_type integrity_target = {
46634738
.name = "integrity",
4664-
.version = {1, 10, 0},
4739+
.version = {1, 11, 0},
46654740
.module = THIS_MODULE,
46664741
.features = DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY,
46674742
.ctr = dm_integrity_ctr,

0 commit comments

Comments
 (0)