Skip to content

Commit d442670

Browse files
Jonathan CurleyAnna Schumaker
Jonathan Curley
authored and
Anna Schumaker
committed
NFSv4/flexfiles: Add data structure support for striped layouts
Adds a new struct nfs4_ff_layout_ds_stripe that represents a data server stripe within a layout. A new dynamically allocated array of this type has been added to nfs4_ff_layout_mirror and per stripe configuration information has been moved from the mirror type to the stripe based on the RFC. Signed-off-by: Jonathan Curley <jcurley@purestorage.com> Signed-off-by: Anna Schumaker <anna.schumaker@oracle.com>
1 parent eb71428 commit d442670

3 files changed

Lines changed: 117 additions & 98 deletions

File tree

‎fs/nfs/flexfilelayout/flexfilelayout.c‎

Lines changed: 73 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,7 @@ ff_local_open_fh(struct pnfs_layout_segment *lseg, u32 ds_idx,
171171
#if IS_ENABLED(CONFIG_NFS_LOCALIO)
172172
struct nfs4_ff_layout_mirror *mirror = FF_LAYOUT_COMP(lseg, ds_idx);
173173

174-
return nfs_local_open_fh(clp, cred, fh, &mirror->nfl, mode);
174+
return nfs_local_open_fh(clp, cred, fh, &mirror->dss[0].nfl, mode);
175175
#else
176176
return NULL;
177177
#endif
@@ -182,13 +182,13 @@ static bool ff_mirror_match_fh(const struct nfs4_ff_layout_mirror *m1,
182182
{
183183
int i, j;
184184

185-
if (m1->fh_versions_cnt != m2->fh_versions_cnt)
185+
if (m1->dss[0].fh_versions_cnt != m2->dss[0].fh_versions_cnt)
186186
return false;
187-
for (i = 0; i < m1->fh_versions_cnt; i++) {
187+
for (i = 0; i < m1->dss[0].fh_versions_cnt; i++) {
188188
bool found_fh = false;
189-
for (j = 0; j < m2->fh_versions_cnt; j++) {
190-
if (nfs_compare_fh(&m1->fh_versions[i],
191-
&m2->fh_versions[j]) == 0) {
189+
for (j = 0; j < m2->dss[0].fh_versions_cnt; j++) {
190+
if (nfs_compare_fh(&m1->dss[0].fh_versions[i],
191+
&m2->dss[0].fh_versions[j]) == 0) {
192192
found_fh = true;
193193
break;
194194
}
@@ -209,7 +209,8 @@ ff_layout_add_mirror(struct pnfs_layout_hdr *lo,
209209

210210
spin_lock(&inode->i_lock);
211211
list_for_each_entry(pos, &ff_layout->mirrors, mirrors) {
212-
if (memcmp(&mirror->devid, &pos->devid, sizeof(pos->devid)) != 0)
212+
if (memcmp(&mirror->dss[0].devid, &pos->dss[0].devid,
213+
sizeof(pos->dss[0].devid)) != 0)
213214
continue;
214215
if (!ff_mirror_match_fh(mirror, pos))
215216
continue;
@@ -246,23 +247,27 @@ static struct nfs4_ff_layout_mirror *ff_layout_alloc_mirror(gfp_t gfp_flags)
246247
spin_lock_init(&mirror->lock);
247248
refcount_set(&mirror->ref, 1);
248249
INIT_LIST_HEAD(&mirror->mirrors);
249-
nfs_localio_file_init(&mirror->nfl);
250+
nfs_localio_file_init(&mirror->dss[0].nfl);
250251
}
251252
return mirror;
252253
}
253254

254255
static void ff_layout_free_mirror(struct nfs4_ff_layout_mirror *mirror)
255256
{
256-
const struct cred *cred;
257+
const struct cred *cred;
258+
int dss_id = 0;
257259

258260
ff_layout_remove_mirror(mirror);
259-
kfree(mirror->fh_versions);
260-
nfs_close_local_fh(&mirror->nfl);
261-
cred = rcu_access_pointer(mirror->ro_cred);
261+
262+
kfree(mirror->dss[dss_id].fh_versions);
263+
nfs_close_local_fh(&mirror->dss[dss_id].nfl);
264+
cred = rcu_access_pointer(mirror->dss[dss_id].ro_cred);
262265
put_cred(cred);
263-
cred = rcu_access_pointer(mirror->rw_cred);
266+
cred = rcu_access_pointer(mirror->dss[dss_id].rw_cred);
264267
put_cred(cred);
265-
nfs4_ff_layout_put_deviceid(mirror->mirror_ds);
268+
nfs4_ff_layout_put_deviceid(mirror->dss[dss_id].mirror_ds);
269+
270+
kfree(mirror->dss);
266271
kfree(mirror);
267272
}
268273

@@ -372,8 +377,8 @@ static void ff_layout_sort_mirrors(struct nfs4_ff_layout_segment *fls)
372377

373378
for (i = 0; i < fls->mirror_array_cnt - 1; i++) {
374379
for (j = i + 1; j < fls->mirror_array_cnt; j++)
375-
if (fls->mirror_array[i]->efficiency <
376-
fls->mirror_array[j]->efficiency)
380+
if (fls->mirror_array[i]->dss[0].efficiency <
381+
fls->mirror_array[j]->dss[0].efficiency)
377382
swap(fls->mirror_array[i],
378383
fls->mirror_array[j]);
379384
}
@@ -427,23 +432,25 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh,
427432
fls->mirror_array_cnt = mirror_array_cnt;
428433
fls->stripe_unit = stripe_unit;
429434

435+
u32 dss_count = 0;
430436
for (i = 0; i < fls->mirror_array_cnt; i++) {
431437
struct nfs4_ff_layout_mirror *mirror;
432438
struct cred *kcred;
433439
const struct cred __rcu *cred;
434440
kuid_t uid;
435441
kgid_t gid;
436-
u32 ds_count, fh_count, id;
437-
int j;
442+
u32 fh_count, id;
443+
int j, dss_id = 0;
438444

439445
rc = -EIO;
440446
p = xdr_inline_decode(&stream, 4);
441447
if (!p)
442448
goto out_err_free;
443-
ds_count = be32_to_cpup(p);
449+
450+
dss_count = be32_to_cpup(p);
444451

445452
/* FIXME: allow for striping? */
446-
if (ds_count != 1)
453+
if (dss_count != 1)
447454
goto out_err_free;
448455

449456
fls->mirror_array[i] = ff_layout_alloc_mirror(gfp_flags);
@@ -452,10 +459,13 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh,
452459
goto out_err_free;
453460
}
454461

455-
fls->mirror_array[i]->ds_count = ds_count;
462+
fls->mirror_array[i]->dss_count = dss_count;
463+
fls->mirror_array[i]->dss =
464+
kcalloc(dss_count, sizeof(struct nfs4_ff_layout_ds_stripe),
465+
gfp_flags);
456466

457467
/* deviceid */
458-
rc = decode_deviceid(&stream, &fls->mirror_array[i]->devid);
468+
rc = decode_deviceid(&stream, &fls->mirror_array[i]->dss[dss_id].devid);
459469
if (rc)
460470
goto out_err_free;
461471

@@ -464,10 +474,10 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh,
464474
p = xdr_inline_decode(&stream, 4);
465475
if (!p)
466476
goto out_err_free;
467-
fls->mirror_array[i]->efficiency = be32_to_cpup(p);
477+
fls->mirror_array[i]->dss[dss_id].efficiency = be32_to_cpup(p);
468478

469479
/* stateid */
470-
rc = decode_pnfs_stateid(&stream, &fls->mirror_array[i]->stateid);
480+
rc = decode_pnfs_stateid(&stream, &fls->mirror_array[i]->dss[dss_id].stateid);
471481
if (rc)
472482
goto out_err_free;
473483

@@ -478,22 +488,22 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh,
478488
goto out_err_free;
479489
fh_count = be32_to_cpup(p);
480490

481-
fls->mirror_array[i]->fh_versions =
482-
kcalloc(fh_count, sizeof(struct nfs_fh),
483-
gfp_flags);
484-
if (fls->mirror_array[i]->fh_versions == NULL) {
491+
fls->mirror_array[i]->dss[dss_id].fh_versions =
492+
kcalloc(fh_count, sizeof(struct nfs_fh),
493+
gfp_flags);
494+
if (fls->mirror_array[i]->dss[dss_id].fh_versions == NULL) {
485495
rc = -ENOMEM;
486496
goto out_err_free;
487497
}
488498

489499
for (j = 0; j < fh_count; j++) {
490500
rc = decode_nfs_fh(&stream,
491-
&fls->mirror_array[i]->fh_versions[j]);
501+
&fls->mirror_array[i]->dss[dss_id].fh_versions[j]);
492502
if (rc)
493503
goto out_err_free;
494504
}
495505

496-
fls->mirror_array[i]->fh_versions_cnt = fh_count;
506+
fls->mirror_array[i]->dss[dss_id].fh_versions_cnt = fh_count;
497507

498508
/* user */
499509
rc = decode_name(&stream, &id);
@@ -524,19 +534,21 @@ ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh,
524534
cred = RCU_INITIALIZER(kcred);
525535

526536
if (lgr->range.iomode == IOMODE_READ)
527-
rcu_assign_pointer(fls->mirror_array[i]->ro_cred, cred);
537+
rcu_assign_pointer(fls->mirror_array[i]->dss[dss_id].ro_cred, cred);
528538
else
529-
rcu_assign_pointer(fls->mirror_array[i]->rw_cred, cred);
539+
rcu_assign_pointer(fls->mirror_array[i]->dss[dss_id].rw_cred, cred);
530540

531541
mirror = ff_layout_add_mirror(lh, fls->mirror_array[i]);
532542
if (mirror != fls->mirror_array[i]) {
533543
/* swap cred ptrs so free_mirror will clean up old */
534544
if (lgr->range.iomode == IOMODE_READ) {
535-
cred = xchg(&mirror->ro_cred, fls->mirror_array[i]->ro_cred);
536-
rcu_assign_pointer(fls->mirror_array[i]->ro_cred, cred);
545+
cred = xchg(&mirror->dss[dss_id].ro_cred,
546+
fls->mirror_array[i]->dss[dss_id].ro_cred);
547+
rcu_assign_pointer(fls->mirror_array[i]->dss[dss_id].ro_cred, cred);
537548
} else {
538-
cred = xchg(&mirror->rw_cred, fls->mirror_array[i]->rw_cred);
539-
rcu_assign_pointer(fls->mirror_array[i]->rw_cred, cred);
549+
cred = xchg(&mirror->dss[dss_id].rw_cred,
550+
fls->mirror_array[i]->dss[dss_id].rw_cred);
551+
rcu_assign_pointer(fls->mirror_array[i]->dss[dss_id].rw_cred, cred);
540552
}
541553
ff_layout_free_mirror(fls->mirror_array[i]);
542554
fls->mirror_array[i] = mirror;
@@ -624,8 +636,8 @@ nfs4_ff_layoutstat_start_io(struct nfs4_ff_layout_mirror *mirror,
624636
struct nfs4_flexfile_layout *ffl = FF_LAYOUT_FROM_HDR(mirror->layout);
625637

626638
nfs4_ff_start_busy_timer(&layoutstat->busy_timer, now);
627-
if (!mirror->start_time)
628-
mirror->start_time = now;
639+
if (!mirror->dss[0].start_time)
640+
mirror->dss[0].start_time = now;
629641
if (mirror->report_interval != 0)
630642
report_interval = (s64)mirror->report_interval * 1000LL;
631643
else if (layoutstats_timer != 0)
@@ -680,8 +692,8 @@ nfs4_ff_layout_stat_io_start_read(struct inode *inode,
680692
bool report;
681693

682694
spin_lock(&mirror->lock);
683-
report = nfs4_ff_layoutstat_start_io(mirror, &mirror->read_stat, now);
684-
nfs4_ff_layout_stat_io_update_requested(&mirror->read_stat, requested);
695+
report = nfs4_ff_layoutstat_start_io(mirror, &mirror->dss[0].read_stat, now);
696+
nfs4_ff_layout_stat_io_update_requested(&mirror->dss[0].read_stat, requested);
685697
set_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags);
686698
spin_unlock(&mirror->lock);
687699

@@ -696,7 +708,7 @@ nfs4_ff_layout_stat_io_end_read(struct rpc_task *task,
696708
__u64 completed)
697709
{
698710
spin_lock(&mirror->lock);
699-
nfs4_ff_layout_stat_io_update_completed(&mirror->read_stat,
711+
nfs4_ff_layout_stat_io_update_completed(&mirror->dss[0].read_stat,
700712
requested, completed,
701713
ktime_get(), task->tk_start);
702714
set_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags);
@@ -711,8 +723,8 @@ nfs4_ff_layout_stat_io_start_write(struct inode *inode,
711723
bool report;
712724

713725
spin_lock(&mirror->lock);
714-
report = nfs4_ff_layoutstat_start_io(mirror , &mirror->write_stat, now);
715-
nfs4_ff_layout_stat_io_update_requested(&mirror->write_stat, requested);
726+
report = nfs4_ff_layoutstat_start_io(mirror, &mirror->dss[0].write_stat, now);
727+
nfs4_ff_layout_stat_io_update_requested(&mirror->dss[0].write_stat, requested);
716728
set_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags);
717729
spin_unlock(&mirror->lock);
718730

@@ -731,7 +743,7 @@ nfs4_ff_layout_stat_io_end_write(struct rpc_task *task,
731743
requested = completed = 0;
732744

733745
spin_lock(&mirror->lock);
734-
nfs4_ff_layout_stat_io_update_completed(&mirror->write_stat,
746+
nfs4_ff_layout_stat_io_update_completed(&mirror->dss[0].write_stat,
735747
requested, completed, ktime_get(), task->tk_start);
736748
set_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags);
737749
spin_unlock(&mirror->lock);
@@ -773,7 +785,7 @@ ff_layout_choose_ds_for_read(struct pnfs_layout_segment *lseg,
773785
continue;
774786

775787
if (check_device &&
776-
nfs4_test_deviceid_unavailable(&mirror->mirror_ds->id_node)) {
788+
nfs4_test_deviceid_unavailable(&mirror->dss[0].mirror_ds->id_node)) {
777789
// reinitialize the error state in case if this is the last iteration
778790
ds = ERR_PTR(-EINVAL);
779791
continue;
@@ -882,7 +894,7 @@ ff_layout_pg_init_read(struct nfs_pageio_descriptor *pgio,
882894

883895
mirror = FF_LAYOUT_COMP(pgio->pg_lseg, ds_idx);
884896
pgm = &pgio->pg_mirrors[0];
885-
pgm->pg_bsize = mirror->mirror_ds->ds_versions[0].rsize;
897+
pgm->pg_bsize = mirror->dss[0].mirror_ds->ds_versions[0].rsize;
886898

887899
pgio->pg_mirror_idx = ds_idx;
888900
return;
@@ -954,7 +966,7 @@ ff_layout_pg_init_write(struct nfs_pageio_descriptor *pgio,
954966
goto retry;
955967
}
956968
pgm = &pgio->pg_mirrors[i];
957-
pgm->pg_bsize = mirror->mirror_ds->ds_versions[0].wsize;
969+
pgm->pg_bsize = mirror->dss[0].mirror_ds->ds_versions[0].wsize;
958970
}
959971

960972
if (NFS_SERVER(pgio->pg_inode)->flags &
@@ -2032,7 +2044,7 @@ select_ds_fh_from_commit(struct pnfs_layout_segment *lseg, u32 i)
20322044
/* FIXME: Assume that there is only one NFS version available
20332045
* for the DS.
20342046
*/
2035-
return &flseg->mirror_array[i]->fh_versions[0];
2047+
return &flseg->mirror_array[i]->dss[0].fh_versions[0];
20362048
}
20372049

20382050
static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how)
@@ -2148,10 +2160,10 @@ static void ff_layout_cancel_io(struct pnfs_layout_segment *lseg)
21482160

21492161
for (idx = 0; idx < flseg->mirror_array_cnt; idx++) {
21502162
mirror = flseg->mirror_array[idx];
2151-
mirror_ds = mirror->mirror_ds;
2163+
mirror_ds = mirror->dss[0].mirror_ds;
21522164
if (IS_ERR_OR_NULL(mirror_ds))
21532165
continue;
2154-
ds = mirror->mirror_ds->ds;
2166+
ds = mirror->dss[0].mirror_ds->ds;
21552167
if (!ds)
21562168
continue;
21572169
ds_clp = ds->ds_clp;
@@ -2552,8 +2564,8 @@ ff_layout_encode_ff_layoutupdate(struct xdr_stream *xdr,
25522564
struct nfs4_ff_layout_mirror *mirror)
25532565
{
25542566
struct nfs4_pnfs_ds_addr *da;
2555-
struct nfs4_pnfs_ds *ds = mirror->mirror_ds->ds;
2556-
struct nfs_fh *fh = &mirror->fh_versions[0];
2567+
struct nfs4_pnfs_ds *ds = mirror->dss[0].mirror_ds->ds;
2568+
struct nfs_fh *fh = &mirror->dss[0].fh_versions[0];
25572569
__be32 *p;
25582570

25592571
da = list_first_entry(&ds->ds_addrs, struct nfs4_pnfs_ds_addr, da_node);
@@ -2566,12 +2578,12 @@ ff_layout_encode_ff_layoutupdate(struct xdr_stream *xdr,
25662578
xdr_encode_opaque(p, fh->data, fh->size);
25672579
/* ff_io_latency4 read */
25682580
spin_lock(&mirror->lock);
2569-
ff_layout_encode_io_latency(xdr, &mirror->read_stat.io_stat);
2581+
ff_layout_encode_io_latency(xdr, &mirror->dss[0].read_stat.io_stat);
25702582
/* ff_io_latency4 write */
2571-
ff_layout_encode_io_latency(xdr, &mirror->write_stat.io_stat);
2583+
ff_layout_encode_io_latency(xdr, &mirror->dss[0].write_stat.io_stat);
25722584
spin_unlock(&mirror->lock);
25732585
/* nfstime4 */
2574-
ff_layout_encode_nfstime(xdr, ktime_sub(ktime_get(), mirror->start_time));
2586+
ff_layout_encode_nfstime(xdr, ktime_sub(ktime_get(), mirror->dss[0].start_time));
25752587
/* bool */
25762588
p = xdr_reserve_space(xdr, 4);
25772589
*p = cpu_to_be32(false);
@@ -2618,7 +2630,7 @@ ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo,
26182630
list_for_each_entry(mirror, &ff_layout->mirrors, mirrors) {
26192631
if (i >= dev_limit)
26202632
break;
2621-
if (IS_ERR_OR_NULL(mirror->mirror_ds))
2633+
if (IS_ERR_OR_NULL(mirror->dss[0].mirror_ds))
26222634
continue;
26232635
if (!test_and_clear_bit(NFS4_FF_MIRROR_STAT_AVAIL,
26242636
&mirror->flags) &&
@@ -2627,15 +2639,15 @@ ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo,
26272639
/* mirror refcount put in cleanup_layoutstats */
26282640
if (!refcount_inc_not_zero(&mirror->ref))
26292641
continue;
2630-
dev = &mirror->mirror_ds->id_node;
2642+
dev = &mirror->dss[0].mirror_ds->id_node;
26312643
memcpy(&devinfo->dev_id, &dev->deviceid, NFS4_DEVICEID4_SIZE);
26322644
devinfo->offset = 0;
26332645
devinfo->length = NFS4_MAX_UINT64;
26342646
spin_lock(&mirror->lock);
2635-
devinfo->read_count = mirror->read_stat.io_stat.ops_completed;
2636-
devinfo->read_bytes = mirror->read_stat.io_stat.bytes_completed;
2637-
devinfo->write_count = mirror->write_stat.io_stat.ops_completed;
2638-
devinfo->write_bytes = mirror->write_stat.io_stat.bytes_completed;
2647+
devinfo->read_count = mirror->dss[0].read_stat.io_stat.ops_completed;
2648+
devinfo->read_bytes = mirror->dss[0].read_stat.io_stat.bytes_completed;
2649+
devinfo->write_count = mirror->dss[0].write_stat.io_stat.ops_completed;
2650+
devinfo->write_bytes = mirror->dss[0].write_stat.io_stat.bytes_completed;
26392651
spin_unlock(&mirror->lock);
26402652
devinfo->layout_type = LAYOUT_FLEX_FILES;
26412653
devinfo->ld_private.ops = &layoutstat_ops;

0 commit comments

Comments
 (0)