Skip to content

Commit a149191

Browse files
Jonathan CurleyAnna Schumaker
Jonathan Curley
authored and
Anna Schumaker
committed
NFSv4/flexfiles: Update low level helper functions to be DS stripe aware.
Updates common helper functions to be dss_id aware. Most cases simply add a dss_id parameter. The has_available functions have been updated with a loop. Signed-off-by: Jonathan Curley <jcurley@purestorage.com> Signed-off-by: Anna Schumaker <anna.schumaker@oracle.com>
1 parent d442670 commit a149191

3 files changed

Lines changed: 115 additions & 85 deletions

File tree

‎fs/nfs/flexfilelayout/flexfilelayout.c‎

Lines changed: 28 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -164,14 +164,14 @@ decode_name(struct xdr_stream *xdr, u32 *id)
164164
}
165165

166166
static struct nfsd_file *
167-
ff_local_open_fh(struct pnfs_layout_segment *lseg, u32 ds_idx,
167+
ff_local_open_fh(struct pnfs_layout_segment *lseg, u32 ds_idx, u32 dss_id,
168168
struct nfs_client *clp, const struct cred *cred,
169169
struct nfs_fh *fh, fmode_t mode)
170170
{
171171
#if IS_ENABLED(CONFIG_NFS_LOCALIO)
172172
struct nfs4_ff_layout_mirror *mirror = FF_LAYOUT_COMP(lseg, ds_idx);
173173

174-
return nfs_local_open_fh(clp, cred, fh, &mirror->dss[0].nfl, mode);
174+
return nfs_local_open_fh(clp, cred, fh, &mirror->dss[dss_id].nfl, mode);
175175
#else
176176
return NULL;
177177
#endif
@@ -752,7 +752,7 @@ nfs4_ff_layout_stat_io_end_write(struct rpc_task *task,
752752
static void
753753
ff_layout_mark_ds_unreachable(struct pnfs_layout_segment *lseg, u32 idx)
754754
{
755-
struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx);
755+
struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx, 0);
756756

757757
if (devid)
758758
nfs4_mark_deviceid_unavailable(devid);
@@ -761,7 +761,7 @@ ff_layout_mark_ds_unreachable(struct pnfs_layout_segment *lseg, u32 idx)
761761
static void
762762
ff_layout_mark_ds_reachable(struct pnfs_layout_segment *lseg, u32 idx)
763763
{
764-
struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx);
764+
struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx, 0);
765765

766766
if (devid)
767767
nfs4_mark_deviceid_available(devid);
@@ -780,7 +780,7 @@ ff_layout_choose_ds_for_read(struct pnfs_layout_segment *lseg,
780780
/* mirrors are initially sorted by efficiency */
781781
for (idx = start_idx; idx < fls->mirror_array_cnt; idx++) {
782782
mirror = FF_LAYOUT_COMP(lseg, idx);
783-
ds = nfs4_ff_layout_prepare_ds(lseg, mirror, false);
783+
ds = nfs4_ff_layout_prepare_ds(lseg, mirror, 0, false);
784784
if (IS_ERR(ds))
785785
continue;
786786

@@ -956,7 +956,7 @@ ff_layout_pg_init_write(struct nfs_pageio_descriptor *pgio,
956956

957957
for (i = 0; i < pgio->pg_mirror_count; i++) {
958958
mirror = FF_LAYOUT_COMP(pgio->pg_lseg, i);
959-
ds = nfs4_ff_layout_prepare_ds(pgio->pg_lseg, mirror, true);
959+
ds = nfs4_ff_layout_prepare_ds(pgio->pg_lseg, mirror, 0, true);
960960
if (IS_ERR(ds)) {
961961
if (!ff_layout_no_fallback_to_mds(pgio->pg_lseg))
962962
goto out_mds;
@@ -1130,7 +1130,7 @@ static int ff_layout_async_handle_error_v4(struct rpc_task *task,
11301130
{
11311131
struct pnfs_layout_hdr *lo = lseg->pls_layout;
11321132
struct inode *inode = lo->plh_inode;
1133-
struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx);
1133+
struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx, 0);
11341134
struct nfs4_slot_table *tbl = &clp->cl_session->fc_slot_table;
11351135

11361136
switch (op_status) {
@@ -1229,7 +1229,7 @@ static int ff_layout_async_handle_error_v3(struct rpc_task *task,
12291229
struct pnfs_layout_segment *lseg,
12301230
u32 idx)
12311231
{
1232-
struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx);
1232+
struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx, 0);
12331233

12341234
switch (op_status) {
12351235
case NFS_OK:
@@ -1359,7 +1359,7 @@ static void ff_layout_io_track_ds_error(struct pnfs_layout_segment *lseg,
13591359

13601360
mirror = FF_LAYOUT_COMP(lseg, idx);
13611361
err = ff_layout_track_ds_error(FF_LAYOUT_FROM_HDR(lseg->pls_layout),
1362-
mirror, offset, length, status, opnum,
1362+
mirror, 0, offset, length, status, opnum,
13631363
nfs_io_gfp_mask());
13641364

13651365
switch (status) {
@@ -1891,34 +1891,34 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr)
18911891
hdr->args.pgbase, (size_t)hdr->args.count, offset);
18921892

18931893
mirror = FF_LAYOUT_COMP(lseg, idx);
1894-
ds = nfs4_ff_layout_prepare_ds(lseg, mirror, false);
1894+
ds = nfs4_ff_layout_prepare_ds(lseg, mirror, 0, false);
18951895
if (IS_ERR(ds)) {
18961896
ds_fatal_error = nfs_error_is_fatal(PTR_ERR(ds));
18971897
goto out_failed;
18981898
}
18991899

19001900
ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp,
1901-
hdr->inode);
1901+
hdr->inode, 0);
19021902
if (IS_ERR(ds_clnt))
19031903
goto out_failed;
19041904

1905-
ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, hdr->cred);
1905+
ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, hdr->cred, 0);
19061906
if (!ds_cred)
19071907
goto out_failed;
19081908

1909-
vers = nfs4_ff_layout_ds_version(mirror);
1909+
vers = nfs4_ff_layout_ds_version(mirror, 0);
19101910

19111911
dprintk("%s USE DS: %s cl_count %d vers %d\n", __func__,
19121912
ds->ds_remotestr, refcount_read(&ds->ds_clp->cl_count), vers);
19131913

19141914
hdr->pgio_done_cb = ff_layout_read_done_cb;
19151915
refcount_inc(&ds->ds_clp->cl_count);
19161916
hdr->ds_clp = ds->ds_clp;
1917-
fh = nfs4_ff_layout_select_ds_fh(mirror);
1917+
fh = nfs4_ff_layout_select_ds_fh(mirror, 0);
19181918
if (fh)
19191919
hdr->args.fh = fh;
19201920

1921-
nfs4_ff_layout_select_ds_stateid(mirror, &hdr->args.stateid);
1921+
nfs4_ff_layout_select_ds_stateid(mirror, 0, &hdr->args.stateid);
19221922

19231923
/*
19241924
* Note that if we ever decide to split across DSes,
@@ -1928,7 +1928,7 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr)
19281928
hdr->mds_offset = offset;
19291929

19301930
/* Start IO accounting for local read */
1931-
localio = ff_local_open_fh(lseg, idx, ds->ds_clp, ds_cred, fh, FMODE_READ);
1931+
localio = ff_local_open_fh(lseg, idx, 0, ds->ds_clp, ds_cred, fh, FMODE_READ);
19321932
if (localio) {
19331933
hdr->task.tk_start = ktime_get();
19341934
ff_layout_read_record_layoutstats_start(&hdr->task, hdr);
@@ -1968,22 +1968,22 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync)
19681968
bool ds_fatal_error = false;
19691969

19701970
mirror = FF_LAYOUT_COMP(lseg, idx);
1971-
ds = nfs4_ff_layout_prepare_ds(lseg, mirror, true);
1971+
ds = nfs4_ff_layout_prepare_ds(lseg, mirror, 0, true);
19721972
if (IS_ERR(ds)) {
19731973
ds_fatal_error = nfs_error_is_fatal(PTR_ERR(ds));
19741974
goto out_failed;
19751975
}
19761976

19771977
ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp,
1978-
hdr->inode);
1978+
hdr->inode, 0);
19791979
if (IS_ERR(ds_clnt))
19801980
goto out_failed;
19811981

1982-
ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, hdr->cred);
1982+
ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, hdr->cred, 0);
19831983
if (!ds_cred)
19841984
goto out_failed;
19851985

1986-
vers = nfs4_ff_layout_ds_version(mirror);
1986+
vers = nfs4_ff_layout_ds_version(mirror, 0);
19871987

19881988
dprintk("%s ino %lu sync %d req %zu@%llu DS: %s cl_count %d vers %d\n",
19891989
__func__, hdr->inode->i_ino, sync, (size_t) hdr->args.count,
@@ -1994,11 +1994,11 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync)
19941994
refcount_inc(&ds->ds_clp->cl_count);
19951995
hdr->ds_clp = ds->ds_clp;
19961996
hdr->ds_commit_idx = idx;
1997-
fh = nfs4_ff_layout_select_ds_fh(mirror);
1997+
fh = nfs4_ff_layout_select_ds_fh(mirror, 0);
19981998
if (fh)
19991999
hdr->args.fh = fh;
20002000

2001-
nfs4_ff_layout_select_ds_stateid(mirror, &hdr->args.stateid);
2001+
nfs4_ff_layout_select_ds_stateid(mirror, 0, &hdr->args.stateid);
20022002

20032003
/*
20042004
* Note that if we ever decide to split across DSes,
@@ -2007,7 +2007,7 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync)
20072007
hdr->args.offset = offset;
20082008

20092009
/* Start IO accounting for local write */
2010-
localio = ff_local_open_fh(lseg, idx, ds->ds_clp, ds_cred, fh,
2010+
localio = ff_local_open_fh(lseg, idx, 0, ds->ds_clp, ds_cred, fh,
20112011
FMODE_READ|FMODE_WRITE);
20122012
if (localio) {
20132013
hdr->task.tk_start = ktime_get();
@@ -2065,20 +2065,20 @@ static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how)
20652065

20662066
idx = calc_ds_index_from_commit(lseg, data->ds_commit_index);
20672067
mirror = FF_LAYOUT_COMP(lseg, idx);
2068-
ds = nfs4_ff_layout_prepare_ds(lseg, mirror, true);
2068+
ds = nfs4_ff_layout_prepare_ds(lseg, mirror, 0, true);
20692069
if (IS_ERR(ds))
20702070
goto out_err;
20712071

20722072
ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp,
2073-
data->inode);
2073+
data->inode, 0);
20742074
if (IS_ERR(ds_clnt))
20752075
goto out_err;
20762076

2077-
ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, data->cred);
2077+
ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, data->cred, 0);
20782078
if (!ds_cred)
20792079
goto out_err;
20802080

2081-
vers = nfs4_ff_layout_ds_version(mirror);
2081+
vers = nfs4_ff_layout_ds_version(mirror, 0);
20822082

20832083
dprintk("%s ino %lu, how %d cl_count %d vers %d\n", __func__,
20842084
data->inode->i_ino, how, refcount_read(&ds->ds_clp->cl_count),
@@ -2092,7 +2092,7 @@ static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how)
20922092
data->args.fh = fh;
20932093

20942094
/* Start IO accounting for local commit */
2095-
localio = ff_local_open_fh(lseg, idx, ds->ds_clp, ds_cred, fh,
2095+
localio = ff_local_open_fh(lseg, idx, 0, ds->ds_clp, ds_cred, fh,
20962096
FMODE_READ|FMODE_WRITE);
20972097
if (localio) {
20982098
data->task.tk_start = ktime_get();

‎fs/nfs/flexfilelayout/flexfilelayout.h‎

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -157,12 +157,12 @@ FF_LAYOUT_COMP(struct pnfs_layout_segment *lseg, u32 idx)
157157
}
158158

159159
static inline struct nfs4_deviceid_node *
160-
FF_LAYOUT_DEVID_NODE(struct pnfs_layout_segment *lseg, u32 idx)
160+
FF_LAYOUT_DEVID_NODE(struct pnfs_layout_segment *lseg, u32 idx, u32 dss_id)
161161
{
162162
struct nfs4_ff_layout_mirror *mirror = FF_LAYOUT_COMP(lseg, idx);
163163

164164
if (mirror != NULL) {
165-
struct nfs4_ff_layout_ds *mirror_ds = mirror->dss[0].mirror_ds;
165+
struct nfs4_ff_layout_ds *mirror_ds = mirror->dss[dss_id].mirror_ds;
166166

167167
if (!IS_ERR_OR_NULL(mirror_ds))
168168
return &mirror_ds->id_node;
@@ -189,9 +189,22 @@ ff_layout_no_read_on_rw(struct pnfs_layout_segment *lseg)
189189
}
190190

191191
static inline int
192-
nfs4_ff_layout_ds_version(const struct nfs4_ff_layout_mirror *mirror)
192+
nfs4_ff_layout_ds_version(const struct nfs4_ff_layout_mirror *mirror, u32 dss_id)
193193
{
194-
return mirror->dss[0].mirror_ds->ds_versions[0].version;
194+
return mirror->dss[dss_id].mirror_ds->ds_versions[0].version;
195+
}
196+
197+
static inline u32
198+
nfs4_ff_layout_calc_dss_id(const u64 stripe_unit, const u32 dss_count, const loff_t offset)
199+
{
200+
u64 tmp = offset;
201+
202+
if (dss_count == 1 || stripe_unit == 0)
203+
return 0;
204+
205+
do_div(tmp, stripe_unit);
206+
207+
return do_div(tmp, dss_count);
195208
}
196209

197210
struct nfs4_ff_layout_ds *
@@ -200,9 +213,9 @@ nfs4_ff_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev,
200213
void nfs4_ff_layout_put_deviceid(struct nfs4_ff_layout_ds *mirror_ds);
201214
void nfs4_ff_layout_free_deviceid(struct nfs4_ff_layout_ds *mirror_ds);
202215
int ff_layout_track_ds_error(struct nfs4_flexfile_layout *flo,
203-
struct nfs4_ff_layout_mirror *mirror, u64 offset,
204-
u64 length, int status, enum nfs_opnum4 opnum,
205-
gfp_t gfp_flags);
216+
struct nfs4_ff_layout_mirror *mirror,
217+
u32 dss_id, u64 offset, u64 length, int status,
218+
enum nfs_opnum4 opnum, gfp_t gfp_flags);
206219
void ff_layout_send_layouterror(struct pnfs_layout_segment *lseg);
207220
int ff_layout_encode_ds_ioerr(struct xdr_stream *xdr, const struct list_head *head);
208221
void ff_layout_free_ds_ioerr(struct list_head *head);
@@ -211,23 +224,27 @@ unsigned int ff_layout_fetch_ds_ioerr(struct pnfs_layout_hdr *lo,
211224
struct list_head *head,
212225
unsigned int maxnum);
213226
struct nfs_fh *
214-
nfs4_ff_layout_select_ds_fh(struct nfs4_ff_layout_mirror *mirror);
227+
nfs4_ff_layout_select_ds_fh(struct nfs4_ff_layout_mirror *mirror, u32 dss_id);
215228
void
216229
nfs4_ff_layout_select_ds_stateid(const struct nfs4_ff_layout_mirror *mirror,
217-
nfs4_stateid *stateid);
230+
u32 dss_id,
231+
nfs4_stateid *stateid);
218232

219233
struct nfs4_pnfs_ds *
220234
nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg,
221235
struct nfs4_ff_layout_mirror *mirror,
236+
u32 dss_id,
222237
bool fail_return);
223238

224239
struct rpc_clnt *
225240
nfs4_ff_find_or_create_ds_client(struct nfs4_ff_layout_mirror *mirror,
226241
struct nfs_client *ds_clp,
227-
struct inode *inode);
242+
struct inode *inode,
243+
u32 dss_id);
228244
const struct cred *ff_layout_get_ds_cred(struct nfs4_ff_layout_mirror *mirror,
229245
const struct pnfs_layout_range *range,
230-
const struct cred *mdscred);
246+
const struct cred *mdscred,
247+
u32 dss_id);
231248
bool ff_layout_avoid_mds_available_ds(struct pnfs_layout_segment *lseg);
232249
bool ff_layout_avoid_read_on_rw(struct pnfs_layout_segment *lseg);
233250

0 commit comments

Comments
 (0)