Skip to content

Commit 52f1ca8

Browse files
hygonitehcaster
hygoni
authored and
tehcaster
committed
mm/slab: abstract slabobj_ext access via new slab_obj_ext() helper
Currently, the slab allocator assumes that slab->obj_exts is a pointer to an array of struct slabobj_ext objects. However, to support storage methods where struct slabobj_ext is embedded within objects, the slab allocator should not make this assumption. Instead of directly dereferencing the slabobj_exts array, abstract access to struct slabobj_ext via helper functions. Introduce a new API slabobj_ext metadata access: slab_obj_ext(slab, obj_exts, index) - returns the pointer to struct slabobj_ext element at the given index. Directly dereferencing the return value of slab_obj_exts() is no longer allowed. Instead, slab_obj_ext() must always be used to access individual struct slabobj_ext objects. Convert all users to use these APIs. No functional changes intended. Signed-off-by: Harry Yoo <harry.yoo@oracle.com> Link: https://patch.msgid.link/20260113061845.159790-5-harry.yoo@oracle.com Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
1 parent 43d9bb4 commit 52f1ca8

3 files changed

Lines changed: 79 additions & 32 deletions

File tree

‎mm/memcontrol.c‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2596,16 +2596,18 @@ struct mem_cgroup *mem_cgroup_from_obj_slab(struct slab *slab, void *p)
25962596
* Memcg membership data for each individual object is saved in
25972597
* slab->obj_exts.
25982598
*/
2599-
struct slabobj_ext *obj_exts;
2599+
unsigned long obj_exts;
2600+
struct slabobj_ext *obj_ext;
26002601
unsigned int off;
26012602

26022603
obj_exts = slab_obj_exts(slab);
26032604
if (!obj_exts)
26042605
return NULL;
26052606

26062607
off = obj_to_index(slab->slab_cache, slab, p);
2607-
if (obj_exts[off].objcg)
2608-
return obj_cgroup_memcg(obj_exts[off].objcg);
2608+
obj_ext = slab_obj_ext(slab, obj_exts, off);
2609+
if (obj_ext->objcg)
2610+
return obj_cgroup_memcg(obj_ext->objcg);
26092611

26102612
return NULL;
26112613
}
@@ -3191,6 +3193,9 @@ bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
31913193
}
31923194

31933195
for (i = 0; i < size; i++) {
3196+
unsigned long obj_exts;
3197+
struct slabobj_ext *obj_ext;
3198+
31943199
slab = virt_to_slab(p[i]);
31953200

31963201
if (!slab_obj_exts(slab) &&
@@ -3213,29 +3218,33 @@ bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
32133218
slab_pgdat(slab), cache_vmstat_idx(s)))
32143219
return false;
32153220

3221+
obj_exts = slab_obj_exts(slab);
32163222
off = obj_to_index(s, slab, p[i]);
3223+
obj_ext = slab_obj_ext(slab, obj_exts, off);
32173224
obj_cgroup_get(objcg);
3218-
slab_obj_exts(slab)[off].objcg = objcg;
3225+
obj_ext->objcg = objcg;
32193226
}
32203227

32213228
return true;
32223229
}
32233230

32243231
void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
3225-
void **p, int objects, struct slabobj_ext *obj_exts)
3232+
void **p, int objects, unsigned long obj_exts)
32263233
{
32273234
size_t obj_size = obj_full_size(s);
32283235

32293236
for (int i = 0; i < objects; i++) {
32303237
struct obj_cgroup *objcg;
3238+
struct slabobj_ext *obj_ext;
32313239
unsigned int off;
32323240

32333241
off = obj_to_index(s, slab, p[i]);
3234-
objcg = obj_exts[off].objcg;
3242+
obj_ext = slab_obj_ext(slab, obj_exts, off);
3243+
objcg = obj_ext->objcg;
32353244
if (!objcg)
32363245
continue;
32373246

3238-
obj_exts[off].objcg = NULL;
3247+
obj_ext->objcg = NULL;
32393248
refill_obj_stock(objcg, obj_size, true, -obj_size,
32403249
slab_pgdat(slab), cache_vmstat_idx(s));
32413250
obj_cgroup_put(objcg);

‎mm/slab.h‎

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -507,10 +507,12 @@ static inline bool slab_in_kunit_test(void) { return false; }
507507
* associated with a slab.
508508
* @slab: a pointer to the slab struct
509509
*
510-
* Returns a pointer to the object extension vector associated with the slab,
511-
* or NULL if no such vector has been associated yet.
510+
* Returns the address of the object extension vector associated with the slab,
511+
* or zero if no such vector has been associated yet.
512+
* Do not dereference the return value directly; use slab_obj_ext() to access
513+
* its elements.
512514
*/
513-
static inline struct slabobj_ext *slab_obj_exts(struct slab *slab)
515+
static inline unsigned long slab_obj_exts(struct slab *slab)
514516
{
515517
unsigned long obj_exts = READ_ONCE(slab->obj_exts);
516518

@@ -523,15 +525,44 @@ static inline struct slabobj_ext *slab_obj_exts(struct slab *slab)
523525
obj_exts != OBJEXTS_ALLOC_FAIL, slab_page(slab));
524526
VM_BUG_ON_PAGE(obj_exts & MEMCG_DATA_KMEM, slab_page(slab));
525527
#endif
526-
return (struct slabobj_ext *)(obj_exts & ~OBJEXTS_FLAGS_MASK);
528+
529+
return obj_exts & ~OBJEXTS_FLAGS_MASK;
530+
}
531+
532+
/*
533+
* slab_obj_ext - get the pointer to the slab object extension metadata
534+
* associated with an object in a slab.
535+
* @slab: a pointer to the slab struct
536+
* @obj_exts: a pointer to the object extension vector
537+
* @index: an index of the object
538+
*
539+
* Returns a pointer to the object extension associated with the object.
540+
*/
541+
static inline struct slabobj_ext *slab_obj_ext(struct slab *slab,
542+
unsigned long obj_exts,
543+
unsigned int index)
544+
{
545+
struct slabobj_ext *obj_ext;
546+
547+
VM_WARN_ON_ONCE(obj_exts != slab_obj_exts(slab));
548+
549+
obj_ext = (struct slabobj_ext *)obj_exts;
550+
return &obj_ext[index];
527551
}
528552

529553
int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
530554
gfp_t gfp, bool new_slab);
531555

532556
#else /* CONFIG_SLAB_OBJ_EXT */
533557

534-
static inline struct slabobj_ext *slab_obj_exts(struct slab *slab)
558+
static inline unsigned long slab_obj_exts(struct slab *slab)
559+
{
560+
return 0;
561+
}
562+
563+
static inline struct slabobj_ext *slab_obj_ext(struct slab *slab,
564+
unsigned long obj_exts,
565+
unsigned int index)
535566
{
536567
return NULL;
537568
}
@@ -548,7 +579,7 @@ static inline enum node_stat_item cache_vmstat_idx(struct kmem_cache *s)
548579
bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
549580
gfp_t flags, size_t size, void **p);
550581
void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
551-
void **p, int objects, struct slabobj_ext *obj_exts);
582+
void **p, int objects, unsigned long obj_exts);
552583
#endif
553584

554585
void kvfree_rcu_cb(struct rcu_head *head);

‎mm/slub.c‎

Lines changed: 26 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2055,21 +2055,23 @@ static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,
20552055

20562056
static inline void mark_objexts_empty(struct slabobj_ext *obj_exts)
20572057
{
2058-
struct slabobj_ext *slab_exts;
2058+
unsigned long slab_exts;
20592059
struct slab *obj_exts_slab;
20602060

20612061
obj_exts_slab = virt_to_slab(obj_exts);
20622062
slab_exts = slab_obj_exts(obj_exts_slab);
20632063
if (slab_exts) {
20642064
unsigned int offs = obj_to_index(obj_exts_slab->slab_cache,
20652065
obj_exts_slab, obj_exts);
2066+
struct slabobj_ext *ext = slab_obj_ext(obj_exts_slab,
2067+
slab_exts, offs);
20662068

2067-
if (unlikely(is_codetag_empty(&slab_exts[offs].ref)))
2069+
if (unlikely(is_codetag_empty(&ext->ref)))
20682070
return;
20692071

20702072
/* codetag should be NULL here */
2071-
WARN_ON(slab_exts[offs].ref.ct);
2072-
set_codetag_empty(&slab_exts[offs].ref);
2073+
WARN_ON(ext->ref.ct);
2074+
set_codetag_empty(&ext->ref);
20732075
}
20742076
}
20752077

@@ -2237,7 +2239,7 @@ static inline void free_slab_obj_exts(struct slab *slab)
22372239
{
22382240
struct slabobj_ext *obj_exts;
22392241

2240-
obj_exts = slab_obj_exts(slab);
2242+
obj_exts = (struct slabobj_ext *)slab_obj_exts(slab);
22412243
if (!obj_exts) {
22422244
/*
22432245
* If obj_exts allocation failed, slab->obj_exts is set to
@@ -2284,26 +2286,29 @@ static inline void free_slab_obj_exts(struct slab *slab)
22842286
#ifdef CONFIG_MEM_ALLOC_PROFILING
22852287

22862288
static inline struct slabobj_ext *
2287-
prepare_slab_obj_exts_hook(struct kmem_cache *s, gfp_t flags, void *p)
2289+
prepare_slab_obj_ext_hook(struct kmem_cache *s, gfp_t flags, void *p)
22882290
{
22892291
struct slab *slab;
2292+
unsigned long obj_exts;
22902293

22912294
slab = virt_to_slab(p);
2292-
if (!slab_obj_exts(slab) &&
2295+
obj_exts = slab_obj_exts(slab);
2296+
if (!obj_exts &&
22932297
alloc_slab_obj_exts(slab, s, flags, false)) {
22942298
pr_warn_once("%s, %s: Failed to create slab extension vector!\n",
22952299
__func__, s->name);
22962300
return NULL;
22972301
}
22982302

2299-
return slab_obj_exts(slab) + obj_to_index(s, slab, p);
2303+
obj_exts = slab_obj_exts(slab);
2304+
return slab_obj_ext(slab, obj_exts, obj_to_index(s, slab, p));
23002305
}
23012306

23022307
/* Should be called only if mem_alloc_profiling_enabled() */
23032308
static noinline void
23042309
__alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
23052310
{
2306-
struct slabobj_ext *obj_exts;
2311+
struct slabobj_ext *obj_ext;
23072312

23082313
if (!object)
23092314
return;
@@ -2314,14 +2319,14 @@ __alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
23142319
if (flags & __GFP_NO_OBJ_EXT)
23152320
return;
23162321

2317-
obj_exts = prepare_slab_obj_exts_hook(s, flags, object);
2322+
obj_ext = prepare_slab_obj_ext_hook(s, flags, object);
23182323
/*
23192324
* Currently obj_exts is used only for allocation profiling.
23202325
* If other users appear then mem_alloc_profiling_enabled()
23212326
* check should be added before alloc_tag_add().
23222327
*/
2323-
if (likely(obj_exts))
2324-
alloc_tag_add(&obj_exts->ref, current->alloc_tag, s->size);
2328+
if (likely(obj_ext))
2329+
alloc_tag_add(&obj_ext->ref, current->alloc_tag, s->size);
23252330
else
23262331
alloc_tag_set_inaccurate(current->alloc_tag);
23272332
}
@@ -2338,8 +2343,8 @@ static noinline void
23382343
__alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
23392344
int objects)
23402345
{
2341-
struct slabobj_ext *obj_exts;
23422346
int i;
2347+
unsigned long obj_exts;
23432348

23442349
/* slab->obj_exts might not be NULL if it was created for MEMCG accounting. */
23452350
if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE))
@@ -2352,7 +2357,7 @@ __alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p
23522357
for (i = 0; i < objects; i++) {
23532358
unsigned int off = obj_to_index(s, slab, p[i]);
23542359

2355-
alloc_tag_sub(&obj_exts[off].ref, s->size);
2360+
alloc_tag_sub(&slab_obj_ext(slab, obj_exts, off)->ref, s->size);
23562361
}
23572362
}
23582363

@@ -2411,7 +2416,7 @@ static __fastpath_inline
24112416
void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
24122417
int objects)
24132418
{
2414-
struct slabobj_ext *obj_exts;
2419+
unsigned long obj_exts;
24152420

24162421
if (!memcg_kmem_online())
24172422
return;
@@ -2426,7 +2431,8 @@ void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
24262431
static __fastpath_inline
24272432
bool memcg_slab_post_charge(void *p, gfp_t flags)
24282433
{
2429-
struct slabobj_ext *slab_exts;
2434+
unsigned long obj_exts;
2435+
struct slabobj_ext *obj_ext;
24302436
struct kmem_cache *s;
24312437
struct page *page;
24322438
struct slab *slab;
@@ -2467,10 +2473,11 @@ bool memcg_slab_post_charge(void *p, gfp_t flags)
24672473
return true;
24682474

24692475
/* Ignore already charged objects. */
2470-
slab_exts = slab_obj_exts(slab);
2471-
if (slab_exts) {
2476+
obj_exts = slab_obj_exts(slab);
2477+
if (obj_exts) {
24722478
off = obj_to_index(s, slab, p);
2473-
if (unlikely(slab_exts[off].objcg))
2479+
obj_ext = slab_obj_ext(slab, obj_exts, off);
2480+
if (unlikely(obj_ext->objcg))
24742481
return true;
24752482
}
24762483

0 commit comments

Comments
 (0)