Skip to content

Commit bae0ba7

Browse files
ljskernelakpm00
ljskernel
authored and
akpm00
committed
mm: add basic VMA flag operation helper functions
Now we have the mk_vma_flags() macro helper which permits easy specification of any number of VMA flags, add helper functions which operate with vma_flags_t parameters. This patch provides vma_flags_test[_mask](), vma_flags_set[_mask]() and vma_flags_clear[_mask]() respectively testing, setting and clearing flags with the _mask variants accepting vma_flag_t parameters, and the non-mask variants implemented as macros which accept a list of flags. This allows us to trivially test/set/clear aggregate VMA flag values as necessary, for instance: if (vma_flags_test(&flags, VMA_READ_BIT, VMA_WRITE_BIT)) goto readwrite; vma_flags_set(&flags, VMA_READ_BIT, VMA_WRITE_BIT); vma_flags_clear(&flags, VMA_READ_BIT, VMA_WRITE_BIT); We also add a function for testing that ALL flags are set for convenience, e.g.: if (vma_flags_test_all(&flags, VMA_READ_BIT, VMA_MAYREAD_BIT)) { /* Both READ and MAYREAD flags set */ ... } The compiler generates optimal assembly for each such that they behave as if the caller were setting the bitmap flags manually. This is important for e.g. drivers which manipulate flag values rather than a VMA's specific flag values. We also add helpers for testing, setting and clearing flags for VMA's and VMA descriptors to reduce boilerplate. Also add the EMPTY_VMA_FLAGS define to aid initialisation of empty flags. Finally, update the userland VMA tests to add the helpers there so they can be utilised as part of userland testing. Link: https://lkml.kernel.org/r/885d4897d67a6a57c0b07fa182a7055ad752df11.1769097829.git.lorenzo.stoakes@oracle.com Signed-off-by: Lorenzo Stoakes <lorenzo.stoakes@oracle.com> Reviewed-by: Pedro Falcato <pfalcato@suse.de> Reviewed-by: Liam R. Howlett <Liam.Howlett@oracle.com> Cc: Baolin Wang <baolin.wang@linux.alibaba.com> Cc: Barry Song <baohua@kernel.org> Cc: David Hildenbrand <david@kernel.org> Cc: Dev Jain <dev.jain@arm.com> Cc: Jason Gunthorpe <jgg@nvidia.com> Cc: Suren Baghdasaryan <surenb@google.com> Cc: Vlastimil Babka <vbabka@suse.cz> Cc: Zi Yan <ziy@nvidia.com> Cc: Damien Le Moal <dlemoal@kernel.org> Cc: "Darrick J. Wong" <djwong@kernel.org> Cc: Jarkko Sakkinen <jarkko@kernel.org> Cc: Yury Norov <ynorov@nvidia.com> Cc: Chris Mason <clm@fb.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
1 parent 21c8a5b commit bae0ba7

3 files changed

Lines changed: 295 additions & 21 deletions

File tree

‎include/linux/mm.h‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1059,6 +1059,171 @@ static inline vma_flags_t __mk_vma_flags(size_t count, const vma_flag_t *bits)
10591059
#define mk_vma_flags(...) __mk_vma_flags(COUNT_ARGS(__VA_ARGS__), \
10601060
(const vma_flag_t []){__VA_ARGS__})
10611061

1062+
/* Test each of to_test flags in flags, non-atomically. */
1063+
static __always_inline bool vma_flags_test_mask(const vma_flags_t *flags,
1064+
vma_flags_t to_test)
1065+
{
1066+
const unsigned long *bitmap = flags->__vma_flags;
1067+
const unsigned long *bitmap_to_test = to_test.__vma_flags;
1068+
1069+
return bitmap_intersects(bitmap_to_test, bitmap, NUM_VMA_FLAG_BITS);
1070+
}
1071+
1072+
/*
1073+
* Test whether any specified VMA flag is set, e.g.:
1074+
*
1075+
* if (vma_flags_test(flags, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... }
1076+
*/
1077+
#define vma_flags_test(flags, ...) \
1078+
vma_flags_test_mask(flags, mk_vma_flags(__VA_ARGS__))
1079+
1080+
/* Test that ALL of the to_test flags are set, non-atomically. */
1081+
static __always_inline bool vma_flags_test_all_mask(const vma_flags_t *flags,
1082+
vma_flags_t to_test)
1083+
{
1084+
const unsigned long *bitmap = flags->__vma_flags;
1085+
const unsigned long *bitmap_to_test = to_test.__vma_flags;
1086+
1087+
return bitmap_subset(bitmap_to_test, bitmap, NUM_VMA_FLAG_BITS);
1088+
}
1089+
1090+
/*
1091+
* Test whether ALL specified VMA flags are set, e.g.:
1092+
*
1093+
* if (vma_flags_test_all(flags, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... }
1094+
*/
1095+
#define vma_flags_test_all(flags, ...) \
1096+
vma_flags_test_all_mask(flags, mk_vma_flags(__VA_ARGS__))
1097+
1098+
/* Set each of the to_set flags in flags, non-atomically. */
1099+
static __always_inline void vma_flags_set_mask(vma_flags_t *flags, vma_flags_t to_set)
1100+
{
1101+
unsigned long *bitmap = flags->__vma_flags;
1102+
const unsigned long *bitmap_to_set = to_set.__vma_flags;
1103+
1104+
bitmap_or(bitmap, bitmap, bitmap_to_set, NUM_VMA_FLAG_BITS);
1105+
}
1106+
1107+
/*
1108+
* Set all specified VMA flags, e.g.:
1109+
*
1110+
* vma_flags_set(&flags, VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT);
1111+
*/
1112+
#define vma_flags_set(flags, ...) \
1113+
vma_flags_set_mask(flags, mk_vma_flags(__VA_ARGS__))
1114+
1115+
/* Clear all of the to-clear flags in flags, non-atomically. */
1116+
static __always_inline void vma_flags_clear_mask(vma_flags_t *flags, vma_flags_t to_clear)
1117+
{
1118+
unsigned long *bitmap = flags->__vma_flags;
1119+
const unsigned long *bitmap_to_clear = to_clear.__vma_flags;
1120+
1121+
bitmap_andnot(bitmap, bitmap, bitmap_to_clear, NUM_VMA_FLAG_BITS);
1122+
}
1123+
1124+
/*
1125+
* Clear all specified individual flags, e.g.:
1126+
*
1127+
* vma_flags_clear(&flags, VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT);
1128+
*/
1129+
#define vma_flags_clear(flags, ...) \
1130+
vma_flags_clear_mask(flags, mk_vma_flags(__VA_ARGS__))
1131+
1132+
/*
1133+
* Helper to test that ALL specified flags are set in a VMA.
1134+
*
1135+
* Note: appropriate locks must be held, this function does not acquire them for
1136+
* you.
1137+
*/
1138+
static inline bool vma_test_all_flags_mask(const struct vm_area_struct *vma,
1139+
vma_flags_t flags)
1140+
{
1141+
return vma_flags_test_all_mask(&vma->flags, flags);
1142+
}
1143+
1144+
/*
1145+
* Helper macro for checking that ALL specified flags are set in a VMA, e.g.:
1146+
*
1147+
* if (vma_test_all_flags(vma, VMA_READ_BIT, VMA_MAYREAD_BIT) { ... }
1148+
*/
1149+
#define vma_test_all_flags(vma, ...) \
1150+
vma_test_all_flags_mask(vma, mk_vma_flags(__VA_ARGS__))
1151+
1152+
/*
1153+
* Helper to set all VMA flags in a VMA.
1154+
*
1155+
* Note: appropriate locks must be held, this function does not acquire them for
1156+
* you.
1157+
*/
1158+
static inline void vma_set_flags_mask(struct vm_area_struct *vma,
1159+
vma_flags_t flags)
1160+
{
1161+
vma_flags_set_mask(&vma->flags, flags);
1162+
}
1163+
1164+
/*
1165+
* Helper macro for specifying VMA flags in a VMA, e.g.:
1166+
*
1167+
* vma_set_flags(vma, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT,
1168+
* VMA_DONTDUMP_BIT);
1169+
*
1170+
* Note: appropriate locks must be held, this function does not acquire them for
1171+
* you.
1172+
*/
1173+
#define vma_set_flags(vma, ...) \
1174+
vma_set_flags_mask(vma, mk_vma_flags(__VA_ARGS__))
1175+
1176+
/* Helper to test all VMA flags in a VMA descriptor. */
1177+
static inline bool vma_desc_test_flags_mask(const struct vm_area_desc *desc,
1178+
vma_flags_t flags)
1179+
{
1180+
return vma_flags_test_mask(&desc->vma_flags, flags);
1181+
}
1182+
1183+
/*
1184+
* Helper macro for testing VMA flags for an input pointer to a struct
1185+
* vm_area_desc object describing a proposed VMA, e.g.:
1186+
*
1187+
* if (vma_desc_test_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT,
1188+
* VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT)) { ... }
1189+
*/
1190+
#define vma_desc_test_flags(desc, ...) \
1191+
vma_desc_test_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
1192+
1193+
/* Helper to set all VMA flags in a VMA descriptor. */
1194+
static inline void vma_desc_set_flags_mask(struct vm_area_desc *desc,
1195+
vma_flags_t flags)
1196+
{
1197+
vma_flags_set_mask(&desc->vma_flags, flags);
1198+
}
1199+
1200+
/*
1201+
* Helper macro for specifying VMA flags for an input pointer to a struct
1202+
* vm_area_desc object describing a proposed VMA, e.g.:
1203+
*
1204+
* vma_desc_set_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT,
1205+
* VMA_DONTDUMP_BIT);
1206+
*/
1207+
#define vma_desc_set_flags(desc, ...) \
1208+
vma_desc_set_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
1209+
1210+
/* Helper to clear all VMA flags in a VMA descriptor. */
1211+
static inline void vma_desc_clear_flags_mask(struct vm_area_desc *desc,
1212+
vma_flags_t flags)
1213+
{
1214+
vma_flags_clear_mask(&desc->vma_flags, flags);
1215+
}
1216+
1217+
/*
1218+
* Helper macro for clearing VMA flags for an input pointer to a struct
1219+
* vm_area_desc object describing a proposed VMA, e.g.:
1220+
*
1221+
* vma_desc_clear_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT,
1222+
* VMA_DONTDUMP_BIT);
1223+
*/
1224+
#define vma_desc_clear_flags(desc, ...) \
1225+
vma_desc_clear_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
1226+
10621227
static inline void vma_set_anonymous(struct vm_area_struct *vma)
10631228
{
10641229
vma->vm_ops = NULL;

‎include/linux/mm_types.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -844,7 +844,7 @@ struct mmap_action {
844844

845845
/*
846846
* If specified, this hook is invoked when an error occurred when
847-
* attempting the selection action.
847+
* attempting the selected action.
848848
*
849849
* The hook can return an error code in order to filter the error, but
850850
* it is not valid to clear the error here.
@@ -868,6 +868,8 @@ typedef struct {
868868
DECLARE_BITMAP(__vma_flags, NUM_VMA_FLAG_BITS);
869869
} vma_flags_t;
870870

871+
#define EMPTY_VMA_FLAGS ((vma_flags_t){ })
872+
871873
/*
872874
* Describes a VMA that is about to be mmap()'ed. Drivers may choose to
873875
* manipulate mutable fields which will cause those fields to be updated in the

‎tools/testing/vma/vma_internal.h‎

Lines changed: 127 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,13 @@
2121

2222
#include <stdlib.h>
2323

24+
#ifdef __CONCAT
25+
#undef __CONCAT
26+
#endif
27+
28+
#include <linux/args.h>
2429
#include <linux/atomic.h>
30+
#include <linux/bitmap.h>
2531
#include <linux/list.h>
2632
#include <linux/maple_tree.h>
2733
#include <linux/mm.h>
@@ -38,6 +44,8 @@ extern unsigned long dac_mmap_min_addr;
3844
#define dac_mmap_min_addr 0UL
3945
#endif
4046

47+
#define ACCESS_PRIVATE(p, member) ((p)->member)
48+
4149
#define VM_WARN_ON(_expr) (WARN_ON(_expr))
4250
#define VM_WARN_ON_ONCE(_expr) (WARN_ON_ONCE(_expr))
4351
#define VM_WARN_ON_VMG(_expr, _vmg) (WARN_ON(_expr))
@@ -533,6 +541,8 @@ typedef struct {
533541
DECLARE_BITMAP(__vma_flags, NUM_VMA_FLAG_BITS);
534542
} __private vma_flags_t;
535543

544+
#define EMPTY_VMA_FLAGS ((vma_flags_t){ })
545+
536546
struct mm_struct {
537547
struct maple_tree mm_mt;
538548
int map_count; /* number of VMAs */
@@ -882,6 +892,123 @@ static inline pgprot_t vm_get_page_prot(vm_flags_t vm_flags)
882892
return __pgprot(vm_flags);
883893
}
884894

895+
static inline void vma_flags_clear_all(vma_flags_t *flags)
896+
{
897+
bitmap_zero(flags->__vma_flags, NUM_VMA_FLAG_BITS);
898+
}
899+
900+
static inline void vma_flag_set(vma_flags_t *flags, vma_flag_t bit)
901+
{
902+
unsigned long *bitmap = flags->__vma_flags;
903+
904+
__set_bit((__force int)bit, bitmap);
905+
}
906+
907+
static inline vma_flags_t __mk_vma_flags(size_t count, const vma_flag_t *bits)
908+
{
909+
vma_flags_t flags;
910+
int i;
911+
912+
vma_flags_clear_all(&flags);
913+
for (i = 0; i < count; i++)
914+
vma_flag_set(&flags, bits[i]);
915+
return flags;
916+
}
917+
918+
#define mk_vma_flags(...) __mk_vma_flags(COUNT_ARGS(__VA_ARGS__), \
919+
(const vma_flag_t []){__VA_ARGS__})
920+
921+
static __always_inline bool vma_flags_test_mask(const vma_flags_t *flags,
922+
vma_flags_t to_test)
923+
{
924+
const unsigned long *bitmap = flags->__vma_flags;
925+
const unsigned long *bitmap_to_test = to_test.__vma_flags;
926+
927+
return bitmap_intersects(bitmap_to_test, bitmap, NUM_VMA_FLAG_BITS);
928+
}
929+
930+
#define vma_flags_test(flags, ...) \
931+
vma_flags_test_mask(flags, mk_vma_flags(__VA_ARGS__))
932+
933+
static __always_inline bool vma_flags_test_all_mask(const vma_flags_t *flags,
934+
vma_flags_t to_test)
935+
{
936+
const unsigned long *bitmap = flags->__vma_flags;
937+
const unsigned long *bitmap_to_test = to_test.__vma_flags;
938+
939+
return bitmap_subset(bitmap_to_test, bitmap, NUM_VMA_FLAG_BITS);
940+
}
941+
942+
#define vma_flags_test_all(flags, ...) \
943+
vma_flags_test_all_mask(flags, mk_vma_flags(__VA_ARGS__))
944+
945+
static __always_inline void vma_flags_set_mask(vma_flags_t *flags, vma_flags_t to_set)
946+
{
947+
unsigned long *bitmap = flags->__vma_flags;
948+
const unsigned long *bitmap_to_set = to_set.__vma_flags;
949+
950+
bitmap_or(bitmap, bitmap, bitmap_to_set, NUM_VMA_FLAG_BITS);
951+
}
952+
953+
#define vma_flags_set(flags, ...) \
954+
vma_flags_set_mask(flags, mk_vma_flags(__VA_ARGS__))
955+
956+
static __always_inline void vma_flags_clear_mask(vma_flags_t *flags, vma_flags_t to_clear)
957+
{
958+
unsigned long *bitmap = flags->__vma_flags;
959+
const unsigned long *bitmap_to_clear = to_clear.__vma_flags;
960+
961+
bitmap_andnot(bitmap, bitmap, bitmap_to_clear, NUM_VMA_FLAG_BITS);
962+
}
963+
964+
#define vma_flags_clear(flags, ...) \
965+
vma_flags_clear_mask(flags, mk_vma_flags(__VA_ARGS__))
966+
967+
static inline bool vma_test_all_flags_mask(const struct vm_area_struct *vma,
968+
vma_flags_t flags)
969+
{
970+
return vma_flags_test_all_mask(&vma->flags, flags);
971+
}
972+
973+
#define vma_test_all_flags(vma, ...) \
974+
vma_test_all_flags_mask(vma, mk_vma_flags(__VA_ARGS__))
975+
976+
static inline void vma_set_flags_mask(struct vm_area_struct *vma,
977+
vma_flags_t flags)
978+
{
979+
vma_flags_set_mask(&vma->flags, flags);
980+
}
981+
982+
#define vma_set_flags(vma, ...) \
983+
vma_set_flags_mask(vma, mk_vma_flags(__VA_ARGS__))
984+
985+
static inline bool vma_desc_test_flags_mask(const struct vm_area_desc *desc,
986+
vma_flags_t flags)
987+
{
988+
return vma_flags_test_mask(&desc->vma_flags, flags);
989+
}
990+
991+
#define vma_desc_test_flags(desc, ...) \
992+
vma_desc_test_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
993+
994+
static inline void vma_desc_set_flags_mask(struct vm_area_desc *desc,
995+
vma_flags_t flags)
996+
{
997+
vma_flags_set_mask(&desc->vma_flags, flags);
998+
}
999+
1000+
#define vma_desc_set_flags(desc, ...) \
1001+
vma_desc_set_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
1002+
1003+
static inline void vma_desc_clear_flags_mask(struct vm_area_desc *desc,
1004+
vma_flags_t flags)
1005+
{
1006+
vma_flags_clear_mask(&desc->vma_flags, flags);
1007+
}
1008+
1009+
#define vma_desc_clear_flags(desc, ...) \
1010+
vma_desc_clear_flags_mask(desc, mk_vma_flags(__VA_ARGS__))
1011+
8851012
static inline bool is_shared_maywrite(vm_flags_t vm_flags)
8861013
{
8871014
return (vm_flags & (VM_SHARED | VM_MAYWRITE)) ==
@@ -1540,31 +1667,11 @@ static inline void userfaultfd_unmap_complete(struct mm_struct *mm,
15401667
{
15411668
}
15421669

1543-
#define ACCESS_PRIVATE(p, member) ((p)->member)
1544-
1545-
#define bitmap_size(nbits) (ALIGN(nbits, BITS_PER_LONG) / BITS_PER_BYTE)
1546-
1547-
static __always_inline void bitmap_zero(unsigned long *dst, unsigned int nbits)
1548-
{
1549-
unsigned int len = bitmap_size(nbits);
1550-
1551-
if (small_const_nbits(nbits))
1552-
*dst = 0;
1553-
else
1554-
memset(dst, 0, len);
1555-
}
1556-
15571670
static inline bool mm_flags_test(int flag, const struct mm_struct *mm)
15581671
{
15591672
return test_bit(flag, ACCESS_PRIVATE(&mm->flags, __mm_flags));
15601673
}
15611674

1562-
/* Clears all bits in the VMA flags bitmap, non-atomically. */
1563-
static inline void vma_flags_clear_all(vma_flags_t *flags)
1564-
{
1565-
bitmap_zero(ACCESS_PRIVATE(flags, __vma_flags), NUM_VMA_FLAG_BITS);
1566-
}
1567-
15681675
/*
15691676
* Copy value to the first system word of VMA flags, non-atomically.
15701677
*

0 commit comments

Comments
 (0)