Skip to content

Commit 76d4acf

Browse files
torvaldstorvalds
committed
Merge tag 'perf-kprobes-2020-12-14' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull perf/kprobes updates from Thomas Gleixner: "Make kretprobes lockless to avoid the rp->lock performance and potential lock ordering issues" * tag 'perf-kprobes-2020-12-14' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: locking/atomics: Regenerate the atomics-check SHA1's kprobes: Replace rp->free_instance with freelist freelist: Implement lockless freelist asm-generic/atomic: Add try_cmpxchg() fallbacks kprobes: Remove kretprobe hash llist: Add nonatomic __llist_add() and __llist_dell_all()
2 parents 8a8ca83 + a70a04b commit 76d4acf

16 files changed

Lines changed: 646 additions & 350 deletions

File tree

‎arch/x86/include/asm/atomic.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,7 @@ static __always_inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new)
199199

200200
static __always_inline bool arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
201201
{
202-
return try_cmpxchg(&v->counter, old, new);
202+
return arch_try_cmpxchg(&v->counter, old, new);
203203
}
204204
#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
205205

‎arch/x86/include/asm/atomic64_64.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ static inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
187187

188188
static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
189189
{
190-
return try_cmpxchg(&v->counter, old, new);
190+
return arch_try_cmpxchg(&v->counter, old, new);
191191
}
192192
#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
193193

‎arch/x86/include/asm/cmpxchg.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -221,7 +221,7 @@ extern void __add_wrong_size(void)
221221
#define __try_cmpxchg(ptr, pold, new, size) \
222222
__raw_try_cmpxchg((ptr), (pold), (new), (size), LOCK_PREFIX)
223223

224-
#define try_cmpxchg(ptr, pold, new) \
224+
#define arch_try_cmpxchg(ptr, pold, new) \
225225
__try_cmpxchg((ptr), (pold), (new), sizeof(*(ptr)))
226226

227227
/*

‎include/asm-generic/atomic-instrumented.h‎

Lines changed: 130 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -1642,148 +1642,192 @@ atomic64_dec_if_positive(atomic64_t *v)
16421642
#endif
16431643

16441644
#if !defined(arch_xchg_relaxed) || defined(arch_xchg)
1645-
#define xchg(ptr, ...) \
1646-
({ \
1647-
typeof(ptr) __ai_ptr = (ptr); \
1648-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1649-
arch_xchg(__ai_ptr, __VA_ARGS__); \
1645+
#define xchg(ptr, ...) \
1646+
({ \
1647+
typeof(ptr) __ai_ptr = (ptr); \
1648+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1649+
arch_xchg(__ai_ptr, __VA_ARGS__); \
16501650
})
16511651
#endif
16521652

16531653
#if defined(arch_xchg_acquire)
1654-
#define xchg_acquire(ptr, ...) \
1655-
({ \
1656-
typeof(ptr) __ai_ptr = (ptr); \
1657-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1658-
arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
1654+
#define xchg_acquire(ptr, ...) \
1655+
({ \
1656+
typeof(ptr) __ai_ptr = (ptr); \
1657+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1658+
arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
16591659
})
16601660
#endif
16611661

16621662
#if defined(arch_xchg_release)
1663-
#define xchg_release(ptr, ...) \
1664-
({ \
1665-
typeof(ptr) __ai_ptr = (ptr); \
1666-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1667-
arch_xchg_release(__ai_ptr, __VA_ARGS__); \
1663+
#define xchg_release(ptr, ...) \
1664+
({ \
1665+
typeof(ptr) __ai_ptr = (ptr); \
1666+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1667+
arch_xchg_release(__ai_ptr, __VA_ARGS__); \
16681668
})
16691669
#endif
16701670

16711671
#if defined(arch_xchg_relaxed)
1672-
#define xchg_relaxed(ptr, ...) \
1673-
({ \
1674-
typeof(ptr) __ai_ptr = (ptr); \
1675-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1676-
arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
1672+
#define xchg_relaxed(ptr, ...) \
1673+
({ \
1674+
typeof(ptr) __ai_ptr = (ptr); \
1675+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1676+
arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
16771677
})
16781678
#endif
16791679

16801680
#if !defined(arch_cmpxchg_relaxed) || defined(arch_cmpxchg)
1681-
#define cmpxchg(ptr, ...) \
1682-
({ \
1683-
typeof(ptr) __ai_ptr = (ptr); \
1684-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1685-
arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
1681+
#define cmpxchg(ptr, ...) \
1682+
({ \
1683+
typeof(ptr) __ai_ptr = (ptr); \
1684+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1685+
arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
16861686
})
16871687
#endif
16881688

16891689
#if defined(arch_cmpxchg_acquire)
1690-
#define cmpxchg_acquire(ptr, ...) \
1691-
({ \
1692-
typeof(ptr) __ai_ptr = (ptr); \
1693-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1694-
arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
1690+
#define cmpxchg_acquire(ptr, ...) \
1691+
({ \
1692+
typeof(ptr) __ai_ptr = (ptr); \
1693+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1694+
arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
16951695
})
16961696
#endif
16971697

16981698
#if defined(arch_cmpxchg_release)
1699-
#define cmpxchg_release(ptr, ...) \
1700-
({ \
1701-
typeof(ptr) __ai_ptr = (ptr); \
1702-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1703-
arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
1699+
#define cmpxchg_release(ptr, ...) \
1700+
({ \
1701+
typeof(ptr) __ai_ptr = (ptr); \
1702+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1703+
arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
17041704
})
17051705
#endif
17061706

17071707
#if defined(arch_cmpxchg_relaxed)
1708-
#define cmpxchg_relaxed(ptr, ...) \
1709-
({ \
1710-
typeof(ptr) __ai_ptr = (ptr); \
1711-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1712-
arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
1708+
#define cmpxchg_relaxed(ptr, ...) \
1709+
({ \
1710+
typeof(ptr) __ai_ptr = (ptr); \
1711+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1712+
arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
17131713
})
17141714
#endif
17151715

17161716
#if !defined(arch_cmpxchg64_relaxed) || defined(arch_cmpxchg64)
1717-
#define cmpxchg64(ptr, ...) \
1718-
({ \
1719-
typeof(ptr) __ai_ptr = (ptr); \
1720-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1721-
arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
1717+
#define cmpxchg64(ptr, ...) \
1718+
({ \
1719+
typeof(ptr) __ai_ptr = (ptr); \
1720+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1721+
arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
17221722
})
17231723
#endif
17241724

17251725
#if defined(arch_cmpxchg64_acquire)
1726-
#define cmpxchg64_acquire(ptr, ...) \
1727-
({ \
1728-
typeof(ptr) __ai_ptr = (ptr); \
1729-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1730-
arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
1726+
#define cmpxchg64_acquire(ptr, ...) \
1727+
({ \
1728+
typeof(ptr) __ai_ptr = (ptr); \
1729+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1730+
arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
17311731
})
17321732
#endif
17331733

17341734
#if defined(arch_cmpxchg64_release)
1735-
#define cmpxchg64_release(ptr, ...) \
1736-
({ \
1737-
typeof(ptr) __ai_ptr = (ptr); \
1738-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1739-
arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
1735+
#define cmpxchg64_release(ptr, ...) \
1736+
({ \
1737+
typeof(ptr) __ai_ptr = (ptr); \
1738+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1739+
arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
17401740
})
17411741
#endif
17421742

17431743
#if defined(arch_cmpxchg64_relaxed)
1744-
#define cmpxchg64_relaxed(ptr, ...) \
1745-
({ \
1746-
typeof(ptr) __ai_ptr = (ptr); \
1747-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1748-
arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
1744+
#define cmpxchg64_relaxed(ptr, ...) \
1745+
({ \
1746+
typeof(ptr) __ai_ptr = (ptr); \
1747+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1748+
arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
17491749
})
17501750
#endif
17511751

1752-
#define cmpxchg_local(ptr, ...) \
1753-
({ \
1754-
typeof(ptr) __ai_ptr = (ptr); \
1755-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1756-
arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
1752+
#if !defined(arch_try_cmpxchg_relaxed) || defined(arch_try_cmpxchg)
1753+
#define try_cmpxchg(ptr, oldp, ...) \
1754+
({ \
1755+
typeof(ptr) __ai_ptr = (ptr); \
1756+
typeof(oldp) __ai_oldp = (oldp); \
1757+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1758+
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1759+
arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1760+
})
1761+
#endif
1762+
1763+
#if defined(arch_try_cmpxchg_acquire)
1764+
#define try_cmpxchg_acquire(ptr, oldp, ...) \
1765+
({ \
1766+
typeof(ptr) __ai_ptr = (ptr); \
1767+
typeof(oldp) __ai_oldp = (oldp); \
1768+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1769+
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1770+
arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1771+
})
1772+
#endif
1773+
1774+
#if defined(arch_try_cmpxchg_release)
1775+
#define try_cmpxchg_release(ptr, oldp, ...) \
1776+
({ \
1777+
typeof(ptr) __ai_ptr = (ptr); \
1778+
typeof(oldp) __ai_oldp = (oldp); \
1779+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1780+
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1781+
arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1782+
})
1783+
#endif
1784+
1785+
#if defined(arch_try_cmpxchg_relaxed)
1786+
#define try_cmpxchg_relaxed(ptr, oldp, ...) \
1787+
({ \
1788+
typeof(ptr) __ai_ptr = (ptr); \
1789+
typeof(oldp) __ai_oldp = (oldp); \
1790+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1791+
instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \
1792+
arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \
1793+
})
1794+
#endif
1795+
1796+
#define cmpxchg_local(ptr, ...) \
1797+
({ \
1798+
typeof(ptr) __ai_ptr = (ptr); \
1799+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1800+
arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
17571801
})
17581802

1759-
#define cmpxchg64_local(ptr, ...) \
1760-
({ \
1761-
typeof(ptr) __ai_ptr = (ptr); \
1762-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1763-
arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
1803+
#define cmpxchg64_local(ptr, ...) \
1804+
({ \
1805+
typeof(ptr) __ai_ptr = (ptr); \
1806+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1807+
arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
17641808
})
17651809

1766-
#define sync_cmpxchg(ptr, ...) \
1767-
({ \
1768-
typeof(ptr) __ai_ptr = (ptr); \
1769-
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1770-
arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
1810+
#define sync_cmpxchg(ptr, ...) \
1811+
({ \
1812+
typeof(ptr) __ai_ptr = (ptr); \
1813+
instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \
1814+
arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
17711815
})
17721816

1773-
#define cmpxchg_double(ptr, ...) \
1774-
({ \
1775-
typeof(ptr) __ai_ptr = (ptr); \
1776-
instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
1777-
arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
1817+
#define cmpxchg_double(ptr, ...) \
1818+
({ \
1819+
typeof(ptr) __ai_ptr = (ptr); \
1820+
instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
1821+
arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
17781822
})
17791823

17801824

1781-
#define cmpxchg_double_local(ptr, ...) \
1782-
({ \
1783-
typeof(ptr) __ai_ptr = (ptr); \
1784-
instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
1785-
arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
1825+
#define cmpxchg_double_local(ptr, ...) \
1826+
({ \
1827+
typeof(ptr) __ai_ptr = (ptr); \
1828+
instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
1829+
arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
17861830
})
17871831

17881832
#endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */
1789-
// 9d5e6a315fb1335d02f0ccd3655a91c3dafcc63e
1833+
// 4bec382e44520f4d8267e42620054db26a659ea3

0 commit comments

Comments
 (0)