Skip to content

Commit 1c34496

Browse files
Peter Zijlstraingomolnar
Peter Zijlstra
authored and
ingomolnar
committed
objtool: Remove instruction::list
Replace the instruction::list by allocating instructions in arrays of 256 entries and stringing them together by (amortized) find_insn(). This shrinks instruction by 16 bytes and brings it down to 128. struct instruction { - struct list_head list; /* 0 16 */ - struct hlist_node hash; /* 16 16 */ - struct list_head call_node; /* 32 16 */ - struct section * sec; /* 48 8 */ - long unsigned int offset; /* 56 8 */ - /* --- cacheline 1 boundary (64 bytes) --- */ - long unsigned int immediate; /* 64 8 */ - unsigned int len; /* 72 4 */ - u8 type; /* 76 1 */ - - /* Bitfield combined with previous fields */ + struct hlist_node hash; /* 0 16 */ + struct list_head call_node; /* 16 16 */ + struct section * sec; /* 32 8 */ + long unsigned int offset; /* 40 8 */ + long unsigned int immediate; /* 48 8 */ + u8 len; /* 56 1 */ + u8 prev_len; /* 57 1 */ + u8 type; /* 58 1 */ + s8 instr; /* 59 1 */ + u32 idx:8; /* 60: 0 4 */ + u32 dead_end:1; /* 60: 8 4 */ + u32 ignore:1; /* 60: 9 4 */ + u32 ignore_alts:1; /* 60:10 4 */ + u32 hint:1; /* 60:11 4 */ + u32 save:1; /* 60:12 4 */ + u32 restore:1; /* 60:13 4 */ + u32 retpoline_safe:1; /* 60:14 4 */ + u32 noendbr:1; /* 60:15 4 */ + u32 entry:1; /* 60:16 4 */ + u32 visited:4; /* 60:17 4 */ + u32 no_reloc:1; /* 60:21 4 */ - u16 dead_end:1; /* 76: 8 2 */ - u16 ignore:1; /* 76: 9 2 */ - u16 ignore_alts:1; /* 76:10 2 */ - u16 hint:1; /* 76:11 2 */ - u16 save:1; /* 76:12 2 */ - u16 restore:1; /* 76:13 2 */ - u16 retpoline_safe:1; /* 76:14 2 */ - u16 noendbr:1; /* 76:15 2 */ - u16 entry:1; /* 78: 0 2 */ - u16 visited:4; /* 78: 1 2 */ - u16 no_reloc:1; /* 78: 5 2 */ + /* XXX 10 bits hole, try to pack */ - /* XXX 2 bits hole, try to pack */ - /* Bitfield combined with next fields */ - - s8 instr; /* 79 1 */ - struct alt_group * alt_group; /* 80 8 */ - struct instruction * jump_dest; /* 88 8 */ - struct instruction * first_jump_src; /* 96 8 */ + /* --- cacheline 1 boundary (64 bytes) --- */ + struct alt_group * alt_group; /* 64 8 */ + struct instruction * jump_dest; /* 72 8 */ + struct instruction * first_jump_src; /* 80 8 */ union { - struct symbol * _call_dest; /* 104 8 */ - struct reloc * _jump_table; /* 104 8 */ - }; /* 104 8 */ - struct alternative * alts; /* 112 8 */ - struct symbol * sym; /* 120 8 */ - /* --- cacheline 2 boundary (128 bytes) --- */ - struct stack_op * stack_ops; /* 128 8 */ - struct cfi_state * cfi; /* 136 8 */ + struct symbol * _call_dest; /* 88 8 */ + struct reloc * _jump_table; /* 88 8 */ + }; /* 88 8 */ + struct alternative * alts; /* 96 8 */ + struct symbol * sym; /* 104 8 */ + struct stack_op * stack_ops; /* 112 8 */ + struct cfi_state * cfi; /* 120 8 */ - /* size: 144, cachelines: 3, members: 28 */ - /* sum members: 142 */ - /* sum bitfield members: 14 bits, bit holes: 1, sum bit holes: 2 bits */ - /* last cacheline: 16 bytes */ + /* size: 128, cachelines: 2, members: 29 */ + /* sum members: 124 */ + /* sum bitfield members: 22 bits, bit holes: 1, sum bit holes: 10 bits */ }; pre: 5:38.18 real, 213.25 user, 124.90 sys, 23449040 mem post: 5:03.34 real, 210.75 user, 88.80 sys, 20241232 mem Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Ingo Molnar <mingo@kernel.org> Acked-by: Josh Poimboeuf <jpoimboe@kernel.org> Tested-by: Nathan Chancellor <nathan@kernel.org> # build only Tested-by: Thomas Weißschuh <linux@weissschuh.net> # compile and run Link: https://lore.kernel.org/r/20230208172245.851307606@infradead.org
1 parent 6ea17e8 commit 1c34496

4 files changed

Lines changed: 133 additions & 86 deletions

File tree

‎tools/objtool/check.c‎

Lines changed: 106 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -47,27 +47,29 @@ struct instruction *find_insn(struct objtool_file *file,
4747
return NULL;
4848
}
4949

50-
static struct instruction *next_insn_same_sec(struct objtool_file *file,
51-
struct instruction *insn)
50+
struct instruction *next_insn_same_sec(struct objtool_file *file,
51+
struct instruction *insn)
5252
{
53-
struct instruction *next = list_next_entry(insn, list);
53+
if (insn->idx == INSN_CHUNK_MAX)
54+
return find_insn(file, insn->sec, insn->offset + insn->len);
5455

55-
if (!next || &next->list == &file->insn_list || next->sec != insn->sec)
56+
insn++;
57+
if (!insn->len)
5658
return NULL;
5759

58-
return next;
60+
return insn;
5961
}
6062

6163
static struct instruction *next_insn_same_func(struct objtool_file *file,
6264
struct instruction *insn)
6365
{
64-
struct instruction *next = list_next_entry(insn, list);
66+
struct instruction *next = next_insn_same_sec(file, insn);
6567
struct symbol *func = insn_func(insn);
6668

6769
if (!func)
6870
return NULL;
6971

70-
if (&next->list != &file->insn_list && insn_func(next) == func)
72+
if (next && insn_func(next) == func)
7173
return next;
7274

7375
/* Check if we're already in the subfunction: */
@@ -78,34 +80,49 @@ static struct instruction *next_insn_same_func(struct objtool_file *file,
7880
return find_insn(file, func->cfunc->sec, func->cfunc->offset);
7981
}
8082

83+
static struct instruction *prev_insn_same_sec(struct objtool_file *file,
84+
struct instruction *insn)
85+
{
86+
if (insn->idx == 0) {
87+
if (insn->prev_len)
88+
return find_insn(file, insn->sec, insn->offset - insn->prev_len);
89+
return NULL;
90+
}
91+
92+
return insn - 1;
93+
}
94+
8195
static struct instruction *prev_insn_same_sym(struct objtool_file *file,
82-
struct instruction *insn)
96+
struct instruction *insn)
8397
{
84-
struct instruction *prev = list_prev_entry(insn, list);
98+
struct instruction *prev = prev_insn_same_sec(file, insn);
8599

86-
if (&prev->list != &file->insn_list && insn_func(prev) == insn_func(insn))
100+
if (prev && insn_func(prev) == insn_func(insn))
87101
return prev;
88102

89103
return NULL;
90104
}
91105

106+
#define for_each_insn(file, insn) \
107+
for (struct section *__sec, *__fake = (struct section *)1; \
108+
__fake; __fake = NULL) \
109+
for_each_sec(file, __sec) \
110+
sec_for_each_insn(file, __sec, insn)
111+
92112
#define func_for_each_insn(file, func, insn) \
93113
for (insn = find_insn(file, func->sec, func->offset); \
94114
insn; \
95115
insn = next_insn_same_func(file, insn))
96116

97117
#define sym_for_each_insn(file, sym, insn) \
98118
for (insn = find_insn(file, sym->sec, sym->offset); \
99-
insn && &insn->list != &file->insn_list && \
100-
insn->sec == sym->sec && \
101-
insn->offset < sym->offset + sym->len; \
102-
insn = list_next_entry(insn, list))
119+
insn && insn->offset < sym->offset + sym->len; \
120+
insn = next_insn_same_sec(file, insn))
103121

104122
#define sym_for_each_insn_continue_reverse(file, sym, insn) \
105-
for (insn = list_prev_entry(insn, list); \
106-
&insn->list != &file->insn_list && \
107-
insn->sec == sym->sec && insn->offset >= sym->offset; \
108-
insn = list_prev_entry(insn, list))
123+
for (insn = prev_insn_same_sec(file, insn); \
124+
insn && insn->offset >= sym->offset; \
125+
insn = prev_insn_same_sec(file, insn))
109126

110127
#define sec_for_each_insn_from(file, insn) \
111128
for (; insn; insn = next_insn_same_sec(file, insn))
@@ -384,6 +401,9 @@ static int decode_instructions(struct objtool_file *file)
384401
int ret;
385402

386403
for_each_sec(file, sec) {
404+
struct instruction *insns = NULL;
405+
u8 prev_len = 0;
406+
u8 idx = 0;
387407

388408
if (!(sec->sh.sh_flags & SHF_EXECINSTR))
389409
continue;
@@ -409,22 +429,31 @@ static int decode_instructions(struct objtool_file *file)
409429
sec->init = true;
410430

411431
for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) {
412-
insn = malloc(sizeof(*insn));
413-
if (!insn) {
414-
WARN("malloc failed");
415-
return -1;
432+
if (!insns || idx == INSN_CHUNK_MAX) {
433+
insns = calloc(sizeof(*insn), INSN_CHUNK_SIZE);
434+
if (!insns) {
435+
WARN("malloc failed");
436+
return -1;
437+
}
438+
idx = 0;
439+
} else {
440+
idx++;
416441
}
417-
memset(insn, 0, sizeof(*insn));
418-
INIT_LIST_HEAD(&insn->call_node);
442+
insn = &insns[idx];
443+
insn->idx = idx;
419444

445+
INIT_LIST_HEAD(&insn->call_node);
420446
insn->sec = sec;
421447
insn->offset = offset;
448+
insn->prev_len = prev_len;
422449

423450
ret = arch_decode_instruction(file, sec, offset,
424451
sec->sh.sh_size - offset,
425452
insn);
426453
if (ret)
427-
goto err;
454+
return ret;
455+
456+
prev_len = insn->len;
428457

429458
/*
430459
* By default, "ud2" is a dead end unless otherwise
@@ -435,10 +464,11 @@ static int decode_instructions(struct objtool_file *file)
435464
insn->dead_end = true;
436465

437466
hash_add(file->insn_hash, &insn->hash, sec_offset_hash(sec, insn->offset));
438-
list_add_tail(&insn->list, &file->insn_list);
439467
nr_insns++;
440468
}
441469

470+
// printf("%s: last chunk used: %d\n", sec->name, (int)idx);
471+
442472
list_for_each_entry(func, &sec->symbol_list, list) {
443473
if (func->type != STT_NOTYPE && func->type != STT_FUNC)
444474
continue;
@@ -481,10 +511,6 @@ static int decode_instructions(struct objtool_file *file)
481511
printf("nr_insns: %lu\n", nr_insns);
482512

483513
return 0;
484-
485-
err:
486-
free(insn);
487-
return ret;
488514
}
489515

490516
/*
@@ -599,7 +625,7 @@ static int add_dead_ends(struct objtool_file *file)
599625
}
600626
insn = find_insn(file, reloc->sym->sec, reloc->addend);
601627
if (insn)
602-
insn = list_prev_entry(insn, list);
628+
insn = prev_insn_same_sec(file, insn);
603629
else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
604630
insn = find_last_insn(file, reloc->sym->sec);
605631
if (!insn) {
@@ -634,7 +660,7 @@ static int add_dead_ends(struct objtool_file *file)
634660
}
635661
insn = find_insn(file, reloc->sym->sec, reloc->addend);
636662
if (insn)
637-
insn = list_prev_entry(insn, list);
663+
insn = prev_insn_same_sec(file, insn);
638664
else if (reloc->addend == reloc->sym->sec->sh.sh_size) {
639665
insn = find_last_insn(file, reloc->sym->sec);
640666
if (!insn) {
@@ -1775,6 +1801,7 @@ static int handle_group_alt(struct objtool_file *file,
17751801
orig_alt_group->orig_group = NULL;
17761802
orig_alt_group->first_insn = orig_insn;
17771803
orig_alt_group->last_insn = last_orig_insn;
1804+
orig_alt_group->nop = NULL;
17781805
} else {
17791806
if (orig_alt_group->last_insn->offset + orig_alt_group->last_insn->len -
17801807
orig_alt_group->first_insn->offset != special_alt->orig_len) {
@@ -1876,12 +1903,11 @@ static int handle_group_alt(struct objtool_file *file,
18761903
return -1;
18771904
}
18781905

1879-
if (nop)
1880-
list_add(&nop->list, &last_new_insn->list);
18811906
end:
18821907
new_alt_group->orig_group = orig_alt_group;
18831908
new_alt_group->first_insn = *new_insn;
1884-
new_alt_group->last_insn = nop ? : last_new_insn;
1909+
new_alt_group->last_insn = last_new_insn;
1910+
new_alt_group->nop = nop;
18851911
new_alt_group->cfi = orig_alt_group->cfi;
18861912
return 0;
18871913
}
@@ -1931,7 +1957,7 @@ static int handle_jump_alt(struct objtool_file *file,
19311957
else
19321958
file->jl_long++;
19331959

1934-
*new_insn = list_next_entry(orig_insn, list);
1960+
*new_insn = next_insn_same_sec(file, orig_insn);
19351961
return 0;
19361962
}
19371963

@@ -3522,11 +3548,28 @@ static struct instruction *next_insn_to_validate(struct objtool_file *file,
35223548
* Simulate the fact that alternatives are patched in-place. When the
35233549
* end of a replacement alt_group is reached, redirect objtool flow to
35243550
* the end of the original alt_group.
3551+
*
3552+
* insn->alts->insn -> alt_group->first_insn
3553+
* ...
3554+
* alt_group->last_insn
3555+
* [alt_group->nop] -> next(orig_group->last_insn)
35253556
*/
3526-
if (alt_group && insn == alt_group->last_insn && alt_group->orig_group)
3527-
return next_insn_same_sec(file, alt_group->orig_group->last_insn);
3557+
if (alt_group) {
3558+
if (alt_group->nop) {
3559+
/* ->nop implies ->orig_group */
3560+
if (insn == alt_group->last_insn)
3561+
return alt_group->nop;
3562+
if (insn == alt_group->nop)
3563+
goto next_orig;
3564+
}
3565+
if (insn == alt_group->last_insn && alt_group->orig_group)
3566+
goto next_orig;
3567+
}
35283568

35293569
return next_insn_same_sec(file, insn);
3570+
3571+
next_orig:
3572+
return next_insn_same_sec(file, alt_group->orig_group->last_insn);
35303573
}
35313574

35323575
/*
@@ -3777,34 +3820,37 @@ static int validate_branch(struct objtool_file *file, struct symbol *func,
37773820
return 0;
37783821
}
37793822

3823+
static int validate_unwind_hint(struct objtool_file *file,
3824+
struct instruction *insn,
3825+
struct insn_state *state)
3826+
{
3827+
if (insn->hint && !insn->visited && !insn->ignore) {
3828+
int ret = validate_branch(file, insn_func(insn), insn, *state);
3829+
if (ret && opts.backtrace)
3830+
BT_FUNC("<=== (hint)", insn);
3831+
return ret;
3832+
}
3833+
3834+
return 0;
3835+
}
3836+
37803837
static int validate_unwind_hints(struct objtool_file *file, struct section *sec)
37813838
{
37823839
struct instruction *insn;
37833840
struct insn_state state;
3784-
int ret, warnings = 0;
3841+
int warnings = 0;
37853842

37863843
if (!file->hints)
37873844
return 0;
37883845

37893846
init_insn_state(file, &state, sec);
37903847

37913848
if (sec) {
3792-
insn = find_insn(file, sec, 0);
3793-
if (!insn)
3794-
return 0;
3849+
sec_for_each_insn(file, sec, insn)
3850+
warnings += validate_unwind_hint(file, insn, &state);
37953851
} else {
3796-
insn = list_first_entry(&file->insn_list, typeof(*insn), list);
3797-
}
3798-
3799-
while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) {
3800-
if (insn->hint && !insn->visited && !insn->ignore) {
3801-
ret = validate_branch(file, insn_func(insn), insn, state);
3802-
if (ret && opts.backtrace)
3803-
BT_FUNC("<=== (hint)", insn);
3804-
warnings += ret;
3805-
}
3806-
3807-
insn = list_next_entry(insn, list);
3852+
for_each_insn(file, insn)
3853+
warnings += validate_unwind_hint(file, insn, &state);
38083854
}
38093855

38103856
return warnings;
@@ -4070,7 +4116,7 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio
40704116
*
40714117
* It may also insert a UD2 after calling a __noreturn function.
40724118
*/
4073-
prev_insn = list_prev_entry(insn, list);
4119+
prev_insn = prev_insn_same_sec(file, insn);
40744120
if ((prev_insn->dead_end ||
40754121
dead_end_function(file, insn_call_dest(prev_insn))) &&
40764122
(insn->type == INSN_BUG ||
@@ -4102,7 +4148,7 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio
41024148
if (insn->offset + insn->len >= insn_func(insn)->offset + insn_func(insn)->len)
41034149
break;
41044150

4105-
insn = list_next_entry(insn, list);
4151+
insn = next_insn_same_sec(file, insn);
41064152
}
41074153

41084154
return false;
@@ -4115,10 +4161,10 @@ static int add_prefix_symbol(struct objtool_file *file, struct symbol *func,
41154161
return 0;
41164162

41174163
for (;;) {
4118-
struct instruction *prev = list_prev_entry(insn, list);
4164+
struct instruction *prev = prev_insn_same_sec(file, insn);
41194165
u64 offset;
41204166

4121-
if (&prev->list == &file->insn_list)
4167+
if (!prev)
41224168
break;
41234169

41244170
if (prev->type != INSN_NOP)
@@ -4517,7 +4563,7 @@ int check(struct objtool_file *file)
45174563

45184564
warnings += ret;
45194565

4520-
if (list_empty(&file->insn_list))
4566+
if (!nr_insns)
45214567
goto out;
45224568

45234569
if (opts.retpoline) {
@@ -4626,7 +4672,7 @@ int check(struct objtool_file *file)
46264672
warnings += ret;
46274673
}
46284674

4629-
if (opts.orc && !list_empty(&file->insn_list)) {
4675+
if (opts.orc && nr_insns) {
46304676
ret = orc_create(file);
46314677
if (ret < 0)
46324678
goto out;

0 commit comments

Comments
 (0)