Skip to content

No public description #2124

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
copybara-service wants to merge 1 commit into
base: main
Choose a base branch
from
+111 −17
Open

No public description #2124

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions centipede/centipede_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1406,9 +1406,8 @@ TEST_F(CentipedeWithTemporaryLocalDir, EngineWorksInWorkerMode) {
fuzztest::internal::DefaultCallbacksFactory<
fuzztest::internal::CentipedeDefaultCallbacks>
callbacks;
EXPECT_DEATH(
[&] { std::exit(CentipedeMain(env, callbacks)); }(),
ContainsRegex("Failure *: INPUT FAILURE: some_failure_description"));
EXPECT_DEATH([&] { std::exit(CentipedeMain(env, callbacks)); }(),
ContainsRegex("Failure *: some_failure_description"));
}

TEST_F(CentipedeWithTemporaryLocalDir, EngineWorksInStandaloneMode) {
Expand All @@ -1428,7 +1427,7 @@ TEST_F(CentipedeWithTemporaryLocalDir, EngineWorksInStandaloneMode) {
const int status = std::system(test_command.c_str());
std::exit(WEXITSTATUS(status));
}(),
ContainsRegex("Failure *: INPUT FAILURE: some_failure_description"));
ContainsRegex("Failure *: some_failure_description"));
}

} // namespace
Expand Down
121 changes: 108 additions & 13 deletions centipede/engine_worker.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <fcntl.h>
#include <sys/resource.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <sys/un.h>
Expand All @@ -26,6 +27,7 @@
#include <cstdlib>
#include <cstring>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <type_traits>
Expand Down Expand Up @@ -217,6 +219,7 @@ enum class WorkerAction {
kTestGetSeeds,
kTestMutate,
kTestExecute,
kNoOp,
};

constexpr std::string_view kWorkerBinaryIdOutputFlagHeader =
Expand All @@ -237,6 +240,7 @@ constexpr std::string_view kWorkerOutputsBlobSequencePathFlagHeader =
constexpr std::string_view kWorkerPersistentModeSocketPathFlagHeader =
"persistent_mode_socket="; // TODO: Use better flag names when
// standardizing the protocol.
constexpr std::string_view kWorkerCrossOverLevel = "crossover_level=";

struct WorkerState {
std::atomic<bool> has_failure_output = false;
Expand Down Expand Up @@ -295,14 +299,28 @@ void WorkerEmitError(std::string_view message) {
WorkerEmitFailureOutput("SETUP FAILURE: ", message);
}

// TODO(xinhaoyuan): Some of them are planned to be removed.
static constexpr std::array<std::string_view, 3> kNonFindingPrefixes = {
"SETUP FAILURE:",
"IGNORED FAILURE:",
"SKIPPED TEST:",
};

void WorkerEmitFinding(std::string_view description,
std::string_view signature) {
WorkerCheck(
GetWorkerState().in_adapter_execute.load(std::memory_order_relaxed),
"Must emit finding in adapter execute");
for (auto bad_prefix : kNonFindingPrefixes) {
if (description.substr(0, bad_prefix.size()) == bad_prefix) {
WorkerLog("Got bad prefix in the finding description: ", description);
WorkerEmitError("Bad prefix in the finding description");
return;
}
}
const bool ignored = GetWorkerState().has_finding.exchange(true);
if (!ignored) {
WorkerCheck(WorkerEmitFailureOutput("INPUT FAILURE: ", description),
WorkerCheck(WorkerEmitFailureOutput(/*prefix=*/"", description),
"Failed to emit failure output for the finding");
if (const char* finding_signature_path =
GetWorkerFlag(kWorkerFailureSignaturePathFlagHeader);
Expand Down Expand Up @@ -415,8 +433,25 @@ BlobSequence* GetOutputsBlobSequence() {
return result;
}

WorkerAction GetWorkerAction() {
static WorkerAction worker_action = [] {
int GetCrossOverLevel() {
static int result = []() {
const char* cross_over_level_str = GetWorkerFlag(kWorkerCrossOverLevel);
if (cross_over_level_str != nullptr) {
const int parsed =
atoi(cross_over_level_str); // NOLINT: can't use strto64, etc.
if (0 <= parsed && parsed <= 100) return parsed;
}
// Default
return 50;
}();
return result;
}

std::optional<WorkerAction> GetWorkerAction() {
static auto worker_action = []() -> std::optional<WorkerAction> {
if (HasWorkerSwitchFlag("dump_configuration")) {
return WorkerAction::kNoOp;
}
if (HasWorkerSwitchFlag("dump_binary_id")) {
return WorkerAction::kGetBinaryId;
}
Expand All @@ -427,7 +462,9 @@ WorkerAction GetWorkerAction() {
return WorkerAction::kTestGetSeeds;
}
auto* inputs_blobseq = GetInputsBlobSequence();
WorkerCheck(inputs_blobseq != nullptr, "input blob sequence is not found");
if (inputs_blobseq == nullptr) {
return std::nullopt;
}
auto request_type_blob = inputs_blobseq->Read();
if (IsMutationRequest(request_type_blob)) {
inputs_blobseq->Reset();
Expand All @@ -437,9 +474,7 @@ WorkerAction GetWorkerAction() {
inputs_blobseq->Reset();
return WorkerAction::kTestExecute;
}
WorkerCheck(false, "unknown worker action from the flags");
// should not reach here.
std::abort();
return std::nullopt;
}();
return worker_action;
}
Expand Down Expand Up @@ -587,11 +622,20 @@ void WorkerDoMutate(const FuzzTestAdapter& adapter) {
// Assume RAND_MAX is large enough and not worry about fairness for now.
const auto origin = rand_r(&seed) % origin_inputs.size();
emitted_inputs.clear();
adapter.Mutate(adapter.ctx, origin_inputs[origin], /*shrink=*/0,
&input_sink);
const bool do_cross_over = adapter.CrossOver != nullptr &&
rand_r(&seed) % 100 < GetCrossOverLevel();
if (do_cross_over) {
const auto other = rand_r(&seed) % origin_inputs.size();
adapter.CrossOver(adapter.ctx, origin_inputs[origin],
origin_inputs[other], &input_sink);
} else {
adapter.Mutate(adapter.ctx, origin_inputs[origin], /*shrink=*/0,
&input_sink);
}
WORKER_CHECK_FOR_ERROR();
WorkerCheck(emitted_inputs.size() == 1,
"Mutate must emit exactly one input");
do_cross_over ? "CrossOver must emit exactly one input"
: "Mutate must emit exactly one input");
mutant_bytes.clear();
adapter.SerializeInputContent(adapter.ctx, emitted_inputs[0],
&mutant_bytes_sink);
Expand Down Expand Up @@ -663,13 +707,46 @@ void WorkerDoExecute(const FuzzTestAdapter& adapter) {
return true;
}();

// Utils to get execution stats.

size_t last_time_usec = 0;
auto UsecSinceLast = [&last_time_usec]() {
const uint64_t t = TimeInUsec();
const uint64_t ret_val = t - last_time_usec;
last_time_usec = t;
return ret_val;
};

auto GetPeakRSSMb = []() -> size_t {
struct rusage usage = {};
if (getrusage(RUSAGE_SELF, &usage) != 0) return 0;
#ifdef __APPLE__
// On MacOS, the unit seems to be byte according to experiment, while some
// documents mentioned KiB. This could depend on OS variants.
return usage.ru_maxrss >> 20;
#else // __APPLE__
// On Linux, ru_maxrss is in KiB
return usage.ru_maxrss >> 10;
#endif // __APPLE__
};

// In-loop variables declared outside to save allocations.
std::vector<uint64_t> features;
std::vector<uint8_t> serialized_metadata;
std::vector<FuzzTestInputHandle> emitted_inputs;
const auto input_sink = GetInputSinkTo(emitted_inputs);

for (size_t i = 0; i < num_inputs; i++) {
// The stats gathered here are slightly different than
// the original definition: Here the exec_time_usec will cover the time used
// to process coverage feedback, which is usually fine for the purpose of
// corpus scheduling, but coverage processing can be non-deterministic due
// to optimization. E.g. Execute() may perform deep and slow cleanup on
// the beginning of consecutive Execute() calls to reduce noise.
//
// TODO(xinhaoyuan): Consider improving it. E.g. let user override stats.
ExecutionResult::Stats stats = {};
UsecSinceLast();
auto blob = inputs_blobseq->Read();
if (!blob.IsValid()) return; // no more blobs to read.
WorkerCheck(IsDataInput(blob), "Must read data input");
Expand Down Expand Up @@ -698,7 +775,9 @@ void WorkerDoExecute(const FuzzTestAdapter& adapter) {
}};

GetWorkerState().in_adapter_execute = true;
stats.prep_time_usec = UsecSinceLast();
adapter.Execute(adapter.ctx, input, &feedback_sink);
stats.exec_time_usec = UsecSinceLast();
GetWorkerState().in_adapter_execute = false;
WORKER_CHECK_FOR_ERROR();

Expand Down Expand Up @@ -748,6 +827,12 @@ void WorkerDoExecute(const FuzzTestAdapter& adapter) {
WorkerLog("failed to write input metadata");
break;
}
stats.post_time_usec = UsecSinceLast();
stats.peak_rss_mb = GetPeakRSSMb();
if (!BatchResult::WriteStats(stats, *outputs_blobseq)) {
WorkerLog("failed to write stats");
break;
}
if (!BatchResult::WriteInputEnd(*outputs_blobseq)) {
WorkerLog("failed to write input end");
break;
Expand Down Expand Up @@ -837,6 +922,12 @@ FuzzTestWorkerStatus WorkerRun(const FuzzTestAdapterManager& manager) {
}

const auto action = GetWorkerAction();
WorkerCheck(action.has_value(), "No worker action to run");

if (action == WorkerAction::kNoOp) {
return kFuzzTestWorkerSuccess;
}

if (action == WorkerAction::kGetBinaryId) {
WorkerDoGetBinaryId(manager);
return kFuzzTestWorkerSuccess;
Expand Down Expand Up @@ -881,8 +972,8 @@ FuzzTestWorkerStatus WorkerRun(const FuzzTestAdapterManager& manager) {
WorkerCheck(manager.ConstructAdapter != nullptr,
"ConstructAdapter is not defined");
FuzzTestAdapter adapter = {};
manager.ConstructAdapter(manager.ctx, /*diagnostic_sink=*/&diagnostic_sink,
&adapter);
manager.ConstructAdapter(manager.ctx,
/*diagnostic_sink=*/&diagnostic_sink, &adapter);
WORKER_CHECK_FOR_ERROR();
WorkerCheck(adapter.SetUpCoverageDomains != nullptr,
"SetUpCoverageDomains must be defined");
Expand Down Expand Up @@ -929,7 +1020,11 @@ using ::fuzztest::internal::WorkerRun;

} // namespace

int FuzzTestWorkerIsRequired() { return GetWorkerFlags().present; }
int FuzzTestWorkerIsRequired() {
static int result = GetWorkerFlags().present &&
fuzztest::internal::GetWorkerAction().has_value();
return result;
}

FuzzTestWorkerStatus FuzzTestWorkerMaybeRun(
const FuzzTestAdapterManager* manager) {
Expand Down
Loading