Skip to content

Commit e0666fc

Browse files
MarkoutteMarkoutte
authored
Add tests for fuzzing platform (#1516)
1 parent 047ea48 commit e0666fc

10 files changed

Lines changed: 501 additions & 34 deletions

File tree

‎utbot-framework/src/main/kotlin/org/utbot/engine/UtBotSymbolicEngine.kt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -344,7 +344,6 @@ class UtBotSymbolicEngine(
344344
methodUnderTest,
345345
collectConstantsForFuzzer(graph),
346346
names,
347-
{ mockStrategy.eligibleToMock(it, classUnderTest) },
348347
listOf(transform(ValueProvider.of(defaultValueProviders(defaultIdGenerator))))
349348
) { thisInstance, descr, values ->
350349
if (controller.job?.isActive == false || System.currentTimeMillis() >= until) {

‎utbot-fuzzers/src/main/kotlin/org/utbot/fuzzing/JavaLanguage.kt‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@ suspend fun runJavaFuzzing(
4444
methodUnderTest: ExecutableId,
4545
constants: Collection<FuzzedConcreteValue>,
4646
names: List<String>,
47-
mock: (ClassId) -> Boolean = { false },
4847
providers: List<ValueProvider<FuzzedType, FuzzedValue, FuzzedDescription>> = defaultValueProviders(idGenerator),
4948
exec: suspend (thisInstance: FuzzedValue?, description: FuzzedDescription, values: List<FuzzedValue>) -> BaseFeedback<Trie.Node<Instruction>, FuzzedType, FuzzedValue>
5049
) {
@@ -83,7 +82,7 @@ suspend fun runJavaFuzzing(
8382
null
8483
}
8584
}
86-
shouldMock = mock
85+
shouldMock = { false }
8786
}
8887

8988
val thisInstance = with(methodUnderTest) {

‎utbot-fuzzers/src/test/java/org/utbot/fuzzing/samples/Stubs.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
package org.utbot.fuzzing.samples;
2+
3+
@SuppressWarnings("unused")
4+
public class Stubs {
5+
6+
public static void name(String value) {}
7+
8+
}

‎utbot-fuzzers/src/test/kotlin/org/utbot/fuzzing/JavaFuzzingTest.kt‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package org.utbot.fuzzing
2+
3+
import kotlinx.coroutines.runBlocking
4+
import org.junit.jupiter.api.Assertions
5+
import org.junit.jupiter.api.Test
6+
import org.utbot.framework.plugin.api.MethodId
7+
import org.utbot.framework.plugin.api.TestIdentityPreservingIdGenerator
8+
import org.utbot.framework.plugin.api.UtPrimitiveModel
9+
import org.utbot.framework.plugin.api.util.*
10+
import org.utbot.fuzzer.FuzzedConcreteValue
11+
import org.utbot.fuzzing.samples.Stubs
12+
import org.utbot.fuzzing.utils.Trie
13+
14+
class JavaFuzzingTest {
15+
16+
@Test
17+
fun `string generates same values`() {
18+
fun collect(): List<String> {
19+
return runBlockingWithContext {
20+
val results = mutableListOf<String>()
21+
var count = 0
22+
val probes = 10000
23+
runJavaFuzzing(
24+
TestIdentityPreservingIdGenerator,
25+
methodUnderTest = MethodId(Stubs::class.id, "name", voidClassId, listOf(stringClassId)),
26+
constants = listOf(
27+
FuzzedConcreteValue(stringClassId, "Hello"),
28+
FuzzedConcreteValue(stringClassId, "World"),
29+
FuzzedConcreteValue(stringClassId, "!"),
30+
),
31+
names = emptyList()
32+
) { _, _, values ->
33+
results += (values.first().model as UtPrimitiveModel).value as String
34+
BaseFeedback(Trie.emptyNode(), if (++count < probes) Control.CONTINUE else Control.STOP)
35+
}
36+
Assertions.assertEquals(count, results.size)
37+
results
38+
}
39+
}
40+
41+
val probe1 = collect()
42+
val probe2 = collect()
43+
Assertions.assertEquals(probe1, probe2)
44+
}
45+
46+
private fun <T> runBlockingWithContext(block: suspend () -> T) : T {
47+
return withUtContext(UtContext(this::class.java.classLoader)) {
48+
runBlocking {
49+
block()
50+
}
51+
}
52+
}
53+
}

‎utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Api.kt‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
@file:JvmName("FuzzingApi")
22
package org.utbot.fuzzing
33

4+
import kotlinx.coroutines.yield
45
import mu.KotlinLogging
56
import org.utbot.fuzzing.seeds.KnownValue
67
import org.utbot.fuzzing.utils.chooseOne
@@ -259,7 +260,7 @@ suspend fun <T, R, D : Description<T>, F : Feedback<T, R>> Fuzzing<T, R, D, F>.f
259260
val seeds = Statistics<T, R, F>()
260261
run breaking@ {
261262
sequence {
262-
while (true) {
263+
while (description.parameters.isNotEmpty()) {
263264
if (dynamicallyGenerated.isNotEmpty()) {
264265
yield(dynamicallyGenerated.removeFirst())
265266
} else {
@@ -280,15 +281,11 @@ suspend fun <T, R, D : Description<T>, F : Feedback<T, R>> Fuzzing<T, R, D, F>.f
280281
}
281282
}
282283
}.forEach execution@ { values ->
284+
yield()
283285
check(values.parameters.size == values.result.size) { "Cannot create value for ${values.parameters}" }
284286
val valuesCache = mutableMapOf<Result<T, R>, R>()
285287
val result = values.result.map { valuesCache.computeIfAbsent(it) { r -> create(r) } }
286-
val feedback = try {
287-
fuzzing.handle(description, result)
288-
} catch (t: Throwable) {
289-
logger.error(t) { "Error when running fuzzing with $values" }
290-
return@execution
291-
}
288+
val feedback = fuzzing.handle(description, result)
292289
when (feedback.control) {
293290
Control.CONTINUE -> {
294291
seeds.put(random, configuration, feedback, values)
@@ -634,8 +631,8 @@ private class Node<TYPE, RESULT>(
634631

635632

636633
private class Statistics<TYPE, RESULT, FEEDBACK : Feedback<TYPE, RESULT>> {
637-
private val seeds = hashMapOf<FEEDBACK, Node<TYPE, RESULT>>()
638-
private val count = hashMapOf<FEEDBACK, Long>()
634+
private val seeds = linkedMapOf<FEEDBACK, Node<TYPE, RESULT>>()
635+
private val count = linkedMapOf<FEEDBACK, Long>()
639636

640637
fun put(random: Random, configuration: Configuration, feedback: FEEDBACK, seed: Node<TYPE, RESULT>) {
641638
if (random.flipCoin(configuration.probUpdateSeedInsteadOfKeepOld)) {

‎utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/Providers.kt‎

Lines changed: 6 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ fun interface ValueProvider<T, R, D : Description<T>> {
7070
*
7171
* This model provider is called before `anotherValueProviders`.
7272
*/
73-
fun with(anotherValueProvider: ValueProvider<T, R, D>): ValueProvider<T, R, D> {
73+
infix fun with(anotherValueProvider: ValueProvider<T, R, D>): ValueProvider<T, R, D> {
7474
fun toList(m: ValueProvider<T, R, D>) = if (m is Combined<T, R, D>) m.providers else listOf(m)
7575
return Combined(toList(this) + toList(anotherValueProvider))
7676
}
@@ -122,15 +122,8 @@ fun interface ValueProvider<T, R, D : Description<T>> {
122122
* Creates new value provider that creates default value if no values are generated by this provider.
123123
*/
124124
fun withFallback(fallbackSupplier: (T) -> Seed<T, R>) : ValueProvider<T, R, D> {
125-
val thisProvider = this
126-
return ValueProvider { description, type ->
127-
if (accept(type)) {
128-
thisProvider.generate(description, type)
129-
.takeIf { it.iterator().hasNext() }
130-
?: sequenceOf(fallbackSupplier(type))
131-
} else {
132-
emptySequence()
133-
}
125+
return withFallback { _, type ->
126+
sequenceOf(fallbackSupplier(type))
134127
}
135128
}
136129

@@ -170,10 +163,6 @@ fun interface ValueProvider<T, R, D : Description<T>> {
170163
}
171164
}
172165

173-
fun<T, R, D : Description<T>> List<ValueProvider<T, R, D>>.withFallback(fallbackSupplier: (T) -> Seed<T, R>) : List<ValueProvider<T, R, D>> {
174-
return listOf(ValueProvider.of(this).withFallback(fallbackSupplier))
175-
}
176-
177166
/**
178167
* Simple value provider for a concrete type.
179168
*
@@ -186,6 +175,8 @@ class TypeProvider<T, R, D : Description<T>>(
186175
) : ValueProvider<T, R, D> {
187176
override fun accept(type: T) = this.type == type
188177
override fun generate(description: D, type: T) = sequence {
189-
this.generate(description, type)
178+
if (accept(type)) {
179+
this.generate(description, type)
180+
}
190181
}
191182
}

‎utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/demo/JsonFuzzing.kt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ private class JsonBuilder(
3131
*/
3232
@Suppress("RemoveExplicitTypeArguments")
3333
suspend fun main() {
34+
var count = 0
3435
BaseFuzzing<CustomType, JsonBuilder, Description<CustomType>, Feedback<CustomType, JsonBuilder>>(
3536
TypeProvider(CustomType.INT) { _, _ ->
3637
for (b in Signed.values()) {
@@ -79,7 +80,7 @@ suspend fun main() {
7980
},
8081
) { _, values ->
8182
println(values)
82-
emptyFeedback()
83+
if (++count < 1000) emptyFeedback() else error("")
8384
}.fuzz(
8485
Description(listOf(CustomType.LST, CustomType.OBJ)),
8586
Random(0),

‎utbot-fuzzing/src/main/kotlin/org/utbot/fuzzing/utils/Trie.kt‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -175,12 +175,6 @@ open class Trie<T, K>(
175175
get() = error("empty node has no data")
176176
override val count: Int
177177
get() = 0
178-
override fun equals(other: Any?): Boolean {
179-
return false
180-
}
181-
override fun hashCode(): Int {
182-
return 0
183-
}
184178
}
185179

186180
companion object {

0 commit comments

Comments
 (0)