diff --git a/README.md b/README.md index 3f16aa7..5ec9cd0 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,6 @@ async Main () { projectFolder, threads, // [Optional] Default = OS defined. 0 for in-process. maxTime, // [Optional] Default = 10s. - maxRuns, // [Optional] Default = 100e3. methods, classes, // [Optional] Default = all. files, // [Optional] Default = all. source, dist, // [Optional] Default = 'src' and 'dist'. @@ -84,8 +83,7 @@ fast-fuzz -V, --version Output the version number -p, --threads The number of parallel threads. Default = OS defined. 0 for in-process. - -t, --maxTime The maximum time(ms) per function. Default = 10s. - -n, --maxRuns The maximum count of runs per function. Default = 100e3. + -t, --maxTime The maximum time(ms) per function. Default = 10 min. -m, --methods A Regex expression to filter the methods to test. -c, --classes A Regex expression to filter the classes to test. -f, --files A Regex expression to filter the files to test. @@ -187,9 +185,7 @@ and can thus be reused in other code. - All unrelated literals should be put at the beginning of the file. - Any methods that are not exported or within a class should not contain literals as these will be picked up by the earlier fuzzed method in the file. Another option is to put ineligible methods before the fuzzed ones. -## TODO Priorities - -- Benchmarking of target functions to determine the best run time and number of tests. +## TODO - Redundant runner for the args-results pairs. - Stuffing for nested type args by registration of generators and collection of values upon successful detection. diff --git a/package.json b/package.json index 6b7ea4a..52ebde7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "fast-fuzz", - "version": "5.0.2", + "version": "5.0.3", "description": "Test case fuzzer with branch coverage guidance.", "main": "./dist/src/fast-fuzz.js", "types": "./dist/src/fast-fuzz.d.ts", diff --git a/src/fuzz/fuzz.ts b/src/fuzz/fuzz.ts index 9841198..7f82d8a 100644 --- a/src/fuzz/fuzz.ts +++ b/src/fuzz/fuzz.ts @@ -43,10 +43,10 @@ let instances: { /** * Inits the local code analysis and type stuffing. - * @param folder - * @param [src] - * @param [dist] - * @param [instances] + * @param folder @type {string} + * @param [src] @type {string} + * @param [dist] @type {string} + * @param [instances] @type {any} */ export async function init( folder: string, @@ -63,9 +63,9 @@ export async function init( /** * Inits the local code analysis. - * @param folder - * @param [src] - * @param [dist] + * @param folder @type {string} + * @param [src] @type {string} + * @param [dist] @type {string} */ async function initLocal( folder: string, @@ -108,15 +108,16 @@ function checkInit(): void { /** * Counts the number of methods. - * @param [methodPattern] - * @param [classPattern] + * @param [methodPattern] @type {RegExp} + * @param [classPattern] @type {RegExp} + * @param [filePattern] @type {RegExp} * @returns count of methods. */ export async function count( methodPattern?: string, classPattern?: string, filePattern?: string - ): Promise { +): Promise { checkInit(); let methodCount = 0; @@ -143,16 +144,14 @@ export async function count( /** * Fuzz the TS folder. - * @param [maxTime] - * @param [maxRuns] - * @param [methodPattern] - * @param [classPattern] - * @param [filePattern] - * @param [resultsOut] + * @param [maxTime] @type {number} + * @param [methodPattern] @type {RegExp} + * @param [classPattern] @type {RegExp} + * @param [filePattern] @type {RegExp} + * @param [resultsOut] @type {Results[]} */ export async function fuzz( maxTime = 1e4, - maxRuns = 1e5, methodPattern?: string, classPattern?: string, filePattern?: string, @@ -184,6 +183,7 @@ export async function fuzz( if (method.name === '__constructor') { continue; } /* #endregion */ + /* #region Check the resume state and skip those methods. */ if (count !== 0 && currentCount >= count) { break; } @@ -194,26 +194,16 @@ export async function fuzz( currentIndex++; currentCount++; + /* #endregion */ // Set the generators to reset with the new literals. - Globals.methodCount++; Globals.literals = method.literals; + // Benchmark for number of runs in the specified time. + const maxRuns = await getMaxRuns(method, file, maxTime); + // Run the appropiate static & async method - let fuzzResults: Result[] = []; - if (method.isStatic || method.className === undefined) { - if (method.isAsync) { - fuzzResults = await fuzzStaticAsync(file, method, maxTime, maxRuns, fuzzResults); - } else { - fuzzResults = fuzzStatic(file, method, maxTime, maxRuns, fuzzResults); - } - } else { - if (method.isAsync) { - fuzzResults = await fuzzMethodAsync(file, method, maxTime, maxRuns, fuzzResults); - } else { - fuzzResults = fuzzMethod(file, method, maxTime, maxRuns, fuzzResults); - } - } + let fuzzResults: Result[] = await fuzzAnyMethod(method, file, maxTime, maxRuns); // Output the method results. resultsOut.push({ @@ -221,6 +211,7 @@ export async function fuzz( className: method.className, namespaces: method.namespaces, file, + avgSpeed: maxRuns / maxTime, results: fuzzResults }); @@ -233,14 +224,68 @@ export async function fuzz( return resultsOut; } + +/** + * Gets max runs from 1000, in 10% of the time. + * @param method @type {ModuleMethod} + * @param file @type {string} + * @param maxTime @type {number} + * @returns maxRuns @type {number} + */ +async function getMaxRuns( + method: ModuleMethod, + file: string, + maxTime: number +): Promise { + const numRuns = 1e3; + const start = Date.now(); + + await fuzzAnyMethod(method, file, maxTime / 10, numRuns); + const result = Math.floor(maxTime / ((Date.now() - start) / numRuns)); + + return Math.max(numRuns, result); +} + +/** + * Fuzzes any method. + * @param method @type {ModuleMethod} + * @param file @type {string} + * @param maxTime @type {number} + * @param maxRuns @type {number} + * @returns @type {Result[]} + */ +async function fuzzAnyMethod( + method: ModuleMethod, + file: string, + maxTime: number, + maxRuns: number +): Promise { + Globals.methodCount++; + let fuzzResults: Result[] = []; + if (method.isStatic || method.className === undefined) { + if (method.isAsync) { + fuzzResults = await fuzzStaticAsync(file, method, maxTime, maxRuns, fuzzResults); + } else { + fuzzResults = fuzzStatic(file, method, maxTime, maxRuns, fuzzResults); + } + } else { + if (method.isAsync) { + fuzzResults = await fuzzMethodAsync(file, method, maxTime, maxRuns, fuzzResults); + } else { + fuzzResults = fuzzMethod(file, method, maxTime, maxRuns, fuzzResults); + } + } + return fuzzResults; +} + /** * Fuzz static methods. - * @param filePath - * @param method - * @param [maxTime] - * @param [maxRuns] - * @param resultsOut - * @returns static + * @param filePath @type {string} + * @param method @type {ModuleMethod} + * @param [maxTime] @type {number} + * @param [maxRuns] @type {number} + * @param resultsOut @type {Result[]} + * @returns results @type {Result[]} */ function fuzzStatic( filePath: string, @@ -297,12 +342,12 @@ function fuzzStatic( /** * Fuzz static methods async. - * @param filePath - * @param method - * @param [maxTime] - * @param [maxRuns] - * @param resultsOut - * @returns static async + * @param filePath @type {string} + * @param method @type {ModuleMethod} + * @param [maxTime] @type {number} + * @param [maxRuns] @type {number} + * @param resultsOut @type {Result[]} + * @returns results @type {Result[]} */ async function fuzzStaticAsync( filePath: string, @@ -359,12 +404,12 @@ async function fuzzStaticAsync( /** * Fuzz methods. - * @param filePath - * @param method - * @param [maxTime] - * @param [maxRuns] - * @param resultsOut - * @returns method + * @param filePath @type {string} + * @param method @type {ModuleMethod} + * @param [maxTime] @type {number} + * @param [maxRuns] @type {number} + * @param resultsOut @type {Result[]} + * @returns results @type {Result[]} */ function fuzzMethod( filePath: string, @@ -436,12 +481,12 @@ function fuzzMethod( /** * Fuzz methods async. - * @param filePath - * @param method - * @param [maxTime] - * @param [maxRuns] - * @param resultsOut - * @returns method async + * @param filePath @type {string} + * @param method @type {ModuleMethod} + * @param [maxTime] @type {number} + * @param [maxRuns] @type {number} + * @param resultsOut @type {Result[]} + * @returns results @type {Result[]} */ async function fuzzMethodAsync( filePath: string, @@ -513,9 +558,9 @@ async function fuzzMethodAsync( /** * Gets args. - * @param method - * @param generator - * @returns args + * @param method @type {ModuleMethod} + * @param generator @type {GeneratorArg} + * @returns args @type {any[]} */ function getArgs(method: ModuleMethod, generator: GeneratorArg): any[] { // Set the method to generate new arguments. @@ -535,8 +580,8 @@ function getArgs(method: ModuleMethod, generator: GeneratorArg): any[] { /** * Loads instances after every method fuzz. - * @param instances - * @param instancesOut + * @param instances @type {any[]} + * @param instancesOut @type {any} */ function loadInstances( instances: { @@ -598,8 +643,8 @@ function loadInstances( /** * Gets instances. - * @returns instances + * @returns instances @type {any} */ export async function getInstances(): Promise { return Globals.instances; -} \ No newline at end of file +} diff --git a/src/fuzz/fuzzAsync.ts b/src/fuzz/fuzzAsync.ts index a91967f..92fad97 100644 --- a/src/fuzz/fuzzAsync.ts +++ b/src/fuzz/fuzzAsync.ts @@ -57,7 +57,9 @@ export async function fuzzAsync( // Vary the number of runs based on target area. let runCount = 0; + let lastIndex = 0; const maxRunsMode: number = maxRunsModes.pop(); + const maxRunsFailFast = Math.floor(0.682 * maxRunsMode); const maxRunsCheck: number = Math.pow(10, Math.max(1, Math.floor(Math.log10(maxRunsMode)) - 1)); const start: number = Date.now(); const runTime: number = maxTime * maxRunsMode / (MODE_SCALE * maxRuns); @@ -74,7 +76,10 @@ export async function fuzzAsync( // eslint-disable-next-line while (true) { // Check the running stats for termination. - if (runCount % maxRunsCheck == 0) { isExpired = (Date.now() - start) > runTime; } + if (runCount % maxRunsCheck == 0) { + isExpired = (Date.now() - start) > runTime; + isExpired = isExpired || ((runCount - lastIndex) > maxRunsFailFast); + } if (isExpired || runCount > maxRunsMode) { break; } @@ -123,6 +128,7 @@ export async function fuzzAsync( cleanupError(result); } + // Update results. resultsOut.push(new Result({ id: resultCount++, modeId: mode, @@ -135,12 +141,12 @@ export async function fuzzAsync( ) })); + // Update accounting vars. + lastIndex = runCount; persistInstances(); } } - resetCoverage(filePath); - // Report the generated tests. return resultsOut; } diff --git a/src/fuzz/fuzzCaller.ts b/src/fuzz/fuzzCaller.ts index 324def7..8b79ceb 100644 --- a/src/fuzz/fuzzCaller.ts +++ b/src/fuzz/fuzzCaller.ts @@ -51,7 +51,6 @@ async function init( * Fuzz the TS folder. * @param folder * @param [maxTime] - * @param [maxRuns] * @param [methodPattern] * @param [classPattern] * @param [src] @@ -64,7 +63,6 @@ export async function fuzz( folder: string, threads?: number, maxTime = 1e4, - maxRuns = 1e5, methodPattern?: string, classPattern?: string, filePattern?: string, @@ -104,7 +102,6 @@ export async function fuzz( if (threads === 0) { await fuzzSingle( maxTime, - maxRuns, methodPattern, classPattern, filePattern, @@ -115,7 +112,6 @@ export async function fuzz( } else { await fuzzRunner.fuzz( maxTime, - maxRuns, methodPattern, classPattern, filePattern, diff --git a/src/fuzz/fuzzRunner.ts b/src/fuzz/fuzzRunner.ts index 4851872..78cc9c4 100644 --- a/src/fuzz/fuzzRunner.ts +++ b/src/fuzz/fuzzRunner.ts @@ -95,14 +95,12 @@ export class FuzzRunner { /** * Fuzz the TS folder. * @param [maxTime] - * @param [maxRuns] * @param [methodPattern] * @param [classPattern] * @param [resultsOut] */ async fuzz( maxTime = 1e4, - maxRuns = 1e5, methodPattern?: string, classPattern?: string, filePattern?: string, @@ -119,7 +117,6 @@ export class FuzzRunner { this.workers[index], finalPromise, maxTime, - maxRuns, methodPattern, classPattern, filePattern, @@ -179,7 +176,6 @@ export class FuzzRunner { * @param count * @param finalPromise * @param maxTime - * @param maxRuns * @param methodPattern * @param classPattern * @param filePattern @@ -189,7 +185,6 @@ export class FuzzRunner { worker: FuzzWorker, finalPromise: any, maxTime: number, - maxRuns: number, methodPattern: string, classPattern: string, filePattern: string, @@ -208,7 +203,6 @@ export class FuzzRunner { // Run one method. resultsP = worker.fuzz( maxTime, - maxRuns, methodPattern, classPattern, filePattern, diff --git a/src/fuzz/fuzzSync.ts b/src/fuzz/fuzzSync.ts index af383a1..e5f2213 100644 --- a/src/fuzz/fuzzSync.ts +++ b/src/fuzz/fuzzSync.ts @@ -57,7 +57,9 @@ export function fuzzSync( // Vary the number of runs based on target area. let runCount = 0; + let lastIndex = 0; const maxRunsMode: number = maxRunsModes.pop(); + const maxRunsFailFast = Math.floor(0.682 * maxRunsMode); const maxRunsCheck: number = Math.pow(10, Math.max(1, Math.floor(Math.log10(maxRunsMode)) - 1)); const start: number = Date.now(); const runTime: number = maxTime * maxRunsMode / (MODE_SCALE * maxRuns); @@ -76,6 +78,7 @@ export function fuzzSync( // Check the running stats for termination. if (runCount % maxRunsCheck == 0) { isExpired = (Date.now() - start) > runTime; + isExpired = isExpired || ((runCount - lastIndex) > maxRunsFailFast); } if (isExpired || runCount > maxRunsMode) { break; @@ -125,6 +128,7 @@ export function fuzzSync( cleanupError(result); } + // Update results. resultsOut.push(new Result({ id: resultCount++, modeId: mode, @@ -137,12 +141,12 @@ export function fuzzSync( ) })); + // Update accounting vars. + lastIndex = runCount; persistInstances(); } } - resetCoverage(filePath); - // Report the generated tests. return resultsOut; } diff --git a/src/fuzz/fuzzWorker.ts b/src/fuzz/fuzzWorker.ts index 434d391..133e8ce 100644 --- a/src/fuzz/fuzzWorker.ts +++ b/src/fuzz/fuzzWorker.ts @@ -46,15 +46,13 @@ export class FuzzWorker { /** * Fuzz the TS folder. - * @param [maxTime] - * @param [maxRuns] + * @param [maxTime] * @param [methodPattern] * @param [classPattern] * @param [resultsOut] */ async fuzz( maxTime = 1e4, - maxRuns = 1e5, methodPattern?: string, classPattern?: string, filePattern?: string, @@ -66,7 +64,6 @@ export class FuzzWorker { name: Call.fuzz, args: [ maxTime, - maxRuns, methodPattern, classPattern, filePattern, diff --git a/src/fuzz/result.ts b/src/fuzz/result.ts index 772b7a8..b58972c 100644 --- a/src/fuzz/result.ts +++ b/src/fuzz/result.ts @@ -23,5 +23,6 @@ export class Results { className?: string; namespaces: string[]; file: string; + avgSpeed: number; results: Result[]; } \ No newline at end of file diff --git a/src/fuzz/worker.ts b/src/fuzz/worker.ts index d6c177c..9d70e8b 100644 --- a/src/fuzz/worker.ts +++ b/src/fuzz/worker.ts @@ -39,8 +39,7 @@ const job = multee.createHandler( callArgs.args[3], callArgs.args[4], callArgs.args[5], - callArgs.args[6], - callArgs.args[7] + callArgs.args[6] )); case Call.getInstances: return safeStringify(await getInstances()); diff --git a/src/index.ts b/src/index.ts index 5fbe017..0ebe88d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,9 +11,8 @@ commander .addHelpCommand() .version(require('../../package').version) .option('-i, --input ', 'Path of the Typescript project.') - .option('-p, --threads ', 'The number of parallel threads. Default = OS defined. 0 for in-process.') + .option('-n, --threads ', 'The number of parallel threads. Default = OS defined. 0 for in-process.') .option('-t, --maxTime ', 'The maximum time(ms) per function. Actual value is multiplied by 4. Default = 10s.') - .option('-n, --maxRuns ', 'The maximum count of runs per function. Default = 100e3.') .option('-m, --methods ', 'A Regex expression to filter the methods to test.') .option('-c, --classes ', 'A Regex expression to filter the classes to test.') .option('-f, --files ', 'A Regex expression to filter the files to test.') @@ -37,10 +36,6 @@ async function Main() { commander.maxTime = Number.parseFloat(commander.maxTime); if (Number.isNaN(commander.maxTime)) { delete commander.maxTime; } } - if (commander.maxRuns !== undefined) { - commander.maxRuns = Number.parseFloat(commander.maxRuns); - if (Number.isNaN(commander.maxRuns)) { delete commander.maxRuns; } - } commander.quiet = commander.quiet !== undefined; commander.force = commander.force !== undefined; @@ -67,7 +62,6 @@ async function Main() { commander.input, commander.threads, commander.maxTime, - commander.maxRuns, commander.methods, commander.classes, commander.files, @@ -87,7 +81,6 @@ async function Main() { commander.input, commander.threads, commander.maxTime, - commander.maxRuns, commander.methods, commander.classes, commander.files, diff --git a/test/naked.test.ts b/test/naked.test.ts index d4bee57..50a4012 100644 --- a/test/naked.test.ts +++ b/test/naked.test.ts @@ -5,7 +5,7 @@ describe('Naked generators with simple values.', function () { this.timeout(3 * 60 * 1e3); before(async () => { - global.fastFuzzResults = await init('Naked', 30e3, 5e5); + global.fastFuzzResults = await init('Naked', 60e3); }); it('Generates single buillt-in argument for non-decorated method.', async () => { diff --git a/test/testRunner.ts b/test/testRunner.ts index 3da4f45..55aa319 100644 --- a/test/testRunner.ts +++ b/test/testRunner.ts @@ -6,8 +6,7 @@ const results: { [key: string]: any[] } = {}; export async function init( name: string, - time = 5e3, - runCount = 1e4 + time = 10e3 ): Promise { if (results[name] === undefined) { // Get rid of previous instances. @@ -16,7 +15,7 @@ export async function init( fs.unlinkSync(fileName); } - const cliResult: string = await execShellCommand(`node ./dist/src/index.js -i "./test/sut" -s "./" -d "../../dist/test/sut" -c "${name}" -t ${time} -n ${runCount} -p 2 -q true`); + const cliResult: string = await execShellCommand(`node ./dist/src/index.js -i "./test/sut" -s "./" -d "../../dist/test/sut" -c "${name}" -t ${time} -n 2 -q true`); let error: any; try { diff --git a/test/typeStuff.test.ts b/test/typeStuff.test.ts index ac8ce2f..ad7d5dd 100644 --- a/test/typeStuff.test.ts +++ b/test/typeStuff.test.ts @@ -6,7 +6,7 @@ describe('Type generators with inherited values', function () { this.timeout(3 * 60 * 1e3); before(async () => { - global.fastFuzzResults = await init('Stuff', undefined, 3e3); + global.fastFuzzResults = await init('Stuff'); }); it('Loads and uses the fuzzInstances.json file.', async () => {