Benchmark functions & relate # of runs to max time

README.md

@@ -61,7 +61,6 @@ async Main () {
     projectFolder,
     threads,          // [Optional] Default = OS defined. 0 for in-process.
     maxTime,          // [Optional] Default = 10s.
-    maxRuns,          // [Optional] Default = 100e3.
     methods, classes, // [Optional] Default = all.
     files,            // [Optional] Default = all.
     source, dist,     // [Optional] Default = 'src' and 'dist'.
@@ -84,8 +83,7 @@ fast-fuzz
   -V, --version                 Output the version number
   -p, --threads <count>         The number of parallel threads.
                                   Default = OS defined. 0 for in-process.
-  -t, --maxTime <milliseconds>  The maximum time(ms) per function. Default = 10s.
-  -n, --maxRuns <milliseconds>  The maximum count of runs per function. Default = 100e3.
+  -t, --maxTime <milliseconds>  The maximum time(ms) per function. Default = 10 min.
   -m, --methods <RegExp>        A Regex expression to filter the methods to test.
   -c, --classes <RegExp>        A Regex expression to filter the classes to test.
   -f, --files <RegExp>          A Regex expression to filter the files to test.
@@ -187,9 +185,7 @@ and can thus be reused in other code.
 - All unrelated literals should be put at the beginning of the file.
 - Any methods that are not exported or within a class should not contain literals as these will be picked up by the earlier fuzzed method in the file. Another option is to put ineligible methods before the fuzzed ones.
 
-## TODO Priorities
-
-- Benchmarking of target functions to determine the best run time and number of tests.
+## TODO
 
 - Redundant runner for the args-results pairs.
 - Stuffing for nested type args by registration of generators and collection of values upon successful detection.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "fast-fuzz",
-  "version": "5.0.2",
+  "version": "5.0.3",
   "description": "Test case fuzzer with branch coverage guidance.",
   "main": "./dist/src/fast-fuzz.js",
   "types": "./dist/src/fast-fuzz.d.ts",

src/fuzz/fuzz.ts

@@ -43,10 +43,10 @@ let instances: {
 
 /**
  * Inits the local code analysis and type stuffing.
- * @param folder 
- * @param [src] 
- * @param [dist] 
- * @param [instances] 
+ * @param folder @type {string} 
+ * @param [src] @type {string} 
+ * @param [dist] @type {string} 
+ * @param [instances] @type {any} 
  */
 export async function init(
   folder: string,
@@ -63,9 +63,9 @@ export async function init(
 
 /**
  * Inits the local code analysis.
- * @param folder 
- * @param [src] 
- * @param [dist] 
+ * @param folder @type {string} 
+ * @param [src] @type {string} 
+ * @param [dist] @type {string} 
  */
 async function initLocal(
   folder: string,
@@ -108,15 +108,16 @@ function checkInit(): void {
 
 /**
  * Counts the number of methods.
- * @param [methodPattern]
- * @param [classPattern]
+ * @param [methodPattern] @type {RegExp} 
+ * @param [classPattern] @type {RegExp} 
+ * @param [filePattern] @type {RegExp} 
  * @returns count of methods.
  */
 export async function count(
   methodPattern?: string,
   classPattern?: string,
   filePattern?: string
- ): Promise<number> {
+): Promise<number> {
   checkInit();
 
   let methodCount = 0;
@@ -143,16 +144,14 @@ export async function count(
 
 /**
  * Fuzz the TS folder.
- * @param [maxTime] 
- * @param [maxRuns] 
- * @param [methodPattern]
- * @param [classPattern]
- * @param [filePattern]
- * @param [resultsOut] 
+ * @param [maxTime] @type {number} 
+ * @param [methodPattern] @type {RegExp} 
+ * @param [classPattern] @type {RegExp} 
+ * @param [filePattern] @type {RegExp} 
+ * @param [resultsOut] @type {Results[]} 
  */
 export async function fuzz(
   maxTime = 1e4,
-  maxRuns = 1e5,
   methodPattern?: string,
   classPattern?: string,
   filePattern?: string,
@@ -184,6 +183,7 @@ export async function fuzz(
       if (method.name === '__constructor') { continue; }
       /* #endregion */
 
+      /* #region  Check the resume state and skip those methods. */
       if (count !== 0 && currentCount >= count) {
         break;
       }
@@ -194,33 +194,24 @@ export async function fuzz(
 
       currentIndex++;
       currentCount++;
+      /* #endregion */
 
       // Set the generators to reset with the new literals.
-      Globals.methodCount++;
       Globals.literals = method.literals;
 
+      // Benchmark for number of runs in the specified time.
+      const maxRuns = await getMaxRuns(method, file, maxTime);
+
       // Run the appropiate static & async method
-      let fuzzResults: Result[] = [];
-      if (method.isStatic || method.className === undefined) {
-        if (method.isAsync) {
-          fuzzResults = await fuzzStaticAsync(file, method, maxTime, maxRuns, fuzzResults);
-        } else {
-          fuzzResults = fuzzStatic(file, method, maxTime, maxRuns, fuzzResults);
-        }
-      } else {
-        if (method.isAsync) {
-          fuzzResults = await fuzzMethodAsync(file, method, maxTime, maxRuns, fuzzResults);
-        } else {
-          fuzzResults = fuzzMethod(file, method, maxTime, maxRuns, fuzzResults);
-        }
-      }
+      let fuzzResults: Result[] = await fuzzAnyMethod(method, file, maxTime, maxRuns);
 
       // Output the method results.
       resultsOut.push({
         name: method.name,
         className: method.className,
         namespaces: method.namespaces,
         file,
+        avgSpeed: maxRuns / maxTime,
         results: fuzzResults
       });
 
@@ -233,14 +224,68 @@ export async function fuzz(
   return resultsOut;
 }
 
+
+/**
+ * Gets max runs from 1000, in 10% of the time.
+ * @param method @type {ModuleMethod} 
+ * @param file @type {string} 
+ * @param maxTime @type {number} 
+ * @returns maxRuns @type {number} 
+ */
+async function getMaxRuns(
+  method: ModuleMethod,
+  file: string,
+  maxTime: number
+): Promise<number> {
+  const numRuns = 1e3;
+  const start = Date.now();
+
+  await fuzzAnyMethod(method, file, maxTime / 10, numRuns);
+  const result = Math.floor(maxTime / ((Date.now() - start) / numRuns));
+
+  return Math.max(numRuns, result);
+}
+
+/**
+ * Fuzzes any method.
+ * @param method @type {ModuleMethod} 
+ * @param file @type {string} 
+ * @param maxTime @type {number} 
+ * @param maxRuns @type {number} 
+ * @returns @type {Result[]} 
+ */
+async function fuzzAnyMethod(
+  method: ModuleMethod,
+  file: string,
+  maxTime: number,
+  maxRuns: number
+): Promise<Result[]> {
+  Globals.methodCount++;
+  let fuzzResults: Result[] = [];
+  if (method.isStatic || method.className === undefined) {
+    if (method.isAsync) {
+      fuzzResults = await fuzzStaticAsync(file, method, maxTime, maxRuns, fuzzResults);
+    } else {
+      fuzzResults = fuzzStatic(file, method, maxTime, maxRuns, fuzzResults);
+    }
+  } else {
+    if (method.isAsync) {
+      fuzzResults = await fuzzMethodAsync(file, method, maxTime, maxRuns, fuzzResults);
+    } else {
+      fuzzResults = fuzzMethod(file, method, maxTime, maxRuns, fuzzResults);
+    }
+  }
+  return fuzzResults;
+}
+
 /**
  * Fuzz static methods.
- * @param filePath 
- * @param method 
- * @param [maxTime] 
- * @param [maxRuns] 
- * @param resultsOut 
- * @returns static 
+ * @param filePath @type {string} 
+ * @param method @type {ModuleMethod} 
+ * @param [maxTime] @type {number} 
+ * @param [maxRuns] @type {number} 
+ * @param resultsOut @type {Result[]} 
+ * @returns results @type {Result[]} 
  */
 function fuzzStatic(
   filePath: string,
@@ -297,12 +342,12 @@ function fuzzStatic(
 
 /**
  * Fuzz static methods async.
- * @param filePath 
- * @param method 
- * @param [maxTime] 
- * @param [maxRuns] 
- * @param resultsOut 
- * @returns static async 
+ * @param filePath @type {string} 
+ * @param method @type {ModuleMethod} 
+ * @param [maxTime] @type {number} 
+ * @param [maxRuns] @type {number} 
+ * @param resultsOut @type {Result[]} 
+ * @returns results @type {Result[]} 
  */
 async function fuzzStaticAsync(
   filePath: string,
@@ -359,12 +404,12 @@ async function fuzzStaticAsync(
 
 /**
  * Fuzz methods.
- * @param filePath 
- * @param method 
- * @param [maxTime] 
- * @param [maxRuns] 
- * @param resultsOut 
- * @returns method 
+ * @param filePath @type {string} 
+ * @param method @type {ModuleMethod} 
+ * @param [maxTime] @type {number} 
+ * @param [maxRuns] @type {number} 
+ * @param resultsOut @type {Result[]} 
+ * @returns results @type {Result[]} 
  */
 function fuzzMethod(
   filePath: string,
@@ -436,12 +481,12 @@ function fuzzMethod(
 
 /**
  * Fuzz methods async.
- * @param filePath 
- * @param method 
- * @param [maxTime] 
- * @param [maxRuns] 
- * @param resultsOut 
- * @returns method async 
+ * @param filePath @type {string} 
+ * @param method @type {ModuleMethod} 
+ * @param [maxTime] @type {number} 
+ * @param [maxRuns] @type {number} 
+ * @param resultsOut @type {Result[]} 
+ * @returns results @type {Result[]} 
  */
 async function fuzzMethodAsync(
   filePath: string,
@@ -513,9 +558,9 @@ async function fuzzMethodAsync(
 
 /**
  * Gets args.
- * @param method 
- * @param generator 
- * @returns args 
+ * @param method @type {ModuleMethod} 
+ * @param generator @type {GeneratorArg} 
+ * @returns args @type {any[]} 
  */
 function getArgs(method: ModuleMethod, generator: GeneratorArg): any[] {
   // Set the method to generate new arguments.
@@ -535,8 +580,8 @@ function getArgs(method: ModuleMethod, generator: GeneratorArg): any[] {
 
 /**
  * Loads instances after every method fuzz.
- * @param instances 
- * @param instancesOut 
+ * @param instances @type {any[]}
+ * @param instancesOut @type {any}
  */
 function loadInstances(
   instances: {
@@ -598,8 +643,8 @@ function loadInstances(
 
 /**
  * Gets instances.
- * @returns instances 
+ * @returns instances @type {any} 
  */
 export async function getInstances(): Promise<any> {
   return Globals.instances;
-}
+}

src/fuzz/fuzzAsync.ts

@@ -57,7 +57,9 @@ export async function fuzzAsync(
 
     // Vary the number of runs based on target area.
     let runCount = 0;
+    let lastIndex = 0;
     const maxRunsMode: number = maxRunsModes.pop();
+    const maxRunsFailFast = Math.floor(0.682 * maxRunsMode);
     const maxRunsCheck: number = Math.pow(10, Math.max(1, Math.floor(Math.log10(maxRunsMode)) - 1));
     const start: number = Date.now();
     const runTime: number = maxTime * maxRunsMode / (MODE_SCALE * maxRuns);
@@ -74,7 +76,10 @@ export async function fuzzAsync(
     // eslint-disable-next-line
     while (true) {
       // Check the running stats for termination.
-      if (runCount % maxRunsCheck == 0) { isExpired = (Date.now() - start) > runTime; }
+      if (runCount % maxRunsCheck == 0) {
+        isExpired = (Date.now() - start) > runTime;
+        isExpired = isExpired || ((runCount - lastIndex) > maxRunsFailFast);
+      }
       if (isExpired || runCount > maxRunsMode) {
         break;
       }
@@ -123,6 +128,7 @@ export async function fuzzAsync(
         cleanupError(result);
       }
 
+      // Update results.
       resultsOut.push(new Result({
         id: resultCount++,
         modeId: mode,
@@ -135,12 +141,12 @@ export async function fuzzAsync(
         )
       }));
 
+      // Update accounting vars.
+      lastIndex = runCount;
       persistInstances();
     }
   }
 
-  resetCoverage(filePath);
-
   // Report the generated tests.
   return resultsOut;
 }

src/fuzz/fuzzCaller.ts

@@ -51,7 +51,6 @@ async function init(
  * Fuzz the TS folder.
  * @param folder 
  * @param [maxTime] 
- * @param [maxRuns] 
  * @param [methodPattern]
  * @param [classPattern]
  * @param [src]
@@ -64,7 +63,6 @@ export async function fuzz(
   folder: string,
   threads?: number,
   maxTime = 1e4,
-  maxRuns = 1e5,
   methodPattern?: string,
   classPattern?: string,
   filePattern?: string,
@@ -104,7 +102,6 @@ export async function fuzz(
   if (threads === 0) {
     await fuzzSingle(
       maxTime,
-      maxRuns,
       methodPattern,
       classPattern,
       filePattern,
@@ -115,7 +112,6 @@ export async function fuzz(
   } else {
     await fuzzRunner.fuzz(
       maxTime,
-      maxRuns,
       methodPattern,
       classPattern,
       filePattern,