Rewrite shell_flags, support new SpiderMonkey shell runtime flags (es…

…p. for wasm) and add tests (#158), r=truber

* Fix #114 - Update gczeal number testing in both funfuzz and jsfunfuzz, now supports up to 18.

* Refactor the chance function and add a test.

* Refactor randomFlagSet to random_flag_set and rename other variables into snake case.

* Refactor shellSupportsFlag, renaming it to shell_supports_flag and add a test.

* Split the test_basic_flag_sets function into two.

* Move shell_flags to double quotes.

* Fix syntax error post-move to double quotes.

* Unbreak shell_flags tests.

* Memoize the get_current_shell_path function and improve the docstring.

* Refactor the random_flag_set function, split off some into other functions and add tests.

* Remove the obsolete testRandomFlags function.

* Improve test for the chance function.

* Remove portion of test_random_flag_set function that may cause random coverage results.

* Add support for "--cache-ir-stubs=[on|off]".

* Add support for "--enable-streams".

* Add support for "--no-array-proto-values".

* Add support for "--ion-scalar-replacement=[on|off]".

* Add support for "--enable-avx" and "--no-avx".

* Add support for "--nursery-strings=[on|off]".

* Update the earliest_known_working_rev function with the latest values from shell_flags.

* Remove support for "--wasm-always-baseline" and update basic_flag_sets with more recent flag sets to test for compare_jit.

* Migrate comment to #160.

* Make sure "--ion-offthread-compile=on" is supported before using it.

* Move asmjs flags into wasm section.

* Ignore pylint error message in earliest_known_working_rev for now.

* Add support for "--no-wasm-baseline" and "--no-wasm-ion" flags.

* Improve level_two gczeal value generation in jsfunfuzz.

* Use the monkeypatch fixture from pytest instead of using an "always" parameter.

* Remove unneeded "always" parameter in functions in test_shell_flags.py as well.

* Documentation changes.

* Make the allocations_number a positive one to ensure test consistency.

* Change the mock_chance function to not be function-specific and make other doc-related additions.

src/funfuzz/autobisectjs/known_broken_earliest_working.py

@@ -86,66 +86,79 @@ def known_broken_ranges(options):  # pylint: disable=missing-param-doc,missing-r
 
 
 def earliest_known_working_rev(options, flags, skip_revs):  # pylint: disable=missing-param-doc,missing-return-doc
-    # pylint: disable=missing-return-type-doc,missing-type-doc,too-many-branches,too-complex
+    # pylint: disable=missing-return-type-doc,missing-type-doc,too-many-branches,too-complex,too-many-statements
     """Return a revset which evaluates to the first revision of the shell that compiles with |options|
     and runs jsfunfuzz successfully with |flags|."""
     assert (not sps.isMac) or (sps.macVer() >= [10, 11])  # Only support at least Mac OS X 10.11
 
-    # This code is kept in case in the future we have an earliest known working rev that requires a flag
-    # These should be in descending order, or bisection will break at earlier changesets.
-    # gczeal_value_flag = False
-    # # flags is a list of flags, and the option must exactly match.
-    # for entry in flags:
-    #     # What comes after these flags needs to be a number, so we look for the string instead.
-    #     if '--gc-zeal=' in entry:
-    #         gczeal_value_flag = True
+    cpu_count_flag = False
+    for entry in flags:  # flags is a list of flags, and the option must exactly match.
+        if "--cpu-count=" in entry:
+            cpu_count_flag = True
 
     required = []
 
+    # These should be in descending order, or bisection will break at earlier changesets.
+    if "--nursery-strings=on" in flags or "--nursery-strings=off" in flags:
+        required.append("321c29f48508")  # m-c 406115 Fx60, 1st w/--nursery-strings=on, see bug 903519
+    if "--no-array-proto-values" in flags:
+        required.append("e1ca344ca6b5")  # m-c 403011 Fx60, 1st w/--no-array-proto-values, see bug 1420101
+    if "--spectre-mitigations=on" in flags or "--spectre-mitigations=off" in flags:
+        required.append("a98f615965d7")  # m-c 399868 Fx59, 1st w/--spectre-mitigations=on, see bug 1430053
+    if "--test-wasm-await-tier2" in flags:
+        required.append("b1dc87a94262")  # m-c 387188 Fx58, 1st w/--test-wasm-await-tier2, see bug 1388785
     if sps.isMac:
-        required.append('e2ecf684f49e')  # m-c 383101 Fx58, 1st w/ successful Xcode 9 builds, see bug 1366564
+        required.append("e2ecf684f49e")  # m-c 383101 Fx58, 1st w/ successful Xcode 9 builds, see bug 1366564
+    if cpu_count_flag:
+        required.append("1b55231e6628")  # m-c 380023 Fx57, 1st w/--cpu-count=<NUM>, see bug 1206770
+    if "--no-wasm-ion" in flags:
+        required.append("158b333a0a89")  # m-c 375650 Fx57, 1st w/--no-wasm-ion, see bug 1277562
+    if "--no-wasm-baseline" in flags:
+        required.append("9ea44ef0c07c")  # m-c 375639 Fx57, 1st w/--no-wasm-baseline, see bug 1277562
+    if "--enable-streams" in flags:
+        required.append("64bbc26920aa")  # m-c 371894 Fx56, 1st w/--enable-streams, see bug 1272697
     if sps.isWin:
-        required.append('530f7bd28399')  # m-c 369571 Fx56, 1st w/ successful MSVC 2017 builds, see bug 1356493
+        required.append("530f7bd28399")  # m-c 369571 Fx56, 1st w/ successful MSVC 2017 builds, see bug 1356493
     # Note that the sed version check only works with GNU sed, not BSD sed found in macOS.
     if sps.isLinux and StrictVersion(subprocess.check_output(["sed", "--version"]).split()[3]) >= StrictVersion("4.3"):
-        required.append('ebcbf47a83e7')  # m-c 328765 Fx53, 1st w/ working builds using sed 4.3+ found on Ubuntu 17.04+
+        required.append("ebcbf47a83e7")  # m-c 328765 Fx53, 1st w/ working builds using sed 4.3+ found on Ubuntu 17.04+
     if options.disableProfiling:
-        required.append('800a887c705e')  # m-c 324836 Fx53, 1st w/ --disable-profiling, see bug 1321065
-    if "--wasm-always-baseline" in flags:
-        required.append('893294e2a387')  # m-c 301769 Fx50, 1st w/--wasm-always-baseline, see bug 1232205
-    if '--ion-aa=flow-sensitive' in flags or '--ion-aa=flow-insensitive' in flags:
+        required.append("800a887c705e")  # m-c 324836 Fx53, 1st w/ --disable-profiling, see bug 1321065
+    if "--no-wasm" in flags:
+        required.append("e9b561d60697")  # m-c 321230 Fx52, 1st w/--no-wasm, see bug 1313180
+    if "--cache-ir-stubs=on" in flags:
+        required.append("1c5b92144e1e")  # m-c 308931 Fx51, 1st w/--cache-ir-stubs=on, see bug 1292659
+    if "--ion-aa=flow-sensitive" in flags or "--ion-aa=flow-insensitive" in flags:
         # m-c 295435 Fx49, 1st w/--ion-aa=[flow-sensitive|flow-insensitive], see bug 1255008
-        required.append('c0c1d923c292')
-    if "--ion-pgo=on" in flags:
-        required.append('b0a0ff5fa705')  # m-c 272274 Fx45, 1st w/--ion-pgo=on, see bug 1209515
+        required.append("c0c1d923c292")
+    if "--ion-pgo=on" in flags or "--ion-pgo=off" in flags:
+        required.append("b0a0ff5fa705")  # m-c 272274 Fx45, 1st w/--ion-pgo=on, see bug 1209515
     if options.buildWithAsan:
-        required.append('d4e0e0e5d26d')  # m-c 268534 Fx44, 1st w/ reliable ASan builds w/ ICU, see bug 1214464
-    if "--ion-sincos=on" in flags:
-        required.append('3dec2b935295')  # m-c 262544 Fx43, 1st w/--ion-sincos=on, see bug 984018
-    if "--ion-instruction-reordering=on" in flags:
-        required.append('59d2f2e62420')  # m-c 259672 Fx43, 1st w/--ion-instruction-reordering=on, see bug 1195545
-    if "--ion-shared-stubs=on" in flags:
-        required.append('3655d19ce241')  # m-c 257573 Fx43, 1st w/--ion-shared-stubs=on, see bug 1168756
+        required.append("d4e0e0e5d26d")  # m-c 268534 Fx44, 1st w/ reliable ASan builds w/ ICU, see bug 1214464
+    if "--ion-sincos=on" in flags or "--ion-sincos=off" in flags:
+        required.append("3dec2b935295")  # m-c 262544 Fx43, 1st w/--ion-sincos=on, see bug 984018
+    if "--ion-instruction-reordering=on" in flags or "--ion-instruction-reordering=off" in flags:
+        required.append("59d2f2e62420")  # m-c 259672 Fx43, 1st w/--ion-instruction-reordering=on, see bug 1195545
+    if "--ion-shared-stubs=on" in flags or "--ion-shared-stubs=off" in flags:
+        required.append("3655d19ce241")  # m-c 257573 Fx43, 1st w/--ion-shared-stubs=on, see bug 1168756
     if options.enableSimulatorArm32 or options.enableSimulatorArm64:
-        # For ARM64: This should get updated whenever ARM64 builds are stable, probably ~end-June 2015
-        # To bisect manually slightly further, use "-s dc4b163f7db7 -e f50a771d7d1b" and:
-        # Also comment out from:
-        # https://github.com/MozillaSecurity/funfuzz/blob/bbc5d5c74d/autobisect-js/autoBisect.py#L176
-        # (line 176) to line 180.
-        required.append('25e99bc12482')  # m-c 249239 Fx41, 1st w/--enable-simulator=[arm|arm64|mips], see bug 1173992
+        # For ARM64: This should get updated whenever ARM64 builds are stable
+        required.append("25e99bc12482")  # m-c 249239 Fx41, 1st w/--enable-simulator=[arm|arm64|mips], see bug 1173992
     if "--ion-regalloc=testbed" in flags:
-        required.append('47e92bae09fd')  # m-c 248962 Fx41, 1st w/--ion-regalloc=testbed, see bug 1170840
-    if '--non-writable-jitcode' in flags:
-        required.append('b46d6692fe50')  # m-c 248578 Fx41, 1st w/--non-writable-jitcode, see bug 977805
-    if '--no-unboxed-objects' in flags:
-        required.append('322487136b28')  # m-c 244297 Fx41, 1st w/--no-unboxed-objects, see bug 1162199
-    if '--unboxed-arrays' in flags:
-        required.append('020c6a559e3a')  # m-c 242167 Fx40, 1st w/--unboxed-arrays, see bug 1146597
-    if '--ion-extra-checks' in flags:
-        required.append('cdf93416b39a')  # m-c 234228 Fx39, 1st w/--ion-extra-checks, see bug 1139152
-    if '--no-cgc' in flags:
-        required.append('b63d7e80709a')  # m-c 227705 Fx38, 1st w/--no-cgc, see bug 1126769 and see bug 1129233
-    required.append('bcacb5692ad9')  # m-c 222786 Fx37, 1st w/ successful GCC 5.2.x builds on Ubuntu 15.10 onwards
+        required.append("47e92bae09fd")  # m-c 248962 Fx41, 1st w/--ion-regalloc=testbed, see bug 1170840
+    if "--non-writable-jitcode" in flags:
+        required.append("b46d6692fe50")  # m-c 248578 Fx41, 1st w/--non-writable-jitcode, see bug 977805
+    if "--execute=setJitCompilerOption(\"ion.forceinlineCaches\",1)" in flags:
+        required.append("ea9608e33abe")  # m-c 247709 Fx41, 1st w/ion.forceinlineCaches, see bug 923717
+    if "--no-unboxed-objects" in flags:
+        required.append("322487136b28")  # m-c 244297 Fx41, 1st w/--no-unboxed-objects, see bug 1162199
+    if "--ion-extra-checks" in flags:
+        required.append("cdf93416b39a")  # m-c 234228 Fx39, 1st w/--ion-extra-checks, see bug 1139152
+    if "--no-cgc" in flags:
+        required.append("b63d7e80709a")  # m-c 227705 Fx38, 1st w/--no-cgc, see bug 1126769 and see bug 1129233
+    if "--enable-avx" in flags or "--no-avx" in flags:
+        required.append("5e6e959f0043")  # m-c 223959 Fx38, 1st w/--enable-avx, see bug 1118235
+    required.append("bcacb5692ad9")  # m-c 222786 Fx37, 1st w/ successful GCC 5.2.x builds on Ubuntu 15.10 onwards
 
     return "first((" + common_descendants(required) + ") - (" + skip_revs + "))"
 

src/funfuzz/js/loop.py

@@ -150,7 +150,7 @@ def many_timed_runs(targetTime, wtmpDir, args, collector):  # pylint: disable=in
         js_interesting_args.append(options.knownPath)
         js_interesting_args.append(options.jsEngine)
         if options.randomFlags:
-            engineFlags = shell_flags.randomFlagSet(options.jsEngine)  # pylint: disable=invalid-name
+            engineFlags = shell_flags.random_flag_set(options.jsEngine)  # pylint: disable=invalid-name
             js_interesting_args.extend(engineFlags)
         js_interesting_args.extend(['-e', 'maxRunTime=' + str(options.timeout * (1000 // 2))])
         js_interesting_args.extend(['-f', fuzzjs])

src/funfuzz/js/shared/testing-functions.js

@@ -18,8 +18,22 @@ function fuzzTestingFunctionsCtor(browser, fGlobal, fObject)
 
   function enableGCZeal()
   {
-    var level = rnd(17);
-    if (browser && level == 9) level = 0; // bug 815241
+    // As of m-c 405243:02aa9c921aed 2018-02-26
+    // https://hg.mozilla.org/mozilla-central/file/02aa9c921aed/js/src/gc/GC.cpp#l1000
+    max_level = 18;
+    max_level++;  // rnd function starts from zero
+    var level = rnd(max_level);
+
+    // Generate the second level.
+    // ref https://hg.mozilla.org/mozilla-central/file/02aa9c921aed/js/src/gc/GC.cpp#l1001
+    var level_two = rnd(max_level - 3); // 3 levels disabled below
+    if (level_two >= 3) level_two++; // level_two 3 does not exist, so repurpose it
+    if (level_two >= 5) level_two++; // level_two 5 does not exist, so repurpose it
+    if (level_two >= 6) level_two++; // level_two 6 does not exist, so repurpose it
+
+    if (level == 3) level = 0; // level 3 does not exist, so repurpose it
+    if (level == 5) level = '"2;' + level_two + '"'; // level 5 does not exist, so repurpose it
+    if (level == 6) level = '"2;' + level_two + '"'; // level 6 does not exist, so repurpose it
     var period = numberOfAllocs();
     return prefix + "gczeal" + "(" + level + ", " + period + ");";
   }