diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 392451b79e6855..ebe72fd9fdee60 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -641,45 +641,18 @@ jobs: run: | "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed - # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: - name: CIFuzz - runs-on: ubuntu-latest - timeout-minutes: 60 needs: build-context if: needs.build-context.outputs.run-ci-fuzz == 'true' - permissions: - security-events: write - strategy: - fail-fast: false - matrix: - sanitizer: [address, undefined, memory] - steps: - - name: Build fuzzers (${{ matrix.sanitizer }}) - id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master - with: - oss-fuzz-project-name: cpython3 - sanitizer: ${{ matrix.sanitizer }} - - name: Run fuzzers (${{ matrix.sanitizer }}) - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master - with: - fuzz-seconds: 600 - oss-fuzz-project-name: cpython3 - output-sarif: true - sanitizer: ${{ matrix.sanitizer }} - - name: Upload crash - if: failure() && steps.build.outcome == 'success' - uses: actions/upload-artifact@v6 - with: - name: ${{ matrix.sanitizer }}-artifacts - path: ./out/artifacts - - name: Upload SARIF - if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v4 - with: - sarif_file: cifuzz-sarif/results.sarif - checkout_path: cifuzz-sarif + uses: ./.github/workflows/reusable-cifuzz.yml + with: + oss-fuzz-project-name: cpython3 + cifuzz-stdlib: + needs: build-context + if: needs.build-context.outputs.run-ci-fuzz-stdlib == 'true' + uses: ./.github/workflows/reusable-cifuzz.yml + with: + oss-fuzz-project-name: python3-libraries all-required-green: # This job does nothing and is only used for the branch protection name: All required checks pass @@ -704,6 +677,7 @@ jobs: - build-san - cross-build-linux - cifuzz + - cifuzz-stdlib if: always() steps: @@ -735,6 +709,7 @@ jobs: }} ${{ !fromJSON(needs.build-context.outputs.run-windows-tests) && 'build-windows,' || '' }} ${{ !fromJSON(needs.build-context.outputs.run-ci-fuzz) && 'cifuzz,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-ci-fuzz-stdlib) && 'cifuzz-stdlib,' || '' }} ${{ !fromJSON(needs.build-context.outputs.run-macos) && 'build-macos,' || '' }} ${{ !fromJSON(needs.build-context.outputs.run-ubuntu) diff --git a/.github/workflows/reusable-cifuzz.yml b/.github/workflows/reusable-cifuzz.yml new file mode 100644 index 00000000000000..4124e0c5fb8aaa --- /dev/null +++ b/.github/workflows/reusable-cifuzz.yml @@ -0,0 +1,50 @@ +# CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ +name: Reusable CIFuzz + +on: + workflow_call: + inputs: + oss-fuzz-project-name: + description: OSS-Fuzz project name + required: true + type: string + +permissions: + contents: read + security-events: write + +jobs: + cifuzz: + name: CIFuzz + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + sanitizer: [address, undefined, memory] + steps: + - name: Build fuzzers (${{ matrix.sanitizer }}) + id: build + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + with: + oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }} + sanitizer: ${{ matrix.sanitizer }} + - name: Run fuzzers (${{ matrix.sanitizer }}) + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + with: + fuzz-seconds: 600 + oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }} + output-sarif: true + sanitizer: ${{ matrix.sanitizer }} + - name: Upload crash + if: failure() && steps.build.outcome == 'success' + uses: actions/upload-artifact@v6 + with: + name: ${{ matrix.sanitizer }}-artifacts + path: ./out/artifacts + - name: Upload SARIF + if: always() && steps.build.outcome == 'success' + uses: github/codeql-action/upload-sarif@v4 + with: + sarif_file: cifuzz-sarif/results.sarif + checkout_path: cifuzz-sarif diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index aa2ee275a57fa9..45a2ff3f09551d 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -21,7 +21,10 @@ on: # yamllint disable-line rule:truthy description: Whether to run the Android tests value: ${{ jobs.compute-changes.outputs.run-android }} # bool run-ci-fuzz: - description: Whether to run the CIFuzz job + description: Whether to run the CIFuzz job for 'cpython' fuzzer + value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }} # bool + run-ci-fuzz-stdlib: + description: Whether to run the CIFuzz job for 'python3-libraries' fuzzer value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }} # bool run-docs: description: Whether to build the docs @@ -56,6 +59,7 @@ jobs: outputs: run-android: ${{ steps.changes.outputs.run-android }} run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }} + run-ci-fuzz-stdlib: ${{ steps.changes.outputs.run-ci-fuzz-stdlib }} run-docs: ${{ steps.changes.outputs.run-docs }} run-ios: ${{ steps.changes.outputs.run-ios }} run-macos: ${{ steps.changes.outputs.run-macos }} diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index c491f06e9968fe..7e2ae5a32b3b44 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -11,7 +11,7 @@ import os import subprocess -from dataclasses import dataclass +from dataclasses import dataclass, fields from pathlib import Path TYPE_CHECKING = False @@ -52,11 +52,58 @@ MACOS_DIRS = frozenset({"Mac"}) WASI_DIRS = frozenset({Path("Tools", "wasm")}) +LIBRARY_FUZZER_PATHS = frozenset({ + # All C/CPP fuzzers. + Path("configure"), + Path(".github/workflows/reusable-cifuzz.yml"), + # ast + Path("Lib/ast.py"), + Path("Python/ast.c"), + # configparser + Path("Lib/configparser.py"), + # csv + Path("Lib/csv.py"), + Path("Modules/_csv.c"), + # decode + Path("Lib/encodings/"), + Path("Modules/_codecsmodule.c"), + Path("Modules/cjkcodecs/"), + Path("Modules/unicodedata*"), + # difflib + Path("Lib/difflib.py"), + # email + Path("Lib/email/"), + # html + Path("Lib/html/"), + Path("Lib/_markupbase.py"), + # http.client + Path("Lib/http/client.py"), + # json + Path("Lib/json/"), + Path("Modules/_json.c"), + # plist + Path("Lib/plistlib.py"), + # re + Path("Lib/re/"), + Path("Modules/_sre/"), + # tarfile + Path("Lib/tarfile.py"), + # tomllib + Path("Modules/tomllib/"), + # xml + Path("Lib/xml/"), + Path("Lib/_markupbase.py"), + Path("Modules/expat/"), + # zipfile + Path("Lib/zipfile/"), +}) + @dataclass(kw_only=True, slots=True) class Outputs: run_android: bool = False run_ci_fuzz: bool = False + run_ci_fuzz_stdlib: bool = False run_docs: bool = False run_ios: bool = False run_macos: bool = False @@ -96,6 +143,11 @@ def compute_changes() -> None: else: print("Branch too old for CIFuzz tests; or no C files were changed") + if outputs.run_ci_fuzz_stdlib: + print("Run CIFuzz tests for libraries") + else: + print("Branch too old for CIFuzz tests; or no library files were changed") + if outputs.run_docs: print("Build documentation") @@ -146,9 +198,18 @@ def get_file_platform(file: Path) -> str | None: return None +def is_fuzzable_library_file(file: Path) -> bool: + return any( + (file.is_relative_to(needs_fuzz) and needs_fuzz.is_dir()) + or (file == needs_fuzz and file.is_file()) + for needs_fuzz in LIBRARY_FUZZER_PATHS + ) + + def process_changed_files(changed_files: Set[Path]) -> Outputs: run_tests = False run_ci_fuzz = False + run_ci_fuzz_stdlib = False run_docs = False run_windows_tests = False run_windows_msi = False @@ -162,8 +223,8 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: doc_file = file.suffix in SUFFIXES_DOCUMENTATION or doc_or_misc if file.parent == GITHUB_WORKFLOWS_PATH: - if file.name == "build.yml": - run_tests = run_ci_fuzz = True + if file.name == "build.yml" or file.name == "reusable-cifuzz.yml": + run_tests = run_ci_fuzz = run_ci_fuzz_stdlib = True has_platform_specific_change = False if file.name == "reusable-docs.yml": run_docs = True @@ -194,6 +255,8 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: ("Modules", "_xxtestfuzz"), }: run_ci_fuzz = True + if not run_ci_fuzz_stdlib and is_fuzzable_library_file(file): + run_ci_fuzz_stdlib = True # Check for changed documentation-related files if doc_file: @@ -227,6 +290,7 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: return Outputs( run_android=run_android, run_ci_fuzz=run_ci_fuzz, + run_ci_fuzz_stdlib=run_ci_fuzz_stdlib, run_docs=run_docs, run_ios=run_ios, run_macos=run_macos, @@ -261,16 +325,10 @@ def write_github_output(outputs: Outputs) -> None: return with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: - f.write(f"run-android={bool_lower(outputs.run_android)}\n") - f.write(f"run-ci-fuzz={bool_lower(outputs.run_ci_fuzz)}\n") - f.write(f"run-docs={bool_lower(outputs.run_docs)}\n") - f.write(f"run-ios={bool_lower(outputs.run_ios)}\n") - f.write(f"run-macos={bool_lower(outputs.run_macos)}\n") - f.write(f"run-tests={bool_lower(outputs.run_tests)}\n") - f.write(f"run-ubuntu={bool_lower(outputs.run_ubuntu)}\n") - f.write(f"run-wasi={bool_lower(outputs.run_wasi)}\n") - f.write(f"run-windows-msi={bool_lower(outputs.run_windows_msi)}\n") - f.write(f"run-windows-tests={bool_lower(outputs.run_windows_tests)}\n") + for field in fields(outputs): + name = field.name.replace("_", "-") + val = bool_lower(getattr(outputs, field.name)) + f.write(f"{name}={val}\n") def bool_lower(value: bool, /) -> str: