Merge branch 'main' into mypy_1.8.0

.asf.yaml

@@ -20,13 +20,33 @@
 github:
   description: "Apache Airflow - A platform to programmatically author, schedule, and monitor workflows"
   homepage: https://airflow.apache.org/
+  # Social media preview image is not supported by Github API/asf.yaml, need to be uploaded
+  # manually in Github repository --> Settings --> click "Edit" in "Social preview"
+  # See also:
+  # https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/customizing-your-repositorys-social-media-preview
+  # social_media_preview: docs/apache-airflow/img/logos/github_repository_social_image.png
   labels:
+    # Note that Github only supports <=20 labels/topics per repo! Pipeline will fail if you add more.
     - airflow
     - apache
     - apache-airflow
+    - automation
+    - dag
+    - data-engineering
+    - data-integration
+    - data-orchestrator
+    - data-pipelines
+    - data-science
+    - elt
+    - etl
+    - machine-learning
+    - mlops
+    - orchestration
     - python
     - scheduler
     - workflow
+    - workflow-engine
+    - workflow-orchestration
   features:
     # Enable issues management
     issues: true
@@ -44,44 +64,56 @@ github:
     main:
       required_pull_request_reviews:
         required_approving_review_count: 1
+      required_linear_history: true
+      required_conversation_resolution: true
     v1-10-stable:
       required_pull_request_reviews:
         required_approving_review_count: 1
     v2-0-stable:
       required_pull_request_reviews:
         required_approving_review_count: 1
+      required_linear_history: true
     v2-1-stable:
       required_pull_request_reviews:
         required_approving_review_count: 1
+      required_linear_history: true
     v2-2-stable:
       required_pull_request_reviews:
         required_approving_review_count: 1
+      required_linear_history: true
     v2-3-stable:
       required_pull_request_reviews:
         required_approving_review_count: 1
+      required_linear_history: true
     v2-4-stable:
       required_pull_request_reviews:
         required_approving_review_count: 1
+      required_linear_history: true
     v2-5-stable:
       required_pull_request_reviews:
         required_approving_review_count: 1
+      required_linear_history: true
     v2-6-stable:
       required_pull_request_reviews:
         required_approving_review_count: 1
+      required_linear_history: true
     v2-7-stable:
       required_pull_request_reviews:
         required_approving_review_count: 1
+      required_linear_history: true
     v2-8-stable:
       required_pull_request_reviews:
         required_approving_review_count: 1
-
+      required_linear_history: true
   collaborators:
-    - mhenc
-    - utkarsharma2
     - Lee-W
-    - sunank200
-    - nathadfield
     - RNHTTR
+    - aritra24
+    - dirrao
+    - mhenc
+    - nathadfield
+    - sunank200
+    - vatsrahul1001
 
 notifications:
   jobs: [email protected]

.github/boring-cyborg.yml

@@ -118,11 +118,6 @@ labelPRBasedOnFilePath:
     - tests/providers/apache/spark/**/*
     - tests/system/providers/apache/spark/**/*
 
-  provider:apache-sqoop:
-    - airflow/providers/apache/sqoop/**/*
-    - docs/apache-airflow-providers-apache-sqoop/**/*
-    - tests/providers/apache/sqoop/**/*
-
   provider:apprise:
     - airflow/providers/apprise/**/*
     - docs/apache-airflow-providers-apprise/**/*
@@ -228,6 +223,11 @@ labelPRBasedOnFilePath:
     - docs/apache-airflow-providers-exasol/**/*
     - tests/providers/exasol/**/*
 
+  provider:fab:
+    - airflow/providers/fab/**/*
+    - docs/apache-airflow-providers-fab/**/*
+    - tests/providers/fab/**/*
+
   provider:facebook:
     - airflow/providers/facebook/**/*
     - docs/apache-airflow-providers-facebook/**/*
@@ -391,12 +391,6 @@ labelPRBasedOnFilePath:
     - tests/providers/pinecone/**/*
     - tests/system/providers/pinecone/**/*
 
-  provider:plexus:
-    - airflow/providers/plexus/**/*
-    - docs/apache-airflow-providers-plexus/**/*
-    - tests/providers/plexus/**/*
-    - tests/system/providers/plexus/**/*
-
   provider:postgres:
     - airflow/providers/postgres/**/*
     - docs/apache-airflow-providers-postgres/**/*

.github/workflows/ci.yml

@@ -86,17 +86,14 @@ jobs:
       postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }}
       default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }}
       mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }}
-      mssql-versions: ${{ steps.selective-checks.outputs.mssql-versions }}
       default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }}
       default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }}
       default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }}
       full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }}
       parallel-test-types-list-as-string: >-
         ${{ steps.selective-checks.outputs.parallel-test-types-list-as-string }}
-      mssql-parallelism: ${{ steps.selective-checks.outputs.mssql-parallelism }}
       postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }}
       mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }}
-      mssql-exclude: ${{ steps.selective-checks.outputs.mssql-exclude }}
       sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }}
       skip-provider-tests: ${{ steps.selective-checks.outputs.skip-provider-tests }}
       run-tests: ${{ steps.selective-checks.outputs.run-tests }}
@@ -107,6 +104,8 @@ jobs:
       ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }}
       prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }}
       docs-build: ${{ steps.selective-checks.outputs.docs-build }}
+      mypy-packages: ${{ steps.selective-checks.outputs.mypy-packages }}
+      needs-mypy: ${{ steps.selective-checks.outputs.needs-mypy }}
       needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }}
       needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }}
       needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }}
@@ -513,7 +512,6 @@ jobs:
           retention-days: 7
           if-no-files-found: error
 
-
   static-checks:
     timeout-minutes: 45
     name: "Static checks"
@@ -553,6 +551,57 @@ jobs:
           DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
           RUFF_FORMAT: "github"
 
+  # Runs static checks for groups of files in the repository in a single process without passing .
+  # List of files
+  mypy:
+    timeout-minutes: 45
+    name: "MyPy checks"
+    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+    needs: [build-info, wait-for-ci-images]
+    strategy:
+      fail-fast: false
+      matrix:
+        mypy-package: ${{fromJson(needs.build-info.outputs.mypy-packages)}}
+    env:
+      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
+      PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
+      UPGRADE_TO_NEWER_DEPENDENCIES: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}"
+    steps:
+      - name: Cleanup repo
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+        if: needs.build-info.outputs.needs-mypy == 'true'
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+        if: needs.build-info.outputs.needs-mypy == 'true'
+      - name: >
+          Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
+        uses: ./.github/actions/prepare_breeze_and_image
+        id: breeze
+        if: needs.build-info.outputs.needs-mypy == 'true'
+      - name: Cache pre-commit envs
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pre-commit
+          # yamllint disable-line rule:line-length
+          key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
+          restore-keys: |
+            pre-commit-${{steps.breeze.outputs.host-python-version}}-
+        if: needs.build-info.outputs.needs-mypy == 'true'
+      - name: "MyPy checks for ${{ matrix.mypy-package }}"
+        run: |
+          pip install pre-commit
+          pre-commit run --color always --verbose --hook-stage manual mypy --all-files
+        env:
+          VERBOSE: "false"
+          COLUMNS: "250"
+          SKIP_GROUP_OUTPUT: "true"
+          DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+          RUFF_FORMAT: "github"
+          MYPY_PACKAGES: ${{ matrix.mypy-package }}
+        if: needs.build-info.outputs.needs-mypy == 'true'
+
   # Those checks are run if no image needs to be built for checks. This is for simple changes that
   # Do not touch any of the python code or any of the important files that might require building
   # The CI Docker image and they can be run entirely using the pre-commit virtual environments on host
@@ -671,7 +720,7 @@ jobs:
         run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs
 
   spellcheck-docs:
-    timeout-minutes: 60
+    timeout-minutes: 120
     name: "Spellcheck docs"
     runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
     needs: [build-info, wait-for-ci-images]
@@ -1236,69 +1285,6 @@ jobs:
         if: failure()
 
 
-  tests-mssql:
-    timeout-minutes: 130
-    name: >
-      DB:MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}:
-      ${{needs.build-info.outputs.parallel-test-types-list-as-string}}
-    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
-    needs: [build-info, wait-for-ci-images]
-    strategy:
-      matrix:
-        python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}"
-        mssql-version: "${{fromJson(needs.build-info.outputs.mssql-versions)}}"
-        exclude: "${{fromJson(needs.build-info.outputs.mssql-exclude)}}"
-      fail-fast: false
-    env:
-      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
-      PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
-      PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}"
-      FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}"
-      DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}"
-      BACKEND: "mssql"
-      ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}"
-      PYTHON_MAJOR_MINOR_VERSION: "${{matrix.python-version}}"
-      MSSQL_VERSION: "${{matrix.mssql-version}}"
-      BACKEND_VERSION: "${{matrix.mssql-version}}"
-      JOB_ID: "mssql-${{matrix.mssql-version}}-${{matrix.python-version}}"
-      # The below (and corresponding selective checks code) can be removed once
-      # https://github.com/apache/airflow/issues/31575 is fixed.
-      # This is a temporary workaround for flaky tests that occur in MSSQL tests for public runners
-      PARALLELISM: "${{needs.build-info.outputs.mssql-parallelism}}"
-    if: >
-      needs.build-info.outputs.run-tests == 'true' &&
-      (needs.build-info.outputs.is-self-hosted-runner == 'true' &&
-       needs.build-info.outputs.is-airflow-runner == 'true' ||
-      needs.build-info.outputs.full-tests-needed == 'true' ||
-      needs.build-info.outputs.has-migrations == 'true')
-    steps:
-      - name: Cleanup repo
-        shell: bash
-        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-      - name: "Prepare breeze & CI image: ${{matrix.python-version}}:${{env.IMAGE_TAG}}"
-        uses: ./.github/actions/prepare_breeze_and_image
-      - name: >
-          Migration Tests:
-          ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
-        uses: ./.github/actions/migration_tests
-      - name: >
-          Tests: ${{matrix.python-version}}:${{needs.build-info.outputs.parallel-test-types-list-as-string}}
-        run: >
-          breeze testing db-tests
-          --parallel-test-types "${{needs.build-info.outputs.parallel-test-types-list-as-string}}"
-      - name: >
-          Post Tests success: MsSQL"
-        uses: ./.github/actions/post_tests_success
-        if: success()
-      - name: >
-          Post Tests failure: MsSQL"
-        uses: ./.github/actions/post_tests_failure
-        if: failure()
-
   tests-sqlite:
     timeout-minutes: 130
     name: >
@@ -1507,20 +1493,12 @@ jobs:
           BACKEND: "mysql"
           BACKEND_VERSION: ${{needs.build-info.outputs.default-mysql-version}}
           MYSQL_VERSION: ${{needs.build-info.outputs.default-mysql-version}}
-      - name: >
-          Tests: mssql:${{needs.build-info.outputs.default-python-version}}:Quarantined
-        run: breeze testing tests || true
-        env:
-          BACKEND: "mssql"
-          BACKEND_VERSION: ${{needs.build-info.outputs.default-mssql-version}}
-          MSSQL_VERSION: ${{needs.build-info.outputs.default-mssql-version}}
       - name: >
           Tests: sqlite:${{needs.build-info.outputs.default-python-version}}:Quarantined
         run: breeze testing tests || true
         env:
           BACKEND: "sqlite"
           BACKEND_VERSION: ""
-          MSSQL_VERSION: ""
       - name: >
           Post Tests success: Quarantined"
         uses: ./.github/actions/post_tests_success
@@ -1712,7 +1690,7 @@ jobs:
   build-prod-images-mysql-client:
     timeout-minutes: 80
     name: >
-      Build MysQL Client PROD images (main)
+      Build MySQL Client PROD images (main)
       ${{needs.build-info.outputs.all-python-versions-list-as-string}}
     runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
     needs: [build-info, build-ci-images]
@@ -1746,7 +1724,7 @@ jobs:
           needs.build-info.outputs.in-workflow-build == 'true' &&
           needs.build-info.outputs.default-branch == 'main'
       - name: >
-          Build Bullseye PROD Images
+          Build MySQL Client PROD Images
           ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
         uses: ./.github/actions/build-prod-images
         if: >
@@ -2115,6 +2093,7 @@ jobs:
       - wait-for-ci-images
       - wait-for-prod-images
       - static-checks
+      - mypy
       - tests-sqlite
       - tests-mysql
       - tests-postgres
@@ -2269,6 +2248,7 @@ jobs:
       - build-docs
       - spellcheck-docs
       - static-checks
+      - mypy
       - tests-sqlite
       - tests-mysql
       - tests-postgres

.github/workflows/recheck-old-bug-report.yml

@@ -28,7 +28,7 @@ jobs:
   recheck-old-bug-report:
     runs-on: "ubuntu-22.04"
     steps:
-      - uses: actions/stale@v8
+      - uses: actions/stale@v9
         with:
           only-issue-labels: 'kind:bug'
           stale-issue-label: 'Stale Bug Report'
@@ -38,8 +38,10 @@ jobs:
           # scan only issues: https://github.com/actions/stale/issues/837
           # To avoid this job scanning also PRs and setting the defaults of the bot to them,
           # we set high number of days thus effectively this job will not do any changes on PRs
-          days-before-pr-stale: 7000
-          days-before-pr-close: 7000
+          # Possible workaround with -1 as mentioned in
+          # https://github.com/actions/stale/issues/1112#issuecomment-1871654196
+          days-before-pr-stale: -1
+          days-before-pr-close: -1
           remove-stale-when-updated: false
           remove-issue-stale-when-updated: true
           labels-to-add-when-unstale: 'needs-triage'

.github/workflows/stale.yml

@@ -29,7 +29,7 @@ jobs:
   stale:
     runs-on: "ubuntu-22.04"
     steps:
-      - uses: actions/stale@v8
+      - uses: actions/stale@v9
         with:
           stale-pr-message: >
             This pull request has been automatically marked as stale because it has not had