Integrate automated PyPI releases into CI

.github/workflows/_publish_to_pypi.yaml

@@ -0,0 +1,63 @@
+name: Publish to PyPI
+
+on:
+  workflow_call:
+
+jobs:
+  publish_to_pypi:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    environment:
+      name: pypi
+      url: https://pypi.org/project/crawlee
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.12
+
+      - name: Install dependencies
+        run: make install-dev
+
+      # Determines the release type based on the event that triggered the workflow.
+      - name: Determine release type
+        id: determine-release-type
+        run: |
+          if [[ ${{ github.event_name }} = release ]]; then
+            release_type="final"
+          elif [[ ${{ github.event_name }} = push ]]; then
+            release_type="beta"
+          elif [[ ${{ github.event_name }} = workflow_dispatch ]]; then
+            release_type=${{ github.event.inputs.release_type }}
+          fi
+          echo "release_type=${release_type}" >> $GITHUB_OUTPUT
+
+      # Updates the version number for pre-releases in the project's configuration.
+      - name: Set pre-release version
+        if: steps.determine-release-type.outputs.release_type != 'final'
+        run: python ./scripts/update_version_for_prerelease.py ${{ steps.determine-release-type.outputs.release_type }}
+
+      # Builds and publishes the package to PyPI, using OIDC for PyPI authentication.
+      - name: Build & publish package to PyPI
+        run: poetry publish --build --no-interaction -vv
+
+      # If this workflow is not triggered by a GitHub release event, manually create and push a Git tag.
+      - name: Create Git tag with the published version
+        if: github.event_name != 'release'
+        run: |
+          GIT_TAG=v$(python ./scripts/print_current_package_version.py)
+          git tag "$GIT_TAG"
+          git push origin $git_tag
+
+      # If triggered by a release, upload build artifacts to the associated GitHub release.
+      - name: Upload the build artifacts to release
+        if: github.event_name == 'release'
+        run: gh release upload ${{ github.ref_name }} dist/*
+        env:
+          GH_TOKEN: ${{ github.token }}

.github/workflows/run_code_checks.yaml

@@ -1,11 +1,17 @@
 name: Run code checks
 
 on:
+  # Trigger code checks on opening a new pull request.
   pull_request_target:
+
+  # Trigger code checks on push to the master branch.
   push:
     branches:
       - master
 
+  # Trigger code checks on workflow call (e.g. from run release workflow).
+  workflow_call:
+
 jobs:
   run_linting:
     name: Run linting

.github/workflows/run_release.yaml

@@ -0,0 +1,39 @@
+name: Run release
+
+on:
+  # Trigger a beta version release (pre-release) on push to the master branch.
+  push:
+    branches:
+      - master
+    tags-ignore:
+      - "**" # Ignore all tags to prevent duplicate builds when tags are pushed.
+
+  # Trigger a stable version release when a GitHub release is published.
+  release:
+    types: [published]
+
+  # Allows manual workflow runs from GitHub's UI, with the ability to specify the type of release.
+  workflow_dispatch:
+    inputs:
+      release_type:
+        description: Release type
+        required: true
+        type: choice
+        default: alpha
+        options:
+          - alpha
+          - beta
+          - final
+
+jobs:
+  run_code_checks:
+    name: Run code checks
+    uses: ./.github/workflows/run_code_checks.yaml
+
+  publish_to_pypi:
+    name: Publish to PyPI
+    needs: [run_code_checks]
+    uses: ./.github/workflows/_publish_to_pypi.yaml
+
+  # TODO: add job for publish package to Conda
+  # https://github.com/apify/crawlee-py/issues/104

Makefile

@@ -1,4 +1,4 @@
-.PHONY: clean install-dev lint type-check unit-tests unit-tests-cov integration-tests check-code format check-version-conflict check-changelog-entry
+.PHONY: clean install-dev build-and-publish-to-pypi lint type-check unit-tests unit-tests-cov integration-tests check-code format check-version-conflict check-changelog-entry
 
 DIRS_WITH_CODE = src tests scripts
 
@@ -13,6 +13,12 @@ install-dev:
 	poetry install --all-extras
 	poetry run pre-commit install
 
+# APIFY_PYPI_TOKEN_CRAWLEE is expected to be set in the environment
+build-and-publish-to-pypi:
+	rm -rf dist
+	poetry config pypi-token.pypi "${APIFY_PYPI_TOKEN_CRAWLEE}"
+	poetry publish --build --no-interaction -vv
+
 lint:
 	poetry run ruff check $(DIRS_WITH_CODE)
 

pyproject.toml

@@ -77,7 +77,6 @@ pytest-timeout = "~2.3.0"
 pytest-xdist = "~3.5.0"
 respx = "~0.21.0"
 ruff = "~0.4.0"
-twine = "~5.0.0"
 types-aiofiles = "^23.2.0.20240106"
 types-beautifulsoup4 = "^4.12.0.20240229"
 types-colorama = "~0.4.15.20240106"

scripts/utils.py

@@ -43,7 +43,7 @@ def set_current_package_version(version: str) -> None:
 
 
 # Load the version numbers of the currently published versions from PyPI
-def get_published_package_versions() -> list:
+def get_published_package_versions() -> list[str]:
     package_info_url = f'https://pypi.org/pypi/{PACKAGE_NAME}/json'
     try:
         package_data = json.load(urlopen(package_info_url))  # noqa: S310