diff --git a/.github/.pull_request_template.md b/.github/pull_request_template.md similarity index 100% rename from .github/.pull_request_template.md rename to .github/pull_request_template.md diff --git a/.github/workflows/_publish_to_pypi.yaml b/.github/workflows/_publish_to_pypi.yaml new file mode 100644 index 000000000..e4c84510e --- /dev/null +++ b/.github/workflows/_publish_to_pypi.yaml @@ -0,0 +1,70 @@ +name: Publish to PyPI + +on: + workflow_call: + +jobs: + publish_to_pypi: + name: Publish to PyPI + runs-on: ubuntu-latest + permissions: + contents: write + environment: + name: pypi + url: https://pypi.org/project/crawlee + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: 3.12 + + - name: Install dependencies + run: make install-dev + + # Determines the release type based on the event that triggered the workflow. + - name: Determine release type + id: determine-release-type + run: | + if [[ ${{ github.event_name }} = release ]]; then + release_type="final" + elif [[ ${{ github.event_name }} = push ]]; then + release_type="beta" + elif [[ ${{ github.event_name }} = workflow_dispatch ]]; then + release_type=${{ github.event.inputs.release_type }} + fi + echo "release_type=${release_type}" >> $GITHUB_OUTPUT + + # Updates the version number for pre-releases in the project's configuration. + - name: Set pre-release version + if: steps.determine-release-type.outputs.release_type != 'final' + run: python ./scripts/update_version_for_prerelease.py ${{ steps.determine-release-type.outputs.release_type }} + + # - name: Set up PyPI token + # run: poetry config pypi-token.pypi "$APIFY_PYPI_TOKEN_CRAWLEE" + # env: + # APIFY_PYPI_TOKEN_CRAWLEE: ${{ secrets.APIFY_PYPI_TOKEN_CRAWLEE }} + + # Builds and publishes the package to PyPI, using environment variables for PyPI authentication. + - name: Build & publish package to PyPI + run: make build-and-publish-to-pypi + env: + APIFY_PYPI_TOKEN_CRAWLEE: ${{ secrets.APIFY_PYPI_TOKEN_CRAWLEE }} + + # If this workflow is not triggered by a GitHub release event, manually create and push a Git tag. + - name: Create Git tag with the published version + if: github.event_name != 'release' + run: | + GIT_TAG=v$(python ./scripts/print_current_package_version.py) + git tag "$GIT_TAG" + git push origin $git_tag + + # If triggered by a release, upload build artifacts to the associated GitHub release. + - name: Upload the build artifacts to release + if: github.event_name == 'release' + run: gh release upload ${{ github.ref_name }} dist/* + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/run_code_checks.yaml b/.github/workflows/run_code_checks.yaml index 61bd17121..267137bbe 100644 --- a/.github/workflows/run_code_checks.yaml +++ b/.github/workflows/run_code_checks.yaml @@ -1,11 +1,17 @@ name: Run code checks on: + # Trigger code checks on opening a new pull request. pull_request_target: + + # Trigger code checks on push to the master branch. push: branches: - master + # Trigger code checks on workflow call (e.g. from run release workflow). + workflow_call: + jobs: run_linting: name: Run linting diff --git a/.github/workflows/run_release.yaml b/.github/workflows/run_release.yaml new file mode 100644 index 000000000..10e279eb1 --- /dev/null +++ b/.github/workflows/run_release.yaml @@ -0,0 +1,39 @@ +name: Run release + +on: + # Trigger a beta version release (pre-release) on push to the master branch. + push: + branches: + - master + tags-ignore: + - "**" # Ignore all tags to prevent duplicate builds when tags are pushed. + + # Trigger a stable version release when a GitHub release is published. + release: + types: [published] + + # Allows manual workflow runs from GitHub's UI, with the ability to specify the type of release. + workflow_dispatch: + inputs: + release_type: + description: Release type + required: true + type: choice + default: alpha + options: + - alpha + - beta + - final + +jobs: + run_code_checks: + name: Run code checks + uses: ./.github/workflows/run_code_checks.yaml + + publish_to_pypi: + name: Publish to PyPI + needs: [run_code_checks] + uses: ./.github/workflows/_publish_to_pypi.yaml + + # TODO: add job for publish package to Conda + # https://github.com/apify/crawlee-py/issues/104 diff --git a/Makefile b/Makefile index 96cbeb49b..9386c118a 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean install-dev lint type-check unit-tests unit-tests-cov integration-tests check-code format check-version-conflict check-changelog-entry +.PHONY: clean install-dev build lint type-check unit-tests unit-tests-cov integration-tests check-code format check-version-conflict check-changelog-entry DIRS_WITH_CODE = src tests scripts @@ -13,6 +13,11 @@ install-dev: poetry install --all-extras poetry run pre-commit install +# APIFY_PYPI_TOKEN_CRAWLEE is expected to be set in the environment +build-and-publish-to-pypi: + poetry config pypi-token.pypi "${APIFY_PYPI_TOKEN_CRAWLEE}" + poetry publish --build + lint: poetry run ruff check $(DIRS_WITH_CODE) diff --git a/pyproject.toml b/pyproject.toml index 3a8619e6c..a614ca175 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,6 @@ pytest-timeout = "~2.3.0" pytest-xdist = "~3.5.0" respx = "~0.21.0" ruff = "~0.4.0" -twine = "~5.0.0" types-aiofiles = "^23.2.0.20240106" types-beautifulsoup4 = "^4.12.0.20240229" types-colorama = "~0.4.15.20240106" diff --git a/scripts/utils.py b/scripts/utils.py index 9de602633..c544f0f37 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -43,7 +43,7 @@ def set_current_package_version(version: str) -> None: # Load the version numbers of the currently published versions from PyPI -def get_published_package_versions() -> list: +def get_published_package_versions() -> list[str]: package_info_url = f'https://pypi.org/pypi/{PACKAGE_NAME}/json' try: package_data = json.load(urlopen(package_info_url)) # noqa: S310