diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 58635976..cb0d124a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -1,5 +1,8 @@ name: Test on: [push, pull_request] +concurrency: # https://stackoverflow.com/questions/66335225#comment133398800_72408109 + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: linters: runs-on: ubuntu-latest @@ -12,10 +15,10 @@ jobs: python-version: "3.11" - name: Update pip - run: python -m pip install -U pip + run: pip install -U pip - name: Install dependencies - run: python -m pip install flake8 + run: pip install flake8 - name: Run flake8 linter (source) run: flake8 --show-source smart_open @@ -26,17 +29,17 @@ jobs: strategy: matrix: include: - - {python: '3.8', os: ubuntu-20.04} - - {python: '3.9', os: ubuntu-20.04} - - {python: '3.10', os: ubuntu-20.04} - - {python: '3.11', os: ubuntu-20.04} - - {python: '3.12', os: ubuntu-20.04} - - - {python: '3.8', os: windows-2019} - - {python: '3.9', os: windows-2019} - - {python: '3.10', os: windows-2019} - - {python: '3.11', os: windows-2019} - - {python: '3.12', os: windows-2019} + - {python-version: '3.8', os: ubuntu-20.04} + - {python-version: '3.9', os: ubuntu-20.04} + - {python-version: '3.10', os: ubuntu-20.04} + - {python-version: '3.11', os: ubuntu-20.04} + - {python-version: '3.12', os: ubuntu-20.04} + + - {python-version: '3.8', os: windows-2019} + - {python-version: '3.9', os: windows-2019} + - {python-version: '3.10', os: windows-2019} + - {python-version: '3.11', os: windows-2019} + - {python-version: '3.12', os: windows-2019} steps: - uses: actions/checkout@v2 @@ -45,13 +48,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Update pip - run: python -m pip install -U pip - - # - # https://askubuntu.com/questions/1428181/module-lib-has-no-attribute-x509-v-flag-cb-issuer-check - # - - name: Upgrade PyOpenSSL - run: python -m pip install pyOpenSSL --upgrade + run: pip install -U pip - name: Install smart_open without dependencies run: pip install -e . @@ -71,20 +68,20 @@ jobs: strategy: matrix: include: - - {python: '3.8', os: ubuntu-20.04} - - {python: '3.9', os: ubuntu-20.04} - - {python: '3.10', os: ubuntu-20.04} - - {python: '3.11', os: ubuntu-20.04} - - {python: '3.12', os: ubuntu-20.04} + - {python-version: '3.8', os: ubuntu-20.04} + - {python-version: '3.9', os: ubuntu-20.04} + - {python-version: '3.10', os: ubuntu-20.04} + - {python-version: '3.11', os: ubuntu-20.04} + - {python-version: '3.12', os: ubuntu-20.04} # # Some of the doctests don't pass on Windows because of Windows-specific # character encoding issues. # - # - {python: '3.7', os: windows-2019} - # - {python: '3.8', os: windows-2019} - # - {python: '3.9', os: windows-2019} - # - {python: '3.10', os: windows-2019} + # - {python-version: '3.7', os: windows-2019} + # - {python-version: '3.8', os: windows-2019} + # - {python-version: '3.9', os: windows-2019} + # - {python-version: '3.10', os: windows-2019} steps: - uses: actions/checkout@v2 @@ -94,10 +91,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Update pip - run: python -m pip install -U pip - - - name: Upgrade PyOpenSSL - run: python -m pip install pyOpenSSL --upgrade + run: pip install -U pip - name: Install smart_open and its dependencies run: pip install -e .[test] @@ -114,18 +108,18 @@ jobs: strategy: matrix: include: - - {python: '3.8', os: ubuntu-20.04} - - {python: '3.9', os: ubuntu-20.04} - - {python: '3.10', os: ubuntu-20.04} - - {python: '3.11', os: ubuntu-20.04} - - {python: '3.12', os: ubuntu-20.04} + - {python-version: '3.8', os: ubuntu-20.04} + - {python-version: '3.9', os: ubuntu-20.04} + - {python-version: '3.10', os: ubuntu-20.04} + - {python-version: '3.11', os: ubuntu-20.04} + - {python-version: '3.12', os: ubuntu-20.04} # Not sure why we exclude these, perhaps for historical reasons? # - # - {python: '3.7', os: windows-2019} - # - {python: '3.8', os: windows-2019} - # - {python: '3.9', os: windows-2019} - # - {python: '3.10', os: windows-2019} + # - {python-version: '3.7', os: windows-2019} + # - {python-version: '3.8', os: windows-2019} + # - {python-version: '3.9', os: windows-2019} + # - {python-version: '3.10', os: windows-2019} steps: - uses: actions/checkout@v2 @@ -135,12 +129,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Update pip - run: python -m pip install -U pip - - - name: Upgrade PyOpenSSL - run: python -m pip install pyOpenSSL --upgrade - - - run: python -m pip install numpy + run: pip install -U pip - name: Install smart_open and its dependencies run: pip install -e .[test] @@ -148,7 +137,9 @@ jobs: - run: bash ci_helpers/helpers.sh enable_moto_server if: ${{ matrix.moto_server }} - - run: | + - name: Start vsftpd + timeout-minutes: 2 + run: | sudo apt-get install vsftpd sudo bash ci_helpers/helpers.sh create_ftp_ftps_servers @@ -169,16 +160,16 @@ jobs: strategy: matrix: include: - - {python: '3.8', os: ubuntu-20.04} - - {python: '3.9', os: ubuntu-20.04} - - {python: '3.10', os: ubuntu-20.04} - - {python: '3.11', os: ubuntu-20.04} - - {python: '3.12', os: ubuntu-20.04} + - {python-version: '3.8', os: ubuntu-20.04} + - {python-version: '3.9', os: ubuntu-20.04} + - {python-version: '3.10', os: ubuntu-20.04} + - {python-version: '3.11', os: ubuntu-20.04} + - {python-version: '3.12', os: ubuntu-20.04} - # - {python: '3.7', os: windows-2019} - # - {python: '3.8', os: windows-2019} - # - {python: '3.9', os: windows-2019} - # - {python: '3.10', os: windows-2019} + # - {python-version: '3.7', os: windows-2019} + # - {python-version: '3.8', os: windows-2019} + # - {python-version: '3.9', os: windows-2019} + # - {python-version: '3.10', os: windows-2019} steps: - uses: actions/checkout@v2 @@ -188,16 +179,11 @@ jobs: python-version: ${{ matrix.python-version }} - name: Update pip - run: python -m pip install -U pip - - - name: Upgrade PyOpenSSL - run: python -m pip install pyOpenSSL --upgrade + run: pip install -U pip - name: Install smart_open and its dependencies run: pip install -e .[test] - - run: pip install awscli pytest_benchmark - - name: Run benchmarks run: python ci_helpers/run_benchmarks.py env: diff --git a/ci_helpers/helpers.sh b/ci_helpers/helpers.sh index 4533b337..e0accc7a 100644 --- a/ci_helpers/helpers.sh +++ b/ci_helpers/helpers.sh @@ -20,6 +20,9 @@ create_ftp_ftps_servers(){ mkdir $home_dir useradd -p $(echo $pass | openssl passwd -1 -stdin) -d $home_dir $user chown $user:$user $home_dir + openssl req -x509 -nodes -new -sha256 -days 10240 -newkey rsa:2048 -keyout /etc/vsftpd.key -out /etc/vsftpd.pem -subj "/C=ZA/CN=localhost" + chmod 755 /etc/vsftpd.key + chmod 755 /etc/vsftpd.pem server_setup=''' listen=YES @@ -32,6 +35,8 @@ chroot_local_user=YES allow_writeable_chroot=YES''' additional_ssl_setup=''' +rsa_cert_file=/etc/vsftpd.pem +rsa_private_key_file=/etc/vsftpd.key ssl_enable=YES allow_anon_ssl=NO force_local_data_ssl=NO diff --git a/integration-tests/test_ftp.py b/integration-tests/test_ftp.py index 000faae7..94b4e037 100644 --- a/integration-tests/test_ftp.py +++ b/integration-tests/test_ftp.py @@ -1,6 +1,11 @@ from __future__ import unicode_literals import pytest from smart_open import open +import ssl +from functools import partial + +# localhost has self-signed cert, see ci_helpers/helpers.sh:create_ftp_ftps_servers +ssl.create_default_context = partial(ssl.create_default_context, cafile="/etc/vsftpd.pem") @pytest.fixture(params=[("ftp", 21), ("ftps", 90)]) @@ -81,4 +86,4 @@ def test_line_endings_binary(server_info): with open(f"{server_type}://user:123@localhost:{port_num}/file4", "rb") as f: for line in f: - assert B_CLRF in line \ No newline at end of file + assert B_CLRF in line diff --git a/setup.py b/setup.py index a9a4fc53..9e738bea 100644 --- a/setup.py +++ b/setup.py @@ -47,9 +47,12 @@ def read(fname): tests_require = all_deps + [ 'moto[server]', 'responses', - 'boto3', 'pytest', 'pytest-rerunfailures', + 'pytest_benchmark', + 'awscli', + 'pyopenssl', + 'numpy', ] setup( diff --git a/smart_open/ftp.py b/smart_open/ftp.py index 40d54ca3..a7212ecd 100644 --- a/smart_open/ftp.py +++ b/smart_open/ftp.py @@ -86,7 +86,7 @@ def convert_transport_params_to_args(transport_params): def _connect(hostname, username, port, password, secure_connection, transport_params): kwargs = convert_transport_params_to_args(transport_params) if secure_connection: - ssl_context = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH) + ssl_context = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH) ftp = FTP_TLS(context=ssl_context, **kwargs) else: ftp = FTP(**kwargs) diff --git a/smart_open/tests/test_http.py b/smart_open/tests/test_http.py index f4f8338a..9433043f 100644 --- a/smart_open/tests/test_http.py +++ b/smart_open/tests/test_http.py @@ -21,24 +21,22 @@ URL = 'http://localhost' HTTPS_URL = 'https://localhost' HEADERS = { - 'Content-Length': str(len(BYTES)), 'Accept-Ranges': 'bytes', } def request_callback(request, headers=HEADERS, data=BYTES): - try: - range_string = request.headers['range'] - except KeyError: - return (200, headers, data) + headers = headers.copy() + range_string = request.headers.get('range', 'bytes=0-') - start, end = range_string.replace('bytes=', '').split('-', 1) + start, end = range_string.replace('bytes=', '', 1).split('-', 1) start = int(start) - if end: - end = int(end) - else: - end = len(data) - return (200, headers, data[start:end]) + end = int(end) if end else len(data) + + data = data[start:end] + headers['Content-Length'] = str(len(data)) + + return (200, headers, data) @unittest.skipIf(os.environ.get('TRAVIS'), 'This test does not work on TravisCI for some reason')