Fix test_http.request_callback

piskvorky · Jul 25, 2024 · ab55812 · ab55812
1 parent 5a82613
commit ab55812
Show file tree

Hide file tree

Showing 6 changed files with 75 additions and 78 deletions.
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -1,5 +1,8 @@
 name: Test
 on: [push, pull_request]
+concurrency:  # https://stackoverflow.com/questions/66335225#comment133398800_72408109
+  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 jobs:
   linters:
     runs-on: ubuntu-latest
@@ -12,10 +15,10 @@ jobs:
           python-version: "3.11"
 
       - name: Update pip
-        run: python -m pip install -U pip
+        run: pip install -U pip
 
       - name: Install dependencies
-        run: python -m pip install flake8
+        run: pip install flake8
 
       - name: Run flake8 linter (source)
         run: flake8 --show-source smart_open
@@ -26,17 +29,17 @@ jobs:
     strategy:
       matrix:
         include:
-          - {python: '3.8', os: ubuntu-20.04}
-          - {python: '3.9', os: ubuntu-20.04}
-          - {python: '3.10', os: ubuntu-20.04}
-          - {python: '3.11', os: ubuntu-20.04}
-          - {python: '3.12', os: ubuntu-20.04}
-
-          - {python: '3.8', os: windows-2019}
-          - {python: '3.9', os: windows-2019}
-          - {python: '3.10', os: windows-2019}
-          - {python: '3.11', os: windows-2019}
-          - {python: '3.12', os: windows-2019}
+          - {python-version: '3.8', os: ubuntu-20.04}
+          - {python-version: '3.9', os: ubuntu-20.04}
+          - {python-version: '3.10', os: ubuntu-20.04}
+          - {python-version: '3.11', os: ubuntu-20.04}
+          - {python-version: '3.12', os: ubuntu-20.04}
+
+          - {python-version: '3.8', os: windows-2019}
+          - {python-version: '3.9', os: windows-2019}
+          - {python-version: '3.10', os: windows-2019}
+          - {python-version: '3.11', os: windows-2019}
+          - {python-version: '3.12', os: windows-2019}
     steps:
       - uses: actions/checkout@v2
 
@@ -45,13 +48,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Update pip
-        run: python -m pip install -U pip
-
-      #
-      # https://askubuntu.com/questions/1428181/module-lib-has-no-attribute-x509-v-flag-cb-issuer-check
-      #
-      - name: Upgrade PyOpenSSL
-        run: python -m pip install pyOpenSSL --upgrade
+        run: pip install -U pip
 
       - name: Install smart_open without dependencies
         run: pip install -e .
@@ -71,20 +68,20 @@ jobs:
     strategy:
       matrix:
         include:
-          - {python: '3.8', os: ubuntu-20.04}
-          - {python: '3.9', os: ubuntu-20.04}
-          - {python: '3.10', os: ubuntu-20.04}
-          - {python: '3.11', os: ubuntu-20.04}
-          - {python: '3.12', os: ubuntu-20.04}
+          - {python-version: '3.8', os: ubuntu-20.04}
+          - {python-version: '3.9', os: ubuntu-20.04}
+          - {python-version: '3.10', os: ubuntu-20.04}
+          - {python-version: '3.11', os: ubuntu-20.04}
+          - {python-version: '3.12', os: ubuntu-20.04}
 
           #
           # Some of the doctests don't pass on Windows because of Windows-specific
           # character encoding issues.
           #
-          # - {python: '3.7', os: windows-2019}
-          # - {python: '3.8', os: windows-2019}
-          # - {python: '3.9', os: windows-2019}
-          # - {python: '3.10', os: windows-2019}
+          # - {python-version: '3.7', os: windows-2019}
+          # - {python-version: '3.8', os: windows-2019}
+          # - {python-version: '3.9', os: windows-2019}
+          # - {python-version: '3.10', os: windows-2019}
 
     steps:
       - uses: actions/checkout@v2
@@ -94,10 +91,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Update pip
-        run: python -m pip install -U pip
-
-      - name: Upgrade PyOpenSSL
-        run: python -m pip install pyOpenSSL --upgrade
+        run: pip install -U pip
 
       - name: Install smart_open and its dependencies
         run: pip install -e .[test]
@@ -114,18 +108,18 @@ jobs:
     strategy:
       matrix:
         include:
-          - {python: '3.8', os: ubuntu-20.04}
-          - {python: '3.9', os: ubuntu-20.04}
-          - {python: '3.10', os: ubuntu-20.04}
-          - {python: '3.11', os: ubuntu-20.04}
-          - {python: '3.12', os: ubuntu-20.04}
+          - {python-version: '3.8', os: ubuntu-20.04}
+          - {python-version: '3.9', os: ubuntu-20.04}
+          - {python-version: '3.10', os: ubuntu-20.04}
+          - {python-version: '3.11', os: ubuntu-20.04}
+          - {python-version: '3.12', os: ubuntu-20.04}
 
           # Not sure why we exclude these, perhaps for historical reasons?
           #
-          # - {python: '3.7', os: windows-2019}
-          # - {python: '3.8', os: windows-2019}
-          # - {python: '3.9', os: windows-2019}
-          # - {python: '3.10', os: windows-2019}
+          # - {python-version: '3.7', os: windows-2019}
+          # - {python-version: '3.8', os: windows-2019}
+          # - {python-version: '3.9', os: windows-2019}
+          # - {python-version: '3.10', os: windows-2019}
 
     steps:
       - uses: actions/checkout@v2
@@ -135,20 +129,17 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Update pip
-        run: python -m pip install -U pip
-
-      - name: Upgrade PyOpenSSL
-        run: python -m pip install pyOpenSSL --upgrade
-
-      - run: python -m pip install numpy
+        run: pip install -U pip
 
       - name: Install smart_open and its dependencies
         run: pip install -e .[test]
 
       - run: bash ci_helpers/helpers.sh enable_moto_server
         if: ${{ matrix.moto_server }}
 
-      - run: |
+      - name: Start vsftpd
+        timeout-minutes: 2
+        run: |
           sudo apt-get install vsftpd
           sudo bash ci_helpers/helpers.sh create_ftp_ftps_servers
 
@@ -169,16 +160,16 @@ jobs:
     strategy:
       matrix:
         include:
-          - {python: '3.8', os: ubuntu-20.04}
-          - {python: '3.9', os: ubuntu-20.04}
-          - {python: '3.10', os: ubuntu-20.04}
-          - {python: '3.11', os: ubuntu-20.04}
-          - {python: '3.12', os: ubuntu-20.04}
+          - {python-version: '3.8', os: ubuntu-20.04}
+          - {python-version: '3.9', os: ubuntu-20.04}
+          - {python-version: '3.10', os: ubuntu-20.04}
+          - {python-version: '3.11', os: ubuntu-20.04}
+          - {python-version: '3.12', os: ubuntu-20.04}
 
-          # - {python: '3.7', os: windows-2019}
-          # - {python: '3.8', os: windows-2019}
-          # - {python: '3.9', os: windows-2019}
-          # - {python: '3.10', os: windows-2019}
+          # - {python-version: '3.7', os: windows-2019}
+          # - {python-version: '3.8', os: windows-2019}
+          # - {python-version: '3.9', os: windows-2019}
+          # - {python-version: '3.10', os: windows-2019}
 
     steps:
       - uses: actions/checkout@v2
@@ -188,16 +179,11 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Update pip
-        run: python -m pip install -U pip
-
-      - name: Upgrade PyOpenSSL
-        run: python -m pip install pyOpenSSL --upgrade
+        run: pip install -U pip
 
       - name: Install smart_open and its dependencies
         run: pip install -e .[test]
 
-      - run: pip install awscli pytest_benchmark
-
       - name: Run benchmarks
         run: python ci_helpers/run_benchmarks.py
         env:

diff --git a/ci_helpers/helpers.sh b/ci_helpers/helpers.sh
@@ -20,6 +20,9 @@ create_ftp_ftps_servers(){
   mkdir $home_dir
   useradd -p $(echo $pass | openssl passwd -1 -stdin) -d $home_dir $user
   chown $user:$user $home_dir
+  openssl req -x509 -nodes -new -sha256 -days 10240 -newkey rsa:2048 -keyout /etc/vsftpd.key -out /etc/vsftpd.pem -subj "/C=ZA/CN=localhost"
+  chmod 755 /etc/vsftpd.key
+  chmod 755 /etc/vsftpd.pem
 
   server_setup='''
 listen=YES
@@ -32,6 +35,8 @@ chroot_local_user=YES
 allow_writeable_chroot=YES'''
 
   additional_ssl_setup='''
+rsa_cert_file=/etc/vsftpd.pem
+rsa_private_key_file=/etc/vsftpd.key
 ssl_enable=YES
 allow_anon_ssl=NO
 force_local_data_ssl=NO

diff --git a/integration-tests/test_ftp.py b/integration-tests/test_ftp.py
@@ -1,6 +1,11 @@
 from __future__ import unicode_literals
 import pytest
 from smart_open import open
+import ssl
+from functools import partial
+
+# localhost has self-signed cert, see ci_helpers/helpers.sh:create_ftp_ftps_servers
+ssl.create_default_context = partial(ssl.create_default_context, cafile="/etc/vsftpd.pem")
 
 
 @pytest.fixture(params=[("ftp", 21), ("ftps", 90)])
@@ -81,4 +86,4 @@ def test_line_endings_binary(server_info):
 
     with open(f"{server_type}://user:123@localhost:{port_num}/file4", "rb") as f:    
         for line in f:
-            assert B_CLRF in line
+            assert B_CLRF in line
diff --git a/setup.py b/setup.py
@@ -47,9 +47,12 @@ def read(fname):
 tests_require = all_deps + [
     'moto[server]',
     'responses',
-    'boto3',
     'pytest',
     'pytest-rerunfailures',
+    'pytest_benchmark',
+    'awscli',
+    'pyopenssl',
+    'numpy',
 ]
 
 setup(

diff --git a/smart_open/ftp.py b/smart_open/ftp.py
@@ -86,7 +86,7 @@ def convert_transport_params_to_args(transport_params):
 def _connect(hostname, username, port, password, secure_connection, transport_params):
     kwargs = convert_transport_params_to_args(transport_params)
     if secure_connection:
-        ssl_context = ssl.create_default_context(purpose=ssl.Purpose.CLIENT_AUTH)
+        ssl_context = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH)
         ftp = FTP_TLS(context=ssl_context, **kwargs)
     else:
         ftp = FTP(**kwargs)

diff --git a/smart_open/tests/test_http.py b/smart_open/tests/test_http.py
@@ -21,24 +21,22 @@
 URL = 'http://localhost'
 HTTPS_URL = 'https://localhost'
 HEADERS = {
-    'Content-Length': str(len(BYTES)),
     'Accept-Ranges': 'bytes',
 }
 
 
 def request_callback(request, headers=HEADERS, data=BYTES):
-    try:
-        range_string = request.headers['range']
-    except KeyError:
-        return (200, headers, data)
+    headers = headers.copy()
+    range_string = request.headers.get('range', 'bytes=0-')
 
-    start, end = range_string.replace('bytes=', '').split('-', 1)
+    start, end = range_string.replace('bytes=', '', 1).split('-', 1)
     start = int(start)
-    if end:
-        end = int(end)
-    else:
-        end = len(data)
-    return (200, headers, data[start:end])
+    end = int(end) if end else len(data)
+
+    data = data[start:end]
+    headers['Content-Length'] = str(len(data))
+
+    return (200, headers, data)
 
 
 @unittest.skipIf(os.environ.get('TRAVIS'), 'This test does not work on TravisCI for some reason')