Fix upload config compatibility and improve test cases (#447)

Author: lihsai0

.github/workflows/ci-test.yml

@@ -35,24 +35,37 @@ jobs:
             wget -qLO get-pip.py "$PIP_BOOTSTRAP_SCRIPT_PREFIX/$MAJOR.$MINOR/get-pip.py"
             python get-pip.py --user
           fi
+      - name: Setup mock server
+        shell: bash -el {0}
+        run: |
+          conda create -y -n mock-server python=3.10
+          conda activate mock-server
+          python3 --version
+          nohup python3 tests/mock_server/main.py --port 9000 > py-mock-server.log &
+          echo $! > mock-server.pid
+          conda deactivate
       - name: Install dependencies
         shell: bash -l {0}
         run: |
           python -m pip install --upgrade pip
           python -m pip install -I -e ".[dev]"
       - name: Run cases
-        shell: bash -l {0}
+        shell: bash -el {0}
         env:
           QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }}
           QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }}
           QINIU_TEST_BUCKET: ${{ secrets.QINIU_TEST_BUCKET }}
           QINIU_TEST_DOMAIN: ${{ secrets.QINIU_TEST_DOMAIN }}
           QINIU_TEST_ENV: "travis"
+          MOCK_SERVER_ADDRESS: "http://127.0.0.1:9000"
           PYTHONPATH: "$PYTHONPATH:."
         run: |
-          set -e
-          flake8 --show-source --max-line-length=160 .
-          py.test --cov qiniu
+          flake8 --show-source --max-line-length=160 ./qiniu
+          coverage run -m pytest ./test_qiniu.py ./tests/cases
           ocular --data-file .coverage
-          coverage run test_qiniu.py
           codecov
+          cat mock-server.pid | xargs kill
+      - name: Print mock server log
+        if: ${{ failure() }}
+        run: |
+          cat py-mock-server.log

CHANGELOG.md

@@ -1,4 +1,7 @@
 # Changelog
+## 7.13.1（2024-02-04）
+* 对象存储，修复上传部分配置项的兼容
+
 ## 7.13.0（2023-12-11）
 * 对象存储，新增支持归档直读存储
 * 对象存储，批量操作支持自动查询 rs 服务域名

qiniu/__init__.py

@@ -9,7 +9,7 @@
 
 # flake8: noqa
 
-__version__ = '7.13.0'
+__version__ = '7.13.1'
 
 from .auth import Auth, QiniuMacAuth
 

qiniu/compat.py

@@ -33,6 +33,7 @@
 # ---------
 
 if is_py2:
+    from urllib import urlencode  # noqa
     from urlparse import urlparse  # noqa
     import StringIO
     StringIO = BytesIO = StringIO.StringIO
@@ -60,7 +61,7 @@ def is_seekable(data):
             return False
 
 elif is_py3:
-    from urllib.parse import urlparse  # noqa
+    from urllib.parse import urlparse, urlencode  # noqa
     import io
     StringIO = io.StringIO
     BytesIO = io.BytesIO

qiniu/http/__init__.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import logging
 import platform
+import functools
 
 import requests
 from requests.adapters import HTTPAdapter
@@ -21,6 +22,19 @@
 )
 
 
+# compatibility with some config from qiniu.config
+def _before_send(func):
+    @functools.wraps(func)
+    def wrapper(self, *args, **kwargs):
+        if _session is None:
+            _init()
+        return func(self, *args, **kwargs)
+
+    return wrapper
+
+
+qn_http_client.send_request = _before_send(qn_http_client.send_request)
+
 _sys_info = '{0}; {1}'.format(platform.system(), platform.machine())
 _python_ver = platform.python_version()
 

qiniu/http/client.py

@@ -3,6 +3,7 @@
 
 import requests
 
+from qiniu.config import get_default
 from .response import ResponseInfo
 from .middleware import compose_middleware
 
@@ -14,6 +15,9 @@ def __init__(self, middlewares=None, send_opts=None):
         self.send_opts = {} if send_opts is None else send_opts
 
     def _wrap_send(self, req, **kwargs):
+        # compatibility with setting timeout by qiniu.config.set_default
+        kwargs.setdefault('timeout', get_default('connection_timeout'))
+
         resp = self.session.send(req.prepare(), **kwargs)
         return ResponseInfo(resp, None)
 

qiniu/http/response.py

@@ -46,7 +46,13 @@ def ok(self):
         return self.status_code // 100 == 2
 
     def need_retry(self):
-        if 0 < self.status_code < 500:
+        if 100 <= self.status_code < 500:
+            return False
+        if all([
+            self.status_code < 0,
+            self.exception is not None,
+            'BadStatusLine' in str(self.exception)
+        ]):
             return False
         # https://developer.qiniu.com/fusion/kb/1352/the-http-request-return-a-status-code
         if self.status_code in [

qiniu/services/storage/uploaders/abc/resume_uploader_base.py

@@ -145,6 +145,7 @@ def initial_parts(
         data_size,
         modify_time,
         part_size,
+        file_name,
         **kwargs
     ):
         """
@@ -157,6 +158,7 @@ def initial_parts(
         data_size: int
         modify_time: int
         part_size: int
+        file_name: str
         kwargs: dict
 
         Returns

qiniu/services/storage/uploaders/abc/uploader_base.py

@@ -84,6 +84,7 @@ def _get_regions(self):
         if self.regions:
             return self.regions
 
+        # handle compatibility for default_zone
         default_region = config.get_default('default_zone')
         if default_region:
             self.regions = [default_region]
@@ -108,11 +109,14 @@ def _get_up_hosts(self, access_key=None):
         if not regions:
             raise ValueError('No region available.')
 
-        if regions[0].up_host and regions[0].up_host_backup:
-            return [
-                regions[0].up_host,
-                regions[0].up_host_backup
-            ]
+        # get up hosts in region
+        up_hosts = [
+            regions[0].up_host,
+            regions[0].up_host_backup
+        ]
+        up_hosts = [h for h in up_hosts if h]
+        if up_hosts:
+            return up_hosts
 
         # this is correct, it does return hosts. bad function name by legacy
         return regions[0].get_up_host(

qiniu/services/storage/uploaders/resume_uploader_v1.py

@@ -1,4 +1,5 @@
 import logging
+import math
 from collections import namedtuple
 from concurrent import futures
 from io import BytesIO
@@ -65,7 +66,11 @@ def _recover_from_record(
             record_context = [
                 ctx
                 for ctx in record_context
-                if ctx.get('expired_at', 0) > now
+                if (
+                    ctx.get('expired_at', 0) > now and
+                    ctx.get('part_no', None) and
+                    ctx.get('ctx', None)
+                )
             ]
 
         # assign to context
@@ -173,6 +178,7 @@ def initial_parts(
         data=None,
         modify_time=None,
         data_size=None,
+        file_name=None,
         **kwargs
     ):
         """
@@ -184,6 +190,7 @@ def initial_parts(
         data
         modify_time
         data_size
+        file_name
 
         kwargs
 
@@ -222,7 +229,8 @@ def initial_parts(
         )
 
         # try to recover from record
-        file_name = path.basename(file_path) if file_path else None
+        if not file_name and file_path:
+            file_name = path.basename(file_path)
         context = self._recover_from_record(
             file_name,
             key,
@@ -275,7 +283,10 @@ def upload_parts(
 
         # initial upload state
         part, resp = None, None
-        uploaded_size = 0
+        uploaded_size = context.part_size * len(context.parts)
+        if math.ceil(data_size / context.part_size) in [p.part_no for p in context.parts]:
+            # if last part uploaded, should correct the uploaded size
+            uploaded_size += (data_size % context.part_size) - context.part_size
         lock = Lock()
 
         if not self.concurrent_executor:
@@ -469,6 +480,7 @@ def upload(
             up_token,
             key,
             file_path=file_path,
+            file_name=file_name,
             data=data,
             data_size=data_size,
             modify_time=modify_time,

qiniu/services/storage/uploaders/resume_uploader_v2.py

@@ -1,3 +1,4 @@
+import math
 from collections import namedtuple
 from concurrent import futures
 from io import BytesIO
@@ -65,7 +66,7 @@ def _recover_from_record(
         if (
             not record_upload_id or
             record_modify_time != context.modify_time or
-            record_expired_at > time()
+            record_expired_at < time()
         ):
             return context
 
@@ -80,6 +81,10 @@ def _recover_from_record(
                     etag=p['etag']
                 )
                 for p in record_etags
+                if (
+                    p.get('partNumber', None) and
+                    p.get('etag', None)
+                )
             ],
             resumed=True
         )
@@ -105,7 +110,7 @@ def _set_to_record(self, file_name, key, context):
             'etags': [
                 {
                     'etag': part.etag,
-                    'part_no': part.part_no
+                    'partNumber': part.part_no
                 }
                 for part in context.parts
             ]
@@ -170,6 +175,7 @@ def initial_parts(
         data_size=None,
         modify_time=None,
         part_size=None,
+        file_name=None,
         **kwargs
     ):
         """
@@ -183,6 +189,7 @@ def initial_parts(
         data_size: int
         modify_time: int
         part_size: int
+        file_name: str
         kwargs
 
         Returns
@@ -222,7 +229,8 @@ def initial_parts(
         )
 
         # try to recover from record
-        file_name = path.basename(file_path) if file_path else None
+        if not file_name and file_path:
+            file_name = path.basename(file_path)
         context = self._recover_from_record(
             file_name,
             key,
@@ -307,7 +315,10 @@ def upload_parts(
 
         # initial upload state
         part, resp = None, None
-        uploaded_size = 0
+        uploaded_size = context.part_size * len(context.parts)
+        if math.ceil(data_size / context.part_size) in [p.part_no for p in context.parts]:
+            # if last part uploaded, should correct the uploaded size
+            uploaded_size += (data_size % context.part_size) - context.part_size
         lock = Lock()
 
         if not self.concurrent_executor:
@@ -500,6 +511,7 @@ def upload(
             up_token,
             key,
             file_path=file_path,
+            file_name=file_name,
             data=data,
             data_size=data_size,
             modify_time=modify_time,