add chunck_upload function refs #36600

Makefile

@@ -32,7 +32,7 @@ endif
 
 test:
 ifndef CI
-	docker run -e CI='1' -e POD_UNIT_TEST_API_KEY=${POD_UNIT_TEST_API_KEY} -v ${CURDIR}:/src -w /src --rm ${DOCKER_IMG} make test
+	docker run -e CI='1' --network host -e POD_UNIT_TEST_API_KEY=${POD_UNIT_TEST_API_KEY} -v ${CURDIR}:/src -w /src --rm ${DOCKER_IMG} make test
 else
 	TOXENV=test tox -- ${PYTEST_ARGS}
 endif

README.md

@@ -47,14 +47,15 @@ response = pod_client.videos.post(data=data, files=files)
 
 `python3 python3-pip python3-requests make`
 
-# Running a pod test server
+# Running a local pod test server
 
 https://plmlab.math.cnrs.fr/docker-images/esup-pod
 
 ```
-git clone git@plmlab.math.cnrs.fr:docker-images/esup-pod.git && cd esup-pod
+git clone https://plmlab.math.cnrs.fr/docker-images/esup-pod.git && cd esup-pod
+chmod +x ./start.sh
 /bin/bash -c ./start.sh
-cd django_projects/podv2/ ; python manage.py migrate ; python manage.py runserver 0.0.0.0:808
+cd django_projects/podv2/ ; python manage.py migrate ; python manage.py runserver 0.0.0.0:8080
 ```
 
 login: pod

pod_client/pod_client.py

 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
+import hashlib
+import logging
+import math
+import os
+import requests
+import time
+
 from .resource import API_URLS, Resource
 
+logger = logging.getLogger('pod_client.resource')
+
 
 class PodClient:
     host_url = ''
@@ -16,3 +25,52 @@ class PodClient:
             raise Exception('Pod api key required')
         for resource_name in API_URLS.keys():
             self.__dict__[resource_name] = Resource(self.host_url, self.api_key, resource_name, site_id)
+
+    def chunked_upload(self, file_path, slug=None, progress_callback=None,
+                       progress_data=None, check_md5=True, timeout=3600):
+        url_prefix = '%s/api/' % self.host_url
+        chunk_size = 1000000
+        total_size = os.path.getsize(file_path)
+        chunks_count = math.ceil(total_size / chunk_size)
+        chunk_index = 0
+        start_offset = 0
+        end_offset = min(chunk_size, total_size) - 1
+        data = dict()
+        if check_md5:
+            md5sum = hashlib.md5()
+        begin = time.time()
+        with open(file_path, 'rb') as file_object:
+            while True:
+                chunk = file_object.read(chunk_size)
+                if not chunk:
+                    break
+                chunk_index += 1
+                logger.debug('Uploading chunk %s/%s.', chunk_index, chunks_count)
+                if check_md5:
+                    md5sum.update(chunk)
+                files = {'file': (os.path.basename(file_path), chunk)}
+                headers = {'Content-Range': 'bytes %s-%s/%s' % (start_offset, end_offset, total_size)}
+                response = requests.request('post', url_prefix + 'chunked_upload/', data=data, files=files, headers=headers, timeout=timeout)
+                if response.status_code != 200:
+                    raise Exception('Error {status_code}: "{text}"'.format(status_code=response.status_code, text=response.text))
+                response = response.json()
+                if progress_callback:
+                    pdata = progress_data or dict()
+                    progress_callback(0.9 * end_offset / total_size, **pdata)
+                if 'upload_id' not in data:
+                    data['upload_id'] = response['upload_id']
+                start_offset += chunk_size
+                end_offset = min(end_offset + chunk_size, total_size - 1)
+        bandwidth = total_size * 8 / ((time.time() - begin) * 1000000)
+        logger.debug('Upload finished, average bandwidth: %.2f Mbits/s', bandwidth)
+        if check_md5:
+            data['md5'] = md5sum.hexdigest()
+        else:
+            data['no_md5'] = 'yes'
+        if slug:
+            data['slug'] = slug
+        requests.request('post', url_prefix + 'chunked_upload/complete/', data=data, timeout=timeout)
+        if progress_callback:
+            pdata = progress_data or dict()
+            progress_callback(1., **pdata)
+        return data['upload_id']

pod_client/resource.py

 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
+import logging
 import os
 import requests
-import logging
+
+
+logger = logging.getLogger('pod_client.resource')
+
 
 API_URLS = {
     'users': '/rest/users/',
@@ -79,7 +83,7 @@ class Resource:
                 requests.ConnectionError,
                 requests.HTTPError,
                 requests.URLRequired) as e:
-            logging.error(e)
+            logger.error(e)
             response['success'] = False
             response['status_code'] = 0
             response['error'] = e

tests/test_pod_client.py

@@ -6,7 +6,7 @@ from pod_client.pod_client import PodClient
 from pod_client.resource import Resource, API_URLS
 
 
-HOST = 'http://pod.ubicast.net'
+HOST = 'http://localhost:8080'
 API_KEY = os.environ.get('POD_UNIT_TEST_API_KEY')
 VIDEO_PATH = 'tests/media/test.mp4'
 
@@ -57,3 +57,13 @@ class PodClientTest(TestCase):
         self.assertTrue(response['success'], response.get('error'))
         response = pod_client.videos.delete(resource_id=response['data']['id'])
         self.assertTrue(response['success'], response.get('error'))
+
+    def test_chunked_upload_video(self):
+        pod_client = PodClient(HOST, API_KEY)
+        data = {
+            'title': 'unittest video',
+            'type': pod_client.types.url(1),
+            'owner': pod_client.users.url(1)
+        }
+        response = pod_client.chunked_upload(VIDEO_PATH)
+        self.assertTrue(response['success'], response.get('error'))

tests/test_resource.py

@@ -4,7 +4,7 @@ import os
 from unittest import TestCase
 from pod_client.resource import Resource, API_URLS
 
-HOST = 'http://pod.ubicast.net'
+HOST = 'http://localhost:8080'
 API_KEY = os.environ.get('POD_UNIT_TEST_API_KEY')