Skip to content
Snippets Groups Projects
Commit 53404663 authored by Stéphane Schoorens's avatar Stéphane Schoorens
Browse files

add chunck_upload function refs #36600

parent b487fd11
No related branches found
No related tags found
1 merge request!2support chuncked uploads, fixes #36600
......@@ -32,7 +32,7 @@ endif
test:
ifndef CI
docker run -e CI='1' -e POD_UNIT_TEST_API_KEY=${POD_UNIT_TEST_API_KEY} -v ${CURDIR}:/src -w /src --rm ${DOCKER_IMG} make test
docker run -e CI='1' --network host -e POD_UNIT_TEST_API_KEY=${POD_UNIT_TEST_API_KEY} -v ${CURDIR}:/src -w /src --rm ${DOCKER_IMG} make test
else
TOXENV=test tox -- ${PYTEST_ARGS}
endif
......@@ -47,14 +47,15 @@ response = pod_client.videos.post(data=data, files=files)
`python3 python3-pip python3-requests make`
# Running a pod test server
# Running a local pod test server
https://plmlab.math.cnrs.fr/docker-images/esup-pod
```
git clone git@plmlab.math.cnrs.fr:docker-images/esup-pod.git && cd esup-pod
git clone https://plmlab.math.cnrs.fr/docker-images/esup-pod.git && cd esup-pod
chmod +x ./start.sh
/bin/bash -c ./start.sh
cd django_projects/podv2/ ; python manage.py migrate ; python manage.py runserver 0.0.0.0:808
cd django_projects/podv2/ ; python manage.py migrate ; python manage.py runserver 0.0.0.0:8080
```
login: pod
......
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import hashlib
import logging
import math
import os
import requests
import time
from .resource import API_URLS, Resource
logger = logging.getLogger('pod_client.resource')
class PodClient:
host_url = ''
......@@ -16,3 +25,52 @@ class PodClient:
raise Exception('Pod api key required')
for resource_name in API_URLS.keys():
self.__dict__[resource_name] = Resource(self.host_url, self.api_key, resource_name, site_id)
def chunked_upload(self, file_path, slug=None, progress_callback=None,
progress_data=None, check_md5=True, timeout=3600):
url_prefix = '%s/api/' % self.host_url
chunk_size = 1000000
total_size = os.path.getsize(file_path)
chunks_count = math.ceil(total_size / chunk_size)
chunk_index = 0
start_offset = 0
end_offset = min(chunk_size, total_size) - 1
data = dict()
if check_md5:
md5sum = hashlib.md5()
begin = time.time()
with open(file_path, 'rb') as file_object:
while True:
chunk = file_object.read(chunk_size)
if not chunk:
break
chunk_index += 1
logger.debug('Uploading chunk %s/%s.', chunk_index, chunks_count)
if check_md5:
md5sum.update(chunk)
files = {'file': (os.path.basename(file_path), chunk)}
headers = {'Content-Range': 'bytes %s-%s/%s' % (start_offset, end_offset, total_size)}
response = requests.request('post', url_prefix + 'chunked_upload/', data=data, files=files, headers=headers, timeout=timeout)
if response.status_code != 200:
raise Exception('Error {status_code}: "{text}"'.format(status_code=response.status_code, text=response.text))
response = response.json()
if progress_callback:
pdata = progress_data or dict()
progress_callback(0.9 * end_offset / total_size, **pdata)
if 'upload_id' not in data:
data['upload_id'] = response['upload_id']
start_offset += chunk_size
end_offset = min(end_offset + chunk_size, total_size - 1)
bandwidth = total_size * 8 / ((time.time() - begin) * 1000000)
logger.debug('Upload finished, average bandwidth: %.2f Mbits/s', bandwidth)
if check_md5:
data['md5'] = md5sum.hexdigest()
else:
data['no_md5'] = 'yes'
if slug:
data['slug'] = slug
requests.request('post', url_prefix + 'chunked_upload/complete/', data=data, timeout=timeout)
if progress_callback:
pdata = progress_data or dict()
progress_callback(1., **pdata)
return data['upload_id']
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import logging
import os
import requests
import logging
logger = logging.getLogger('pod_client.resource')
API_URLS = {
'users': '/rest/users/',
......@@ -79,7 +83,7 @@ class Resource:
requests.ConnectionError,
requests.HTTPError,
requests.URLRequired) as e:
logging.error(e)
logger.error(e)
response['success'] = False
response['status_code'] = 0
response['error'] = e
......
......@@ -6,7 +6,7 @@ from pod_client.pod_client import PodClient
from pod_client.resource import Resource, API_URLS
HOST = 'http://pod.ubicast.net'
HOST = 'http://localhost:8080'
API_KEY = os.environ.get('POD_UNIT_TEST_API_KEY')
VIDEO_PATH = 'tests/media/test.mp4'
......@@ -57,3 +57,13 @@ class PodClientTest(TestCase):
self.assertTrue(response['success'], response.get('error'))
response = pod_client.videos.delete(resource_id=response['data']['id'])
self.assertTrue(response['success'], response.get('error'))
def test_chunked_upload_video(self):
pod_client = PodClient(HOST, API_KEY)
data = {
'title': 'unittest video',
'type': pod_client.types.url(1),
'owner': pod_client.users.url(1)
}
response = pod_client.chunked_upload(VIDEO_PATH)
self.assertTrue(response['success'], response.get('error'))
......@@ -4,7 +4,7 @@ import os
from unittest import TestCase
from pod_client.resource import Resource, API_URLS
HOST = 'http://pod.ubicast.net'
HOST = 'http://localhost:8080'
API_KEY = os.environ.get('POD_UNIT_TEST_API_KEY')
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment