From f0a942dfc6559e556eb48ff42b2dfd1ffda81e3f Mon Sep 17 00:00:00 2001 From: Tris Date: Sun, 4 Oct 2020 17:22:01 +1100 Subject: [PATCH] Added private links --- .gitignore | 5 +- Makefile | 11 ++- interface/forms.py | 7 +- .../migrations/0020_auto_20201003_2103.py | 23 ++++++ interface/models.py | 27 +++++-- .../templates/interface/resource_list.html | 2 +- interface/templates/interface/s3_upload.html | 9 ++- .../interface/submission_create.html | 6 +- .../interface/submission_detail.html | 4 +- .../templates/interface/submission_link.html | 18 +++++ .../templates/interface/submission_list.html | 2 +- interface/tests/test_submission.py | 49 ++++++++---- interface/urls.py | 2 + interface/views.py | 74 +++++++++++++------ 14 files changed, 187 insertions(+), 52 deletions(-) create mode 100644 interface/migrations/0020_auto_20201003_2103.py create mode 100644 interface/templates/interface/submission_link.html diff --git a/.gitignore b/.gitignore index b8e6050..417c0fc 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,6 @@ db.sqlite3 credentials polyphonic/settings.py env -text.* -static \ No newline at end of file +test.* +static +teststore \ No newline at end of file diff --git a/Makefile b/Makefile index 0496122..6636d90 100644 --- a/Makefile +++ b/Makefile @@ -24,4 +24,13 @@ static/fonts/Quicksand_Book.otf: wget -O quicksand.zip https://dl.dafont.com/dl/?f=quicksand mkdir -p static/fonts unzip -d static/fonts quicksand.zip - rm quicksand.zip \ No newline at end of file + rm quicksand.zip + +start_s3_storage: + test ! -f teststore/pid + mkdir -p teststore + MINIO_ACCESS_KEY=polyphonic_test_key MINIO_SECRET_KEY=polyphonic_secret minio server teststore & echo "$$!" > teststore/pid + cat teststore/pid + +stop_s3_storage: + kill `cat teststore/pid` && rm teststore/pid \ No newline at end of file diff --git a/interface/forms.py b/interface/forms.py index 4f6b578..d8f18e3 100644 --- a/interface/forms.py +++ b/interface/forms.py @@ -7,6 +7,11 @@ class CodeForm(forms.Form): passphrase = forms.CharField(max_length=32) class SubmissionForm(forms.ModelForm): + method = forms.ChoiceField(choices=( + ('upload', 'I need to upload a file'), + ('link', 'I have a link from my own cloud storage provider') + ), initial='upload') + class Meta: model = Submission - fields = ['name', 'instrument', 'notes'] \ No newline at end of file + fields = ['name', 'instrument', 'method', 'notes'] \ No newline at end of file diff --git a/interface/migrations/0020_auto_20201003_2103.py b/interface/migrations/0020_auto_20201003_2103.py new file mode 100644 index 0000000..7d594fd --- /dev/null +++ b/interface/migrations/0020_auto_20201003_2103.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1.1 on 2020-10-03 11:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('interface', '0019_project_owner'), + ] + + operations = [ + migrations.RenameField( + model_name='submission', + old_name='key', + new_name='url', + ), + migrations.AddField( + model_name='submission', + name='private', + field=models.BooleanField(default=False), + ), + ] diff --git a/interface/models.py b/interface/models.py index 3b68bce..77a1dab 100644 --- a/interface/models.py +++ b/interface/models.py @@ -9,10 +9,11 @@ import random import boto3 from datetime import datetime +from urllib.parse import urlparse import os.path -s3client = boto3.client('s3') +s3client = boto3.client('s3', **getattr(settings, 'S3_CREDENTIALS', {})) BUCKET = settings.AWS_BUCKET @@ -102,15 +103,29 @@ class Submission(models.Model): instrument = models.CharField(max_length=100, verbose_name="Instrument / Voice") notes = models.TextField(blank=True) complete = models.BooleanField(default=False) - key = models.CharField(max_length=512, blank=True) + url = models.CharField(max_length=512, blank=True) + private = models.BooleanField(default=False) - def presigned_url(self): - params = {'Bucket': BUCKET, 'Key': self.key} + @property + def download_url(self): + if not self.complete: + raise RuntimeError("Submission not complete") + + if self.private: + return self.url + + params = {'Bucket': BUCKET, 'Key': self.url} return s3client.generate_presigned_url('get_object', Params=params, ExpiresIn=3600) + @property + def download_name(self): + uri = urlparse(self.download_url) + _, name = os.path.split(uri.path) + return name or "" + def key_template(self): - return "{}_{}_{}_${{filename}}".format( - timezone.localtime(self.date).isoformat(timespec='seconds').replace(':', ''), + return "submissions/{}_{}_{}_${{filename}}".format( + timezone.localtime(self.date).isoformat(timespec='seconds').replace(':', '')[:17], slugify(self.name), slugify(self.instrument) ) diff --git a/interface/templates/interface/resource_list.html b/interface/templates/interface/resource_list.html index 620ab31..a0fa19f 100644 --- a/interface/templates/interface/resource_list.html +++ b/interface/templates/interface/resource_list.html @@ -24,7 +24,7 @@

{{ resource.name }} {% if download %} - + Download {% endif %} diff --git a/interface/templates/interface/s3_upload.html b/interface/templates/interface/s3_upload.html index a039228..c8f88b5 100644 --- a/interface/templates/interface/s3_upload.html +++ b/interface/templates/interface/s3_upload.html @@ -5,7 +5,7 @@
-

Select a file for upload

+

File upload

{% for field, value in upload.fields.items %} @@ -55,11 +55,11 @@ Dropzone.options.itemUpload = { autoProcessQueue: false, createImageThumbnails: false, //acceptedFiles: acceptFiles, - timeout: 3600000, + timeout: 3600000, maxFiles: 1, addRemoveLinks: true, maxFilesize: 500, - dictDefaultMessage: "Select a file to upload (max 500Mb)", + dictDefaultMessage: "Click to select a file to upload (max 500Mb)", dictFallbackMessage: "Your browser is old!, using alternative method", init: function() { let dz = this; @@ -79,6 +79,9 @@ Dropzone.options.itemUpload = { let s3_location = f.xhr.getResponseHeader("Location"); location.href = "{{ success_url }}?location=" + s3_location; }); + this.on('error', function(file, msg) { + status.html('There was an issue - please see troubleshooting'); + }); } }; diff --git a/interface/templates/interface/submission_create.html b/interface/templates/interface/submission_create.html index 65c6e6d..bd2a4a7 100644 --- a/interface/templates/interface/submission_create.html +++ b/interface/templates/interface/submission_create.html @@ -5,7 +5,10 @@

Excellent, you are ready to make a submission!

Please enter some basic information so we can identify your submission and - note anything that might be relevant. + note anything that might be relevant.
+ Most people will want to upload their + file directly but if you have your own cloud storage provider you can send a + public link to your submission instead.

@@ -13,6 +16,7 @@ {% csrf_token %} {{ form }}
+ Cancel
diff --git a/interface/templates/interface/submission_detail.html b/interface/templates/interface/submission_detail.html index 901e1f9..a9ce0d3 100644 --- a/interface/templates/interface/submission_detail.html +++ b/interface/templates/interface/submission_detail.html @@ -8,8 +8,8 @@ From:{{ submission.name }} Instrument:{{ submission.instrument }} Notes:{{ submission.notes }} - {% if download %} - Download:{{ submission.key }} + {% if can_download %} + Download:{{ submission.download_name }} {% endif %} diff --git a/interface/templates/interface/submission_link.html b/interface/templates/interface/submission_link.html new file mode 100644 index 0000000..339a7ef --- /dev/null +++ b/interface/templates/interface/submission_link.html @@ -0,0 +1,18 @@ +{% extends "interface/project_base.html" %} + +{% block page %} +
+

Link to cloud storage

+

Please paste the full link from your storage provider

+
+
+
+ {% csrf_token %} + {{ form }} +
+ Cancel + +
+
+
+{% endblock %} \ No newline at end of file diff --git a/interface/templates/interface/submission_list.html b/interface/templates/interface/submission_list.html index 95e036d..5089fe4 100644 --- a/interface/templates/interface/submission_list.html +++ b/interface/templates/interface/submission_list.html @@ -16,7 +16,7 @@ {{ submission.instrument }} - + {% endfor %} diff --git a/interface/tests/test_submission.py b/interface/tests/test_submission.py index 5b405b5..415cdea 100644 --- a/interface/tests/test_submission.py +++ b/interface/tests/test_submission.py @@ -3,28 +3,51 @@ from django.test import TestCase, Client from interface import models class SubmissionTestCase(TestCase): + + @staticmethod + def setUpTestData(): + e1 = models.Ensemble.objects.create(name='The Be Sharps', code="1234", passphrase='Homer') + e1.projects.create(name='Baby on Board') + e2 = models.Ensemble.objects.create(name='Lisa and the Bleeding Gums', code="2345", passphrase="Maggie") + e2.projects.create(name='Baker St') def setUp(self): self.client = Client() - def test_submission(self): - ensemble = models.Ensemble.objects.create(name="The Be Sharps", passphrase="Homer") - project = ensemble.projects.create(name='Baby on Board') - - response = self.client.post('/register', {'code': ensemble.code, 'passphrase': ensemble.passphrase}) + def test_submission_upload(self): + response = self.client.post('/register', {'code': '12-34', 'passphrase': 'Homer'}) self.assertRedirects(response, '/') - response = self.client.post(f"/projects/{project.pk}/submission", {'name': 'Ned', 'instrument': 'God'}) - #self.assertRedirects(response, '/projects/1/submission/1/upload') - - self.skipTest("Need to mock S3") + response = self.client.post(f"/projects/1/submission", {'name': 'Ned', 'instrument': 'Harp', 'method': 'upload'}) + self.assertRedirects(response, '/projects/1/submission/1/upload') response = self.client.get(response.url) upload = response.context['upload'] - self.assertEqual(upload['url'], f"https://{ensemble.bucket}.s3.amazonaws.com/") - self.assertRegex(upload['fields']['key'], r'^baby-on-board\/[0-9T\-]+_ned_god_\$\{filename\}$') + self.assertEqual(upload['url'], f"http://localhost:9000/{models.BUCKET}") + self.assertRegex(upload['fields']['key'], r'^baby-on-board\/submissions\/[0-9T\-]+_ned_harp_\$\{filename\}$') self.assertEqual(upload['fields']['success_action_redirect'], 'http://testserver/projects/1/submission/1/complete') self.assertEqual(models.Submission.objects.count(), 1) - self.assertRedirects(self.client.get(f"/projects/{project.pk}/submission/1/cancel"), '/projects/1') - self.assertEqual(models.Submission.objects.count(), 0) \ No newline at end of file + self.assertRedirects(self.client.get(f"/projects/1/submission/1/cancel"), '/projects/1') + self.assertEqual(models.Submission.objects.count(), 0) + + def test_submission_link(self): + response = self.client.post('/register', {'code': '12-34', 'passphrase': 'Homer'}) + self.assertRedirects(response, '/') + + response = self.client.post(f"/projects/1/submission", {'name': 'Ned', 'instrument': 'Harp', 'method': 'link'}) + self.assertRedirects(response, '/projects/1/submission/1/link') + + url = 'https://drive.google.com/a/path/to/a/video.mp4#g6e6e4a23' + + response = self.client.post(f"/projects/1/submission/1/link", {'url': url}) + self.assertRedirects(response, '/projects/1/submission/1') + + response = self.client.get('/projects/1/submission/1') + self.assertContains(response, "Thankyou for your submission") + + response = self.client.get('/projects/1') + self.assertContains(response, 'Ned') + + s = models.Submission.objects.get(pk=1) + self.assertEqual(s.download_url, url) \ No newline at end of file diff --git a/interface/urls.py b/interface/urls.py index f50461c..046bdff 100644 --- a/interface/urls.py +++ b/interface/urls.py @@ -18,8 +18,10 @@ urlpatterns = [ path('projects//submission', views.SubmissionCreateView.as_view(), name="submission_create"), path('projects//submission/', views.SubmissionDetailView.as_view(), name="submission_detail"), + path('projects//submission//link', views.SubmissionLinkView.as_view(), name="submission_link"), path('projects//submission//upload', views.SubmissionUploadView.as_view(), name="submission_upload"), path('projects//submission//cancel', views.SubmissionCancelView.as_view(), name="submission_cancel"), + path('projects//submission//complete', views.SubmissionCompleteView.as_view(), name="submission_complete"), path('projects//submissions', views.SubmissionListView.as_view(), name="submission_list"), path('projects//resources', views.ResourceListView.as_view(), name="resource_list"), diff --git a/interface/views.py b/interface/views.py index 5750dd8..da6b23c 100644 --- a/interface/views.py +++ b/interface/views.py @@ -2,7 +2,7 @@ from django.shortcuts import render, get_object_or_404, redirect, resolve_url from django.views.generic import TemplateView, View, RedirectView from django.views.generic.detail import DetailView, SingleObjectMixin from django.views.generic.list import ListView -from django.views.generic.edit import CreateView, UpdateView +from django.views.generic.edit import CreateView, UpdateView, FormView from django.views.generic.base import ContextMixin from django.http import HttpResponseRedirect from django.core.exceptions import SuspiciousOperation @@ -11,6 +11,7 @@ from django.contrib import auth from markdown2 import markdown from datetime import datetime from urllib.parse import urlparse, urlencode +import os.path from . import models, forms @@ -84,8 +85,7 @@ class S3UploadMixin(ProjectMixin): context['accept_files'] = self.accept_files return context -class S3CompleteMixin(View): - always_set = False +class S3CompleteView(SingleObjectMixin, RedirectView): def complete(self, key): self.object.key = key @@ -94,14 +94,14 @@ class S3CompleteMixin(View): def get(self, request, *args, **kwargs): self.object = self.get_object() - if self.always_set or not self.object.key: - if 'location' in request.GET: - uri = urlparse(request.GET['location']) - self.complete(uri.path[1:]) - elif 'key' in request.GET: - self.complete(request.GET['key']) - else: - raise KeyError("No key or location found") + if 'key' in request.GET: + self.complete(request.GET['key']) + elif 'location' in request.GET: + uri = urlparse(request.GET['location']) + _bucket, key = uri.path[1:].split('/', 1) + self.complete(key) + else: + raise KeyError("No key or location found") return super().get(request, *args, **kwargs) @@ -209,9 +209,10 @@ class WikiEditView(ProjectMixin, UpdateView): model = models.WikiPage fields = ['title', 'markdown'] -class SubmissionCreateView(ProjectMixin, CreateView): - model = models.Submission - fields = ['name', 'instrument', 'notes'] +class SubmissionCreateView(ProjectMixin, FormView): + #model = models.Submission + #fields = ['name', 'instrument', 'url', 'notes'] + form_class = forms.SubmissionForm template_name = "interface/submission_create.html" def form_valid(self, form): @@ -221,24 +222,33 @@ class SubmissionCreateView(ProjectMixin, CreateView): self.request.session['name'] = self.object.name self.request.session['instrument'] = self.object.instrument + + if form.cleaned_data['method'] == 'link': + return redirect('submission_link', project=self.object.project.pk, pk=self.object.pk) return redirect('submission_upload', project=self.object.project.pk, pk=self.object.pk) def get_initial(self): return { k: self.request.session.get(k) for k in ('name', 'instrument') } - -class SubmissionDetailView(ProjectMixin, S3CompleteMixin, DetailView): +class SubmissionCompleteView(ProjectMixin, S3CompleteView): model = models.Submission def complete(self, key): + self.object.url = key + self.object.private = False self.object.complete = True - super().complete(key) + self.object.save() + + def get_redirect_url(self, **kwargs): + return resolve_url('submission_detail', **self.kwargs) + +class SubmissionDetailView(ProjectMixin, DetailView): + model = models.Submission def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - if self.request.is_admin: - context['download'] = self.object.presigned_url() + context['can_download'] = self.request.is_admin return context class SubmissionUploadView(S3UploadMixin, DetailView): @@ -246,12 +256,35 @@ class SubmissionUploadView(S3UploadMixin, DetailView): model = models.Submission accept_files = "video/*" + def get_success_url(self): + return resolve_url('submission_complete', **self.kwargs) + + def get_cancel_url(self): + return resolve_url('submission_cancel', **self.kwargs) + +class SubmissionLinkView(ProjectMixin, UpdateView): + model = models.Submission + template_name = 'interface/submission_link.html' + fields = ['url'] + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + context['cancel_url'] = self.get_cancel_url() + return context + def get_success_url(self): return resolve_url('submission_detail', **self.kwargs) def get_cancel_url(self): return resolve_url('submission_cancel', **self.kwargs) + def form_valid(self, form): + self.object = form.save(commit=False) + self.object.complete = True + self.object.private = True + self.object.save() + return redirect(self.get_success_url()) + class SubmissionCancelView(ProjectMixin, SingleObjectMixin, View): model = models.Submission @@ -294,9 +327,8 @@ class ResourceUploadView(S3UploadMixin, DetailView): def get_cancel_url(self): return resolve_url('resource_list', project=self.kwargs['project']) -class ResourceCompleteView(S3CompleteMixin, SingleObjectMixin, RedirectView): +class ResourceCompleteView(ProjectMixin, S3CompleteView): model = models.Resource - always_set = True def get_redirect_url(self, **kwargs): return resolve_url('resource_list', project=self.kwargs['project'])