Merge pull request 'GdriveFolder code' (#12) from gdrive into master

Reviewed-on: #12

.gitignore

@@ -1,10 +1,13 @@
 __pycache__
 *.pyc
 *.sqlite3
+*.swp
 credentials.json
 credentials
 local_settings.py
 .coverage
+Session.vim
+poetry.lock
 /env
 /data
 /old
@@ -13,3 +16,5 @@ local_settings.py
 /cache
 /local_storage
 /media
+/index
+/dist

Makefile

@@ -1,20 +1,25 @@
 PYTHON=env/bin/python
 DROPZONE=5.7.0
 
-test:
+test: check
-	coverage run --include "app/*" --omit "*/migrations/*" app/manage.py test app
+	poetry run coverage run --include "app/*" --omit "*/migrations/*" app/manage.py test app
-	coverage html
+	poetry run coverage html
-	coverage report
+	poetry run coverage report
+
+check:
+	poetry run ruff check app
+
+build:
+	poetry build
 
 dev-setup:
-	env/bin/pip install -r requirements.txt
+	poetry install --with=dev
-	env/bin/pip install -r dev-requirements.txt
+	poetry run manage migrate
-	${PYTHON} manage.py migrate
+	poetry run manage createsuperuser --username admin --email admin@localhost
-	${PYTHON} manage.py createsuperuser --username admin --email admin@localhost
 
 upgrade:
-	${PYTHON} manage.py migrate
+	poetry run manage migrate
-	${PYTHON} manage.py collectstatic
+	poetry run manage collectstatic
 	${MAKE} libraries
 
 libraries: static/dropzone static/fonts/Quicksand_Book.otf

README.md

@@ -6,20 +6,23 @@ No registration required for ensemble participants - just a one time code and pa
 
 ### Library App
 
-Store all your scores on your own cloud account (Amazon S3, Google Files etc).  Tag up the scores so you can generate
+Store all your scores on your own cloud account (Amazon S3, Google Files etc).
-custom part sets and assign them to projects so people can easily print just their parts.
+Tag up the scores so you can generate custom part sets and assign them to
+projects so people can easily print just their parts.
 
 ### Submissions App
 
-Accept video/audio submissions direct to your cloud storage.  Was developed and used during 2020 lockdown period for
+Accept video/audio submissions direct to your cloud storage.  Was developed and
-virtual choirs/orchestras but could have more uses.
+used during 2020 lockdown period for virtual choirs/orchestras but could have more uses.
 
 ### S3 Setup
 
 #### Bucket setup [virtual-orchestra]
+
 Default block public access
 
 Permissions -> CORS
+
 ```xml
 <?xml version="1.0" encoding="UTF-8"?>
 <CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
@@ -40,6 +43,7 @@ Permissions -> CORS
 
 User
 Create with programatic access (copy keys) and an inline policy for the bucket.
+
 ```json
 {
     "Version": "2012-10-17",
@@ -64,3 +68,4 @@ Create with programatic access (copy keys) and an inline policy for the bucket.
 ```
 
 3.
+

TODO.md

@@ -0,0 +1,19 @@
+## Polyphonic TODO
+
+## Core interface
+
+* Shift from crispy forms to native component templates
+* Make long running calls async (Django 5)
+* Deprecate Django 4 portions
+
+### Library App
+
+* Remove music tags and replace with strings vn1 -> 'Violin 1'
+* GDrive selector
+* Move upload to modal from 'Upload' button
+* Tagging app - migrate to AlpineJS
+* Allow other tags (movements, sections, pieces)
+
+### Submissions App
+
+* None currently pending

app/interface/admin.py

@@ -2,26 +2,31 @@ from django.contrib import admin
 
 from . import models
 
+
 class EnsembleAdmin(admin.ModelAdmin):
-    list_display = ['name', 'slug']
+    list_display = ["name", "slug"]
+
 
 class ModuleInline(admin.StackedInline):
     model = models.Module
     extra = 0
 
-class ProjectAdmin(admin.ModelAdmin):
 
-    list_display = ['name', 'ensemble', 'event_date', 'active']
+class ProjectAdmin(admin.ModelAdmin):
-    list_filter = ['ensemble', 'active']
+    list_display = ["name", "ensemble", "event_date", "active"]
+    list_filter = ["ensemble", "active"]
     inlines = [ModuleInline]
 
+
 class ResourceAdmin(admin.ModelAdmin):
-    list_display = ['name', 'media_type', 'project']
+    list_display = ["name", "media_type", "project"]
-    list_filter = ['project']
+    list_filter = ["project"]
+
 
 class WikiPageAdmin(admin.ModelAdmin):
-    list_display = ['title', 'project']
+    list_display = ["title", "project"]
-    list_filter = ['project']
+    list_filter = ["project"]
+
 
 admin.site.register(models.Ensemble, EnsembleAdmin)
 admin.site.register(models.Project, ProjectAdmin)

app/interface/apps.py

@@ -2,4 +2,4 @@ from django.apps import AppConfig
 
 
 class InterfaceConfig(AppConfig):
-    name = 'interface'
+    name = "interface"

app/interface/fields.py

@@ -1,4 +1,5 @@
 from crispy_forms.layout import Field
 
+
 class BulmaFileUpload(Field):
-    template = 'bulma/file_upload.html'
+    template = "bulma/file_upload.html"

app/interface/forms.py

@@ -1,62 +1,76 @@
 from django import forms
 from crispy_forms.helper import FormHelper
-from crispy_forms.layout import Submit, HTML, Div
+from crispy_forms.layout import Submit, HTML
 from crispy_bulma.layout import FormGroup
 
 from . import models, fields
 
-class BaseForm(forms.Form):
 
+class BaseForm(forms.Form):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.helper = self.get_form_helper()
 
     def get_form_helper(self):
         helper = FormHelper(self)
-        #helper.add_input(Submit('submit', 'Submit', css_class='button is-link'))
+        # helper.add_input(Submit('submit', 'Submit', css_class='button is-link'))
-        #helper.layout.subm append(HTML('<a class="button is-light">Cancel</a>'))
+        # helper.layout.subm append(HTML('<a class="button is-light">Cancel</a>'))
-        #print(helper.layout)
+        # print(helper.layout)
-        helper.layout.append(FormGroup(
+        helper.layout.append(
-            Submit('submit', 'Save', css_class="button is-primary"),
+            FormGroup(
-            HTML('{% if view.cancel_url %}<div class="control"><a href="{{ view.cancel_url }}" class="button is-light">Cancel</a></div>{% endif %}')
+                Submit("submit", "Save", css_class="button is-primary"),
-        ))
+                HTML(
+                    '{% if view.cancel_url %}<div class="control"><a href="{{ view.cancel_url }}" class="button is-light">Cancel</a></div>{% endif %}'
+                ),
+            )
+        )
         return helper
 
 
 class ProjectForm(forms.ModelForm, BaseForm):
-
     class Meta:
         model = models.Project
-        fields = ['name', 'description', 'modules', 'event_date']
+        fields = ["name", "description", "modules", "event_date"]
-        #widgets = {
+        # widgets = {
         #    'event_date': forms.DateTimeInput(attrs={'type': 'date'})
-        #}
+        # }
+
+    modules = forms.MultipleChoiceField(
+        choices=[(x, x.title()) for x in models.settings.POLYPHONIC_MODULES],
+        widget=forms.CheckboxSelectMultiple,
+        required=False,
+    )
 
-    modules = forms.MultipleChoiceField(choices=[ (x, x.title()) for x in models.settings.POLYPHONIC_MODULES ],
-                                        widget=forms.CheckboxSelectMultiple, required=False)
 
 class ResourceForm(forms.ModelForm, BaseForm):
-
     class Meta:
         model = models.Resource
-        fields = ['name', 'media_type', 'description', 'file']
+        fields = ["name", "media_type", "description", "file"]
 
     def get_form_helper(self):
         helper = super().get_form_helper()
         helper[3].wrap(fields.BulmaFileUpload)
         return helper
 
-class WikiForm(forms.ModelForm, BaseForm):
 
+class WikiForm(forms.ModelForm, BaseForm):
     class Meta:
         model = models.WikiPage
-        fields = ['title', 'markdown']
+        fields = ["title", "markdown"]
+
 
 class CodeForm(BaseForm):
-    code = forms.CharField(max_length=14,
+    code = forms.CharField(
-        widget=forms.TextInput(attrs={'placeholder': 'xxx-xxx-xxx', 'inputmode': 'numeric'}))
+        max_length=14,
+        widget=forms.TextInput(
+            attrs={"placeholder": "xxx-xxx-xxx", "inputmode": "numeric"}
+        ),
+    )
     passphrase = forms.CharField(max_length=32)
 
+
 class ResourceUploadForm(forms.Form):
     pass
+
+
 #    file = S3UploadField()

app/interface/models.py

@@ -8,17 +8,16 @@ from byostorage.user import BYOStorage
 
 import random
 
-from urllib.parse import urlparse
 
-import os.path
 from .utils import sign_data
 
 MEDIA_TYPES = [
-    ('audio', "Audio"),
+    ("audio", "Audio"),
-    ('video', "Video"),
+    ("video", "Video"),
-    ('general', "General"),
+    ("general", "General"),
 ]
 
+
 def rough_date(d):
     if not d:
         return False, "sometime..."
@@ -27,22 +26,22 @@ def rough_date(d):
     if in_past:
         days = abs(days)
     if days == 0:
-        m = int((d-timezone.now()).seconds/60)
+        m = int((d - timezone.now()).seconds / 60)
         if m > 60:
             return in_past, "{0:d} hours".format(int(m / 60))
         return in_past, "{0:d} minutes!".format(int(m % 60))
     if days >= 14:
-        return in_past, "{0:d} weeks".format(int(days/7))
+        return in_past, "{0:d} weeks".format(int(days / 7))
     if days >= 7:
         return in_past, "{0:d} weeks, {1:d} days".format(int(days / 7), int(days % 7))
     return in_past, f"{days} days"
 
 
 def generate_code(length=9):
-    return "".join([ random.choice('0123456789') for _ in range(length) ])
+    return "".join([random.choice("0123456789") for _ in range(length)])
+
 
 class EnsembleQuerySet(models.QuerySet):
-    
     def for_user(self, user, ensemble_keys=[], project_keys=[]):
         if user.is_superuser:
             return self
@@ -54,26 +53,35 @@ class EnsembleQuerySet(models.QuerySet):
 
         return self.filter(f).distinct()
 
-class Ensemble(models.Model):
-    ''' A group that plays together
 
-    '''
+class Ensemble(models.Model):
-    name = models.CharField(max_length=100,
+    """A group that plays together"""
-            help_text="Display name")
+
-    slug = models.SlugField(max_length=100, editable=False, unique=True,
+    name = models.CharField(max_length=100, help_text="Display name")
-            help_text="Short name for the ensemble - used for folders")
+    slug = models.SlugField(
-    admins = models.ManyToManyField('auth.User', related_name='ensembles')
+        max_length=100,
-    details = models.TextField(blank=True, 
+        editable=False,
-            help_text="Description of the ensemble (markdown)")
+        unique=True,
-    storage = models.ForeignKey('byostorage.UserStorage', null=True, on_delete=models.SET_NULL,
+        help_text="Short name for the ensemble - used for folders",
-            help_text="Default storage for this ensemble")
+    )
-    nonce = models.SmallIntegerField(default=1,
+    admins = models.ManyToManyField("auth.User", related_name="ensembles")
-            help_text="Increment this to reset the authentication links")
+    details = models.TextField(
+        blank=True, help_text="Description of the ensemble (markdown)"
+    )
+    storage = models.ForeignKey(
+        "byostorage.UserStorage",
+        null=True,
+        on_delete=models.SET_NULL,
+        help_text="Default storage for this ensemble",
+    )
+    nonce = models.SmallIntegerField(
+        default=1, help_text="Increment this to reset the authentication links"
+    )
 
     objects = EnsembleQuerySet.as_manager()
 
     class Meta:
-        ordering = ('slug', )
+        ordering = ("slug",)
 
     def active_projects(self):
         return self.projects.active().current()
@@ -83,7 +91,7 @@ class Ensemble(models.Model):
             return False
         if user.is_superuser:
             return True
-        return user.pk in self.admins.values_list('pk', flat=True)
+        return user.pk in self.admins.values_list("pk", flat=True)
 
     def save(self, **kwargs):
         if not self.slug:
@@ -91,17 +99,21 @@ class Ensemble(models.Model):
         super(Ensemble, self).save(**kwargs)
 
     def get_absolute_url(self):
-        return resolve_url('ensemble_detail', ensemble=self.slug)
+        return resolve_url("ensemble_detail", ensemble=self.slug)
 
     def auth(self):
-        return sign_data(f'{self.pk}-{self.nonce}', 12)
+        return sign_data(f"{self.pk}-{self.nonce}", 12)
 
     def __str__(self):
         return self.name
 
+
 class ProjectQuerySet(models.QuerySet):
     def current(self):
-        return self.filter(models.Q(event_date__gte=(timezone.now()-timezone.timedelta(7))) | models.Q(event_date=None))
+        return self.filter(
+            models.Q(event_date__gte=(timezone.now() - timezone.timedelta(7)))
+            | models.Q(event_date=None)
+        )
 
     def active(self):
         return self.filter(active=True)
@@ -117,23 +129,26 @@ class ProjectQuerySet(models.QuerySet):
 
         return self.filter(f)
 
+
 class Project(models.Model):
-    ''' A Project linked to an ensemble
+    """A Project linked to an ensemble"""
-    '''
+
     name = models.CharField(max_length=100)
-    ensemble = models.ForeignKey(Ensemble, related_name='projects', on_delete=models.CASCADE, null=True)
+    ensemble = models.ForeignKey(
-    description = models.TextField(blank=True, 
+        Ensemble, related_name="projects", on_delete=models.CASCADE, null=True
-            help_text="Markdown format")
+    )
+    description = models.TextField(blank=True, help_text="Markdown format")
     active = models.BooleanField(default=True)
-    event_date =models.DateTimeField(null=True, blank=True)
+    event_date = models.DateTimeField(null=True, blank=True)
     owner = models.CharField(max_length=255, blank=True)
-    nonce = models.SmallIntegerField(default=1,
+    nonce = models.SmallIntegerField(
-            help_text="Increment this to reset the authentication links")
+        default=1, help_text="Increment this to reset the authentication links"
+    )
 
     objects = ProjectQuerySet.as_manager()
 
     class Meta:
-        ordering = ['active', 'event_date']
+        ordering = ["active", "event_date"]
 
     @property
     def days(self):
@@ -162,62 +177,74 @@ class Project(models.Model):
 
     @property
     def active_modules(self):
-        return self.modules.values_list('name', flat=True)
+        return self.modules.values_list("name", flat=True)
 
     def get_absolute_url(self):
-        return resolve_url('project_detail', project=self.pk)
+        return resolve_url("project_detail", project=self.pk)
 
     def auth(self):
-        return sign_data(f'{self.pk}-{self.nonce}', 12)
+        return sign_data(f"{self.pk}-{self.nonce}", 12)
 
     def __str__(self):
         return self.name
 
+
 class Module(models.Model):
-    ''' Enable modules on a oriject
+    """Enable modules on a oriject"""
-    '''
+
-    name = models.SlugField(max_length=20, choices=[ (x, x.title()) for x in settings.POLYPHONIC_MODULES ])
+    name = models.SlugField(
-    project = models.ForeignKey(Project, related_name="modules", on_delete=models.CASCADE)
+        max_length=20, choices=[(x, x.title()) for x in settings.POLYPHONIC_MODULES]
+    )
+    project = models.ForeignKey(
+        Project, related_name="modules", on_delete=models.CASCADE
+    )
 
     def __str__(self):
         return self.name
 
+
 def resource_key(resource, filename):
-    return f'{resource.project.folder}/resources/{filename}'
+    return f"{resource.project.folder}/resources/{filename}"
+
 
 class Resource(models.Model):
-    ''' A viewable file resource attached to a project
+    """A viewable file resource attached to a project
 
     e.g PDF instructions, MP3 backing track
-    '''
+    """
-    project = models.ForeignKey(Project, related_name='resources', on_delete=models.CASCADE)
+
+    project = models.ForeignKey(
+        Project, related_name="resources", on_delete=models.CASCADE
+    )
     name = models.CharField(max_length=100)
     description = models.TextField(blank=True)
     file = models.FileField(storage=BYOStorage(), upload_to=resource_key)
-    media_type = models.CharField(max_length=10, choices=MEDIA_TYPES, default='*')
+    media_type = models.CharField(max_length=10, choices=MEDIA_TYPES, default="*")
     visible = models.BooleanField(default=True)
 
     class Meta:
-        ordering = ['-visible', '-pk']
+        ordering = ["-visible", "-pk"]
 
     def accept(self):
-        if self.media_type == 'general':
+        if self.media_type == "general":
             return ".*"
         return f"{self.media_type}/*"
 
     def __str__(self):
         return self.name
 
-class WikiPage(models.Model):
-    ''' An editable wiki page for the project in markdown format
 
-    '''
+class WikiPage(models.Model):
-    project = models.ForeignKey(Project, related_name='wiki_pages', on_delete=models.CASCADE)
+    """An editable wiki page for the project in markdown format"""
+
+    project = models.ForeignKey(
+        Project, related_name="wiki_pages", on_delete=models.CASCADE
+    )
     title = models.CharField(max_length=255)
     markdown = models.TextField()
 
     def get_absolute_url(self):
-        return resolve_url('wiki', project=self.project_id, pk=self.pk)
+        return resolve_url("wiki", project=self.project_id, pk=self.pk)
 
     def __str__(self):
         return self.title

app/interface/templatetags/path_filters.py

@@ -3,7 +3,9 @@ import os.path
 
 register = template.Library()
 
+
 def basename(value):
     return os.path.basename(value)
 
-register.filter('basename', basename)
+
+register.filter("basename", basename)

app/interface/templatetags/polyphonic.py

@@ -3,12 +3,16 @@ from django.utils import timesince
 
 register = template.Library()
 
+
 def roughtimesince(value):
     return timesince.timesince(value, depth=1)
 
-register.filter('roughtimesince', roughtimesince)
+
+register.filter("roughtimesince", roughtimesince)
+
 
 def roughtimeuntil(value):
     return timesince.timeuntil(value, depth=1)
 
-register.filter('roughtimeuntil', roughtimeuntil)
+
+register.filter("roughtimeuntil", roughtimeuntil)

app/interface/templatetags/url_tools.py

@@ -2,6 +2,7 @@ from django import template
 
 register = template.Library()
 
+
 @register.simple_tag(takes_context=True)
 def url_update(context, **kwargs):
     params = context.request.GET.copy()

app/interface/tests/__init__.py

@@ -1,11 +1,11 @@
 from django.test import TestCase
-from interface import models, utils
+from interface import models
 from django.contrib.auth.models import User
 from django.utils import timezone
 from datetime import timedelta
 
-class AccessTestCase(TestCase):
 
+class AccessTestCase(TestCase):
     USERS = ()
 
     ENSEMBLES = ()
@@ -16,16 +16,15 @@ class AccessTestCase(TestCase):
 
     @classmethod
     def setUpTestData(cls):
-
         cls.users = {}
         for details in cls.USERS:
-            cls.users[details['username']] = User.objects.create_user(**details)
+            cls.users[details["username"]] = User.objects.create_user(**details)
 
         now = timezone.now()
 
         cls.ensembles = {}
         for details in cls.ENSEMBLES:
-            admins = details.pop('admins', [])
+            admins = details.pop("admins", [])
             obj = models.Ensemble.objects.create(**details)
             for admin in admins:
                 obj.admins.add(cls.users[admin])
@@ -33,37 +32,40 @@ class AccessTestCase(TestCase):
 
         cls.projects = {}
         for details in cls.PROJECTS:
-            when = details.pop('when', 0)
+            when = details.pop("when", 0)
-            ensemble = details.pop('ensemble')
+            ensemble = details.pop("ensemble")
-            details['event_date'] = now + timedelta(days=when) if when else None
+            details["event_date"] = now + timedelta(days=when) if when else None
             obj = cls.ensembles[ensemble].projects.create(**details)
-            cls.projects[details['name']] = obj
+            cls.projects[details["name"]] = obj
 
         return
 
     def test_protected_views(self):
+        self.assertAccess({x: False for x in self.PROTECTED_URLS})
 
-        self.assertAccess({ x: False for x in self.PROTECTED_URLS })
+        if "admin" in self.users:
-
+            self.client.force_login(self.users["admin"])
-        if 'admin' in self.users:
+            self.assertAccess({x: True for x in self.PROTECTED_URLS})
-            self.client.force_login(self.users['admin'])
-            self.assertAccess({ x: True for x in self.PROTECTED_URLS })
 
     def login(self, user, passwd):
-        response = self.client.post('/login', {'username': user, 'password': passwd})
+        response = self.client.post("/login", {"username": user, "password": passwd})
         self.assertEqual(response.status_code, 302, f"Failed to login as {user}")
 
     def authorize(self, model, **kwargs):
         object = model.objects.get(**kwargs)
-        response = self.client.get(f'{object.get_absolute_url()}?auth={object.auth()}')
+        response = self.client.get(f"{object.get_absolute_url()}?auth={object.auth()}")
         self.assertEqual(response.status_code, 302)
 
     def assertAccess(self, urls):
         for url, expected in urls.items():
             response = self.client.get(url)
-            self.assertEqual(response.status_code == 200, expected, f"Expected {expected} for {url} (status: {response.status_code})")
+            self.assertEqual(
+                response.status_code == 200,
+                expected,
+                f"Expected {expected} for {url} (status: {response.status_code})",
+            )
 
-    def assertObjectList(self, response, expected, element='name'):
+    def assertObjectList(self, response, expected, element="name"):
         self.assertEqual(response.status_code, 200, "No result returned")
-        objects = response.context['object_list'].values_list(element, flat=True)
+        objects = response.context["object_list"].values_list(element, flat=True)
         self.assertEqual(list(objects), expected)

app/interface/tests/test_access.py

@@ -1,183 +1,215 @@
-from django.test import TestCase, Client
 
-from interface import models, utils
+from interface import models
-from django.contrib.auth.models import User
 
 from . import AccessTestCase
 
-class InterfaceAccessTestCase(AccessTestCase):
 
+class InterfaceAccessTestCase(AccessTestCase):
     USERS = (
-        {'username': 'admin', 'password': 'secret', 'is_superuser': True, 'is_staff': True},
+        {
-        {'username': 'homer', 'password': 'maggie'},
+            "username": "admin",
+            "password": "secret",
+            "is_superuser": True,
+            "is_staff": True,
+        },
+        {"username": "homer", "password": "maggie"},
     )
 
     ENSEMBLES = (
-        {'name': 'The Be Sharps', 'slug': 'be-sharps', 'admins': ['homer']},
+        {"name": "The Be Sharps", "slug": "be-sharps", "admins": ["homer"]},
-        {'name': 'Lisa & the Bleeding Gums', 'slug': 'bleeding-gums'},
+        {"name": "Lisa & the Bleeding Gums", "slug": "bleeding-gums"},
-        {'name': 'Party Posse'},
+        {"name": "Party Posse"},
     )
 
     PROJECTS = (
-        {'name': 'Baker St', 'ensemble': 'bleeding-gums', 'when': -12},
+        {"name": "Baker St", "ensemble": "bleeding-gums", "when": -12},
-        {'name': 'Navy Recruitment Day', 'ensemble': 'party-posse', 'when': 6},
+        {"name": "Navy Recruitment Day", "ensemble": "party-posse", "when": 6},
-        {'name': 'Barbershop Contest', 'ensemble': 'be-sharps', 'when': 28},
+        {"name": "Barbershop Contest", "ensemble": "be-sharps", "when": 28},
-        {'name': 'Open Mic Night', 'ensemble': 'bleeding-gums', 'when': 1 },
+        {"name": "Open Mic Night", "ensemble": "bleeding-gums", "when": 1},
-        {'name': 'Current Repertoire', 'ensemble': 'be-sharps'},
+        {"name": "Current Repertoire", "ensemble": "be-sharps"},
     )
 
     PROTECTED_URLS = (
-        '/ensembles/be-sharps',
+        "/ensembles/be-sharps",
-        '/ensembles/be-sharps/new-project',
+        "/ensembles/be-sharps/new-project",
-
+        "/projects/3",
-        '/projects/3',
+        "/projects/3/resources",
-        '/projects/3/resources',
+        "/projects/3/resources/add",
-        '/projects/3/resources/add',
+        "/admin/interface/ensemble/",
-
+        "/admin/interface/project/",
-        '/admin/interface/ensemble/',
+        "/admin/interface/resource/",
-        '/admin/interface/project/',
+        "/admin/interface/wikipage/",
-        '/admin/interface/resource/',
-        '/admin/interface/wikipage/',
     )
 
     def test_bad_login(self):
-        with self.assertRaisesMessage(self.failureException, 'Failed to login as admin'):
+        with self.assertRaisesMessage(
-            self.login('admin', 'admin')
+            self.failureException, "Failed to login as admin"
+        ):
+            self.login("admin", "admin")
 
     def test_superuser_ensembles(self):
-        self.login('admin', 'secret')
+        self.login("admin", "secret")
-        response = self.client.get('/ensembles')
+        response = self.client.get("/ensembles")
-        self.assertObjectList(response, ['The Be Sharps', 'Lisa & the Bleeding Gums', 'Party Posse'])
+        self.assertObjectList(
-        self.assertContains(response, 'Django Admin')
+            response, ["The Be Sharps", "Lisa & the Bleeding Gums", "Party Posse"]
+        )
+        self.assertContains(response, "Django Admin")
 
     def test_superuser_ensemble_permissions(self):
-        self.login('admin', 'secret')
+        self.login("admin", "secret")
-        response = self.client.get('/ensembles/party-posse')
+        response = self.client.get("/ensembles/party-posse")
-        self.assertTrue(response.context['request'].is_admin)
+        self.assertTrue(response.context["request"].is_admin)
         self.assertContains(response, "Add project")
-        self.assertAccess({
+        self.assertAccess(
-            '/ensembles/be-sharps': True,
+            {
-            '/ensembles/bleeding-gums': True,
+                "/ensembles/be-sharps": True,
-            '/ensembles/party-posse': True,
+                "/ensembles/bleeding-gums": True,
-            '/ensembles/unknown': False,
+                "/ensembles/party-posse": True,
-            '/ensembles/be-sharps/new-project': True,
+                "/ensembles/unknown": False,
-        })
+                "/ensembles/be-sharps/new-project": True,
+            }
+        )
 
     def test_superuser_projects(self):
-        self.login('admin', 'secret')
+        self.login("admin", "secret")
-        response = self.client.get('/projects')
+        response = self.client.get("/projects")
-        self.assertObjectList(response, ['Current Repertoire', 'Open Mic Night', 'Navy Recruitment Day', 'Barbershop Contest'])
+        self.assertObjectList(
-
+            response,
-        self.assertObjectList(self.client.get('/ensembles/bleeding-gums'), ['Open Mic Night'])
+            [
-        self.assertObjectList(self.client.get('/ensembles/bleeding-gums?inactive'), ['Open Mic Night', 'Baker St'])
+                "Current Repertoire",
+                "Open Mic Night",
+                "Navy Recruitment Day",
+                "Barbershop Contest",
+            ],
+        )
 
+        self.assertObjectList(
+            self.client.get("/ensembles/bleeding-gums"), ["Open Mic Night"]
+        )
+        self.assertObjectList(
+            self.client.get("/ensembles/bleeding-gums?inactive"),
+            ["Open Mic Night", "Baker St"],
+        )
 
     def test_user_ensembles(self):
-        self.login('homer', 'maggie')
+        self.login("homer", "maggie")
-        response = self.client.get('/ensembles')
+        response = self.client.get("/ensembles")
-        self.assertObjectList(response, ['The Be Sharps'])
+        self.assertObjectList(response, ["The Be Sharps"])
 
-        self.assertNotContains(response, 'Django Admin')
+        self.assertNotContains(response, "Django Admin")
 
     def test_user_ensemble_permissions(self):
-        self.login('homer', 'maggie')
+        self.login("homer", "maggie")
-        response = self.client.get('/ensembles/be-sharps')
+        response = self.client.get("/ensembles/be-sharps")
-        self.assertTrue(response.context['request'].is_admin)
+        self.assertTrue(response.context["request"].is_admin)
         self.assertContains(response, "Add project")
-        self.assertContains(response, 'Show all')
+        self.assertContains(response, "Show all")
-        self.assertAccess({
+        self.assertAccess(
-            '/ensembles/be-sharps': True,
+            {
-            '/ensembles/bleeding-gums': False,
+                "/ensembles/be-sharps": True,
-            '/ensembles/party-posse': False,
+                "/ensembles/bleeding-gums": False,
-            '/ensembles/be-sharps/new-project': True,
+                "/ensembles/party-posse": False,
-            '/ensembles/party-posse/new-project': False,
+                "/ensembles/be-sharps/new-project": True,
-        })
+                "/ensembles/party-posse/new-project": False,
+            }
+        )
 
-        self.authorize(models.Ensemble, slug='bleeding-gums')
+        self.authorize(models.Ensemble, slug="bleeding-gums")
-        self.assertAccess({
+        self.assertAccess(
-            '/ensembles/be-sharps': True,
+            {
-            '/ensembles/bleeding-gums': True,
+                "/ensembles/be-sharps": True,
-            '/ensembles/party-posse': False,
+                "/ensembles/bleeding-gums": True,
-            '/ensembles/be-sharps/new-project': True,
+                "/ensembles/party-posse": False,
-            '/ensembles/party-posse/new-project': False,
+                "/ensembles/be-sharps/new-project": True,
-        })
+                "/ensembles/party-posse/new-project": False,
-        response = self.client.get('/ensembles/bleeding-gums')
+            }
-        self.assertFalse(response.context['request'].is_admin)
+        )
-        self.assertNotContains(response, 'Add project')
+        response = self.client.get("/ensembles/bleeding-gums")
-        self.assertNotContains(response, 'Show all')
+        self.assertFalse(response.context["request"].is_admin)
+        self.assertNotContains(response, "Add project")
+        self.assertNotContains(response, "Show all")
 
     def test_user_projects(self):
-        self.login('homer', 'maggie')
+        self.login("homer", "maggie")
-        response = self.client.get('/projects')
+        response = self.client.get("/projects")
-        self.assertObjectList(response, ['Current Repertoire', 'Barbershop Contest'])
+        self.assertObjectList(response, ["Current Repertoire", "Barbershop Contest"])
-        response = self.client.get('/projects/3')
+        response = self.client.get("/projects/3")
-        self.assertTrue(response.context['request'].is_admin)
+        self.assertTrue(response.context["request"].is_admin)
 
-
+        self.assertAccess(
-        self.assertAccess({
+            {
-            '/projects/3': True,
+                "/projects/3": True,
-            '/projects/3/resources': True,
+                "/projects/3/resources": True,
-            '/projects/3/resources/add': True,
+                "/projects/3/resources/add": True,
-            '/projects/4': False,
+                "/projects/4": False,
-            '/projects/4/resources': False,
+                "/projects/4/resources": False,
-            '/projects/4/resources/add': False,
+                "/projects/4/resources/add": False,
-        })
+            }
+        )
 
         self.authorize(models.Project, pk=4)
-        response = self.client.get('/projects')
+        response = self.client.get("/projects")
-        self.assertObjectList(response, ['Current Repertoire', 'Open Mic Night', 'Barbershop Contest'])
+        self.assertObjectList(
-        response = self.client.get('/projects/4')
+            response, ["Current Repertoire", "Open Mic Night", "Barbershop Contest"]
-        self.assertFalse(response.context['request'].is_admin)
+        )
+        response = self.client.get("/projects/4")
+        self.assertFalse(response.context["request"].is_admin)
 
     def test_anon_ensembles(self):
-        response = self.client.get('/ensembles')
+        response = self.client.get("/ensembles")
         self.assertObjectList(response, [])
-        self.assertContains(response, 'You don\'t currently have access to any ensembles')
+        self.assertContains(
+            response, "You don't currently have access to any ensembles"
+        )
 
     def test_anon_authorized_ensemble(self):
-        self.authorize(models.Ensemble, slug='party-posse')
+        self.authorize(models.Ensemble, slug="party-posse")
-        response = self.client.get('/ensembles/party-posse')
+        response = self.client.get("/ensembles/party-posse")
-        self.assertContains(response, 'Party Posse')
+        self.assertContains(response, "Party Posse")
 
-        response = self.client.get('/ensembles')
+        response = self.client.get("/ensembles")
-        self.assertObjectList(response, ['Party Posse'])
+        self.assertObjectList(response, ["Party Posse"])
 
-        self.assertAccess({
+        self.assertAccess(
-            '/ensembles/be-sharps': False,
+            {
-            '/ensembles/party-posse': True,
+                "/ensembles/be-sharps": False,
-            '/ensembles/bleeding-gums': False,
+                "/ensembles/party-posse": True,
-            '/ensembles/unknown': False,
+                "/ensembles/bleeding-gums": False,
-        })
+                "/ensembles/unknown": False,
-        response = self.client.get('/projects')
+            }
-        self.assertObjectList(response, ['Navy Recruitment Day'])
+        )
+        response = self.client.get("/projects")
+        self.assertObjectList(response, ["Navy Recruitment Day"])
 
     def test_anon_authorized_project(self):
         self.authorize(models.Project, pk=4)
-        self.assertObjectList(self.client.get('/projects'), ['Open Mic Night'])
+        self.assertObjectList(self.client.get("/projects"), ["Open Mic Night"])
-        self.assertObjectList(self.client.get('/ensembles'), ['Lisa & the Bleeding Gums'])
+        self.assertObjectList(
+            self.client.get("/ensembles"), ["Lisa & the Bleeding Gums"]
+        )
 
-        self.assertAccess({
+        self.assertAccess(
-            '/projects/4': True,
+            {
-            '/projects/4/resources': True,
+                "/projects/4": True,
-            '/projects/1': False,
+                "/projects/4/resources": True,
-            '/projects/1/resources': False,
+                "/projects/1": False,
-        })
+                "/projects/1/resources": False,
+            }
+        )
 
     def test_anon_permission_denied(self):
-        self.assertAccess({
+        self.assertAccess(
-            '/ensembles': True,
+            {
-            '/ensembles/be-sharps': False,
+                "/ensembles": True,
-            '/ensembles/party-posse': False,
+                "/ensembles/be-sharps": False,
-            '/ensembles/bleeding-gums': False,
+                "/ensembles/party-posse": False,
-            '/ensembles/unknown': False,
+                "/ensembles/bleeding-gums": False,
-        })
+                "/ensembles/unknown": False,
+            }
+        )
 
     def test_anon_deauthorize_project(self):
         self.authorize(models.Project, pk=4)
-        self.assertAccess({
+        self.assertAccess({"/projects/4": True})
-            '/projects/4': True
-        })
         models.Project.objects.filter(pk=4).update(nonce=2)
-        self.assertAccess({
+        self.assertAccess({"/projects/4": False})
-            '/projects/4': False
-        })

app/interface/tests/test_integration.py

@@ -1,6 +1,6 @@
 from django.test import TestCase
 
-class IntegrationTestCase(TestCase):
 
+class IntegrationTestCase(TestCase):
     def test_runs(self):
         self.assertTrue(True)

app/interface/urls.py

@@ -1,37 +1,79 @@
 from django.urls import path
 from django.contrib.auth import views as auth_views
 from django.views.generic.base import RedirectView
+from django.conf import settings
 
 from . import views
 
 urlpatterns = [
-
+    path("", RedirectView.as_view(url="projects", permanent=False), name="home"),
-    path('', RedirectView.as_view(url='projects', permanent=False), name='home'),
+    path("login", auth_views.LoginView.as_view(), name="login"),
-    
+    path("logout", views.logout_view, name="logout"),
-    path('login', auth_views.LoginView.as_view(), name='login'),
+    path(
-    path('logout', auth_views.LogoutView.as_view(), name='logout'),
+        "forget/<resource>/<key>",
-    path('forget/<resource>/<key>', views.ForgetResourceView.as_view(), name="forget_resource"),
+        views.ForgetResourceView.as_view(),
-
+        name="forget_resource",
-    path('ensembles', views.EnsembleListView.as_view(), name="ensemble_list"),
+    ),
-    path('ensembles/<slug:ensemble>', views.EnsembleDetailView.as_view(), name='ensemble_detail'),
+    path("ensembles", views.EnsembleListView.as_view(), name="ensemble_list"),
-    path('ensembles/<slug:ensemble>/new-project', views.ProjectCreateView.as_view(), name="project_create"),
+    path(
-
+        "ensembles/<slug:ensemble>",
-    path('projects', views.ProjectListView.as_view(), name="project_list"),
+        views.EnsembleDetailView.as_view(),
-    path('projects/<int:project>', views.ProjectDetailView.as_view(), name="project_detail"),
+        name="ensemble_detail",
-    path('projects/<int:project>/edit', views.ProjectUpdateView.as_view(), name="project_edit"),
+    ),
-    #path('projects/<int:project>/submissions.mk', views.ProjectMakefileView.as_view(), name="project_makefile"),
+    path(
-
+        "ensembles/<slug:ensemble>/new-project",
-    path('projects/<int:project>/page/create', views.WikiCreateView.as_view(), name="wiki_create"),
+        views.ProjectCreateView.as_view(),
-    path('projects/<int:project>/page/<int:pk>', views.WikiView.as_view(), name="wiki"),
+        name="project_create",
-    path('projects/<int:project>/page/<int:pk>/edit', views.WikiEditView.as_view(), name="wiki_edit"),
+    ),
-
+    path("projects", views.ProjectListView.as_view(), name="project_list"),
-    path('projects/<int:project>/resources', views.ResourceListView.as_view(), name="resource_list"),
+    path(
-    path('projects/<int:project>/resources/add', views.ResourceCreateView.as_view(), name="resource_create"),
+        "projects/<int:project>",
-    path('projects/<int:project>/resources/<int:pk>/upload', views.ResourceUploadView.as_view(), name="resource_upload"),
+        views.ProjectDetailView.as_view(),
-    path('projects/<int:project>/resources/<int:pk>/edit', views.ResourceEditView.as_view(), name="resource_edit"),
+        name="project_detail",
+    ),
+    path(
+        "projects/<int:project>/edit",
+        views.ProjectUpdateView.as_view(),
+        name="project_edit",
+    ),
+    # path('projects/<int:project>/submissions.mk', views.ProjectMakefileView.as_view(), name="project_makefile"),
+    path(
+        "projects/<int:project>/page/create",
+        views.WikiCreateView.as_view(),
+        name="wiki_create",
+    ),
+    path("projects/<int:project>/page/<int:pk>", views.WikiView.as_view(), name="wiki"),
+    path(
+        "projects/<int:project>/page/<int:pk>/edit",
+        views.WikiEditView.as_view(),
+        name="wiki_edit",
+    ),
+    path(
+        "projects/<int:project>/resources",
+        views.ResourceListView.as_view(),
+        name="resource_list",
+    ),
+    path(
+        "projects/<int:project>/resources/add",
+        views.ResourceCreateView.as_view(),
+        name="resource_create",
+    ),
+    path(
+        "projects/<int:project>/resources/<int:pk>/upload",
+        views.ResourceUploadView.as_view(),
+        name="resource_upload",
+    ),
+    path(
+        "projects/<int:project>/resources/<int:pk>/edit",
+        views.ResourceEditView.as_view(),
+        name="resource_edit",
+    ),
 ]
 
-from django.conf import settings
+
 if settings.DEBUG:
     from django.views.static import serve
-    urlpatterns.append(path('local_storage/<path:path>', serve, {'document_root': 'local_storage'}))
+
+    urlpatterns.append(
+        path("local_storage/<path:path>", serve, {"document_root": "local_storage"})
+    )

app/interface/utils.py

@@ -1,18 +1,20 @@
 from django.shortcuts import resolve_url
 from django.core.signing import Signer
 from django.core.exceptions import SuspiciousOperation
+import logging
 
 signer = Signer()
 
-import logging
 logger = logging.getLogger(__name__)
 
-def sign_data(data, l=None):
+
+def sign_data(data, length=None):
     sig = signer.sign(data)
-    p = len(data) + 1
+    pos = len(data) + 1
-    if l:
+    if length:
-        l += p
+        length += pos
-    return sig[p:l]
+    return sig[pos:length]
+
 
 def signed_url(name, **kwargs):
     """
@@ -23,9 +25,10 @@ def signed_url(name, **kwargs):
     sep = "&" if "?" in url else "?"
     return sig.replace(":", f"{sep}auth=")
 
+
 def check_signed_url(full_path):
-    p = full_path.rfind('auth')
+    p = full_path.rfind("auth")
-    url = full_path[:p-1]
+    url = full_path[: p - 1]
     logger.debug("check_signed_url: %s", url)
     signed = signed_url(url)
     if signed != full_path:
@@ -33,6 +36,8 @@ def check_signed_url(full_path):
         signed = "_HIDDEN_"
         raise SuspiciousOperation("Bad auth code")
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     import doctest
+
     print(doctest.testmod())

app/interface/views.py

@@ -1,17 +1,16 @@
-"""
+""" """
-
-"""
 
+# pyright: basic
 
 from django.shortcuts import get_object_or_404, redirect, resolve_url
 from django.views.generic import RedirectView
 from django.views.generic.detail import DetailView, SingleObjectMixin
 from django.views.generic.list import ListView
-from django.views.generic.edit import CreateView, UpdateView, FormMixin
+from django.views.generic.edit import CreateView, UpdateView
 from django.core.exceptions import SuspiciousOperation
 from django.http import Http404, HttpResponseRedirect
-from django.db.models import Q
+from django.http.request import HttpRequest
-from django.utils import timezone
+from django.contrib.auth import logout
 
 from markdown2 import markdown
 
@@ -19,8 +18,10 @@ from . import models, forms
 from interface.utils import check_signed_url
 
 import logging
+
 logger = logging.getLogger(__name__)
 
+
 class AuthorizedResourceMixin(object):
     """
     Handles these parts of the permission system:
@@ -29,16 +30,20 @@ class AuthorizedResourceMixin(object):
         * Admin enforcing
     """
 
-    SESSION_KEY = 'authorized'
+    request: HttpRequest
+    kwargs: dict
+    _authorized: dict
+
+    SESSION_KEY = "authorized"
     admin_required = False
 
     def is_authorized(self):
         "By default check if superuser or a signed request"
-        if self.request.is_admin:
+        if self.request.is_admin:  # type: ignore
-            #logger.debug("is_authorized: superuser")
+            # logger.debug("is_authorized: superuser")
             return True
 
-        if 'sig' in self.request.GET:
+        if "sig" in self.request.GET:
             check_signed_url(self.request.get_full_path())
             self.on_signed_request()
             return True
@@ -51,25 +56,25 @@ class AuthorizedResourceMixin(object):
     def is_authorized_key(self, resource, key, auth):
         current = self.get_authorized_keys(resource).get(str(key), None)
         if current is None:
-            #logger.debug("is_authorized_key: %s %s not in session", resource, key)
+            # logger.debug("is_authorized_key: %s %s not in session", resource, key)
             return False
         if auth == current:
             return True
 
-        #logger.info("Authorisation revoked")
+        # logger.info("Authorisation revoked")
         self.del_authorized_key(resource, key)
         return False
 
     def get_authorized_keys(self, resource):
-        'Returns a set of authorized keys for this resource'
+        "Returns a set of authorized keys for this resource"
         return self._authorized.get(resource, {})
 
     def add_authorized_key(self, resource, key, auth):
-        'Adds a key to the authorized list for this resource'
+        "Adds a key to the authorized list for this resource"
         current = self.get_authorized_keys(resource)
         current[str(key)] = auth
         self._authorized[resource] = current
-        self.request.session[self.SESSION_KEY] = self._authorized
+        self.request.session[self.SESSION_KEY] = self._authorized  # type: ignore
 
     def del_authorized_key(self, resource, key):
         logger.info("Revoking authorization for %s %s", resource, key)
@@ -80,15 +85,16 @@ class AuthorizedResourceMixin(object):
             self._authorized[resource] = current
         else:
             self._authorized.pop(resource)
-        self.request.session[self.SESSION_KEY] = self._authorized
+        self.request.session[self.SESSION_KEY] = self._authorized  # type: ignore
         return True
 
     def request_denied(self):
-        raise Http404("Either the given resource doesn't exist or you dont have access to it.")
+        raise Http404(
+            "Either the given resource doesn't exist or you dont have access to it."
+        )
 
     def dispatch(self, request, *args, **kwargs):
-
+        self._authorized = request.session.get("authorized", {})
-        self._authorized = request.session.get('authorized', {})
         request.is_admin = request.user.is_superuser
 
         if not self.is_authorized():
@@ -97,12 +103,18 @@ class AuthorizedResourceMixin(object):
         if self.admin_required and not request.is_admin:
             return self.request_denied()
 
-        return super().dispatch(request, *args, **kwargs)
+        return super().dispatch(request, *args, **kwargs)  # type: ignore
+
 
 # TODO: RevokeResourceView - increment nonce
 
-class ForgetResourceView(AuthorizedResourceMixin, RedirectView):
 
+def logout_view(request):
+    logout(request)
+    return redirect("/")
+
+
+class ForgetResourceView(AuthorizedResourceMixin, RedirectView):
     def is_authorized(self):
         return True
 
@@ -110,9 +122,9 @@ class ForgetResourceView(AuthorizedResourceMixin, RedirectView):
         self.del_authorized_key(resource, key)
         return "/"
 
-class EnsembleMixin(AuthorizedResourceMixin):
 
-    ensemble_slug_kwarg = 'ensemble'
+class EnsembleMixin(AuthorizedResourceMixin):
+    ensemble_slug_kwarg = "ensemble"
     limited_project_access = False
 
     def is_authorized(self):
@@ -122,15 +134,15 @@ class EnsembleMixin(AuthorizedResourceMixin):
         if super().is_authorized():
             return True
 
-        if self.ensemble.has_admin(self.request.user):
+        if self.ensemble.has_admin(self.request.user):  # type: ignore
-            self.request.is_admin = True
+            self.request.is_admin = True  # type: ignore
             return True
 
-        if self.is_authorized_key('ensemble', ensemble_slug, self.ensemble.nonce):
+        if self.is_authorized_key("ensemble", ensemble_slug, self.ensemble.nonce):
             return True
 
-        authorized = set([ int(x) for x in self.get_authorized_keys('project').keys() ])
+        authorized = set([int(x) for x in self.get_authorized_keys("project").keys()])
-        projects = set(self.ensemble.projects.values_list('pk', flat=True))
+        projects = set(self.ensemble.projects.values_list("pk", flat=True))
         logger.debug("is_authorized: %r & %r", authorized, projects)
         if authorized & projects:
             self.limited_project_access = True
@@ -138,131 +150,131 @@ class EnsembleMixin(AuthorizedResourceMixin):
             return True
         return False
 
-    def get_object(self):
+    def get_object(self, queryset=None):
         return self.ensemble
 
 
 class ProjectMixin(AuthorizedResourceMixin):
-
+    project_kwarg = "project"
-    project_kwarg = 'project'
 
     def is_authorized(self):
         project_id = self.kwargs[self.project_kwarg]
-        self.project = get_object_or_404(models.Project.objects.select_related('ensemble'), pk=project_id)
+        self.project = get_object_or_404(
+            models.Project.objects.select_related("ensemble"), pk=project_id
+        )
 
         if super().is_authorized():
             return True
 
-        
         # check if the current user is an admin on the ensemble
-        if self.project.ensemble.has_admin(self.request.user):
+        if self.project.ensemble.has_admin(self.request.user):  # type: ignore
             logger.debug("is_authorized: ensemble admin for project")
-            self.request.is_admin = True
+            self.request.is_admin = True  # type: ignore
             return True
 
-        if self.is_authorized_key('ensemble', self.project.ensemble.pk, self.project.ensemble.nonce):
+        if self.is_authorized_key(
-            logger.debug('is_authorized: has ensemble link for project')
+            "ensemble", self.project.ensemble.pk, self.project.ensemble.nonce
+        ):
+            logger.debug("is_authorized: has ensemble link for project")
             return True
 
-        if self.is_authorized_key('project', project_id, self.project.nonce):
+        if self.is_authorized_key("project", project_id, self.project.nonce):
-            logger.debug('is_authorized: has project link')
+            logger.debug("is_authorized: has project link")
             return True
 
         return False
 
     # filter any generated querysets
     def get_queryset(self):
-        return super().get_queryset().filter(project=self.project)
+        return super().get_queryset().filter(project=self.project)  # type: ignore
 
     def get_context_data(self, **kwargs):
-        context = super().get_context_data(**kwargs)
+        context = super().get_context_data(**kwargs)  # type: ignore
-        if 'project' in self.kwargs:
+        if "project" in self.kwargs:
-            context['project'] = self.project
+            context["project"] = self.project
-            context['modules'] = self.project.active_modules
+            context["modules"] = self.project.active_modules
         return context
 
-class CrispyFormMixin(object):
 
+class CrispyFormMixin(object):
     cancel_url = None
 
     def get_cancel_url(self):
         return self.cancel_url
 
+
 """ ENSEMBLE VIEWS """
 
-class EnsembleListView(AuthorizedResourceMixin, ListView):
 
+class EnsembleListView(AuthorizedResourceMixin, ListView):
     def is_authorized(self):
         return True
 
     def get_queryset(self):
-        return models.Ensemble.objects.for_user(self.request.user,
+        return models.Ensemble.objects.for_user(
-                                                self.get_authorized_keys('ensemble').keys(),
+            self.request.user,  # type: ignore
-                                                self.get_authorized_keys('project').keys())
+            self.get_authorized_keys("ensemble").keys(),
-        #ensembles = models.Ensemble.objects.all()
+            self.get_authorized_keys("project").keys(),
+        )
 
-        #if self.request.is_admin:
-        #    return ensembles
-
-        # limit to registered ensembles
-        #f = Q(slug__in=self.get_authorized_keys('ensemble').keys()) | Q(projects__in=self.get_authorized_keys('project').keys())
-        
-        # or ensembles where the user is admin
-        #if self.request.user.is_authenticated:
-        #    f |= Q(admins=self.request.user.pk)
-
-        #return ensembles.filter(f).distinct()
 
 class EnsembleDetailView(EnsembleMixin, DetailView):
-
     def request_denied(self):
-        if 'auth' in self.request.GET:
+        if "auth" in self.request.GET:
-            if self.request.GET['auth'] != self.ensemble.auth():
+            if self.request.GET["auth"] != self.ensemble.auth():
                 raise SuspiciousOperation("Bad ensemble link")
-            self.add_authorized_key('ensemble', self.ensemble.slug, self.ensemble.nonce)
+            self.add_authorized_key("ensemble", self.ensemble.slug, self.ensemble.nonce)
-            return HttpResponseRedirect(resolve_url('ensemble_detail', self.ensemble.slug))
+            return HttpResponseRedirect(
+                resolve_url("ensemble_detail", self.ensemble.slug)
+            )
 
         return super().request_denied()
 
     def get_context_data(self, **kwargs):
         data = super().get_context_data(**kwargs)
-        inactive = 'inactive' in self.request.GET and self.request.is_admin
+        inactive = "inactive" in self.request.GET and self.request.is_admin  # type: ignore
 
         projects = self.ensemble.projects.all()
         if self.limited_project_access:
-            projects = projects.filter(pk__in=self.get_authorized_keys('project').keys())
+            projects = projects.filter(
+                pk__in=self.get_authorized_keys("project").keys()
+            )
 
         if inactive:
-            projects = projects.order_by('-pk')
+            projects = projects.order_by("-pk")
         else:
             projects = projects.active().current()
 
-        data['inactive'] = inactive
+        data["inactive"] = inactive
-        data['object_list'] = projects
+        data["object_list"] = projects
-        if self.request.is_admin:
+        if self.request.is_admin:  # type: ignore
-            data['ensemble_link'] = self.request.path + "?auth=" + self.ensemble.auth()
+            data["ensemble_link"] = self.request.path + "?auth=" + self.ensemble.auth()
         return data
 
-class EnsembleRevokeView(SingleObjectMixin, RedirectView):
 
+class EnsembleRevokeView(SingleObjectMixin, RedirectView):
     def get_redirect_url(self):
         return
 
+
 """ PROJECT VIEWS """
 
-class ProjectListView(ProjectMixin, ListView):
 
+class ProjectListView(ProjectMixin, ListView):
     def is_authorized(self):
         return True
 
     def get_project_queryset(self):
-        return models.Project.objects.for_user(self.request.user,
+        return models.Project.objects.for_user(
-                                       self.get_authorized_keys('project'),
+            self.request.user,  # type: ignore
-                                       self.get_authorized_keys('ensemble'))
+            self.get_authorized_keys("project"),
+            self.get_authorized_keys("ensemble"),
+        )
 
     def get_queryset(self):
         return self.get_project_queryset().current().active()
 
+
 class ProjectCreateView(EnsembleMixin, CreateView):
     admin_required = True
     model = models.Project
@@ -270,26 +282,25 @@ class ProjectCreateView(EnsembleMixin, CreateView):
     title = "Add a new project"
     form_class = forms.ProjectForm
 
-
     def form_valid(self, form):
         self.object = form.save(commit=False)
         self.object.ensemble = self.ensemble
-        self.object.owner = self.request.user
+        self.object.owner = self.request.user  # type: ignore
         self.object.save()
 
-        for module in form.cleaned_data['modules']:
+        for module in form.cleaned_data["modules"]:
             self.object.modules.create(name=module)
 
-        return redirect('project_detail', project=self.object.pk)
+        return redirect("project_detail", project=self.object.pk)
+
 
 class ProjectDetailView(ProjectMixin, DetailView):
-
     def request_denied(self):
-        if 'auth' in self.request.GET:
+        if "auth" in self.request.GET:
-            if self.request.GET['auth'] != self.project.auth():
+            if self.request.GET["auth"] != self.project.auth():
                 raise SuspiciousOperation("Bad project link")
-            self.add_authorized_key('project', self.project.pk, self.project.nonce)
+            self.add_authorized_key("project", self.project.pk, self.project.nonce)
-            return HttpResponseRedirect(resolve_url('project_detail', self.project.pk))
+            return HttpResponseRedirect(resolve_url("project_detail", self.project.pk))
         return super().request_denied()
 
     def get_object(self, queryset=None):
@@ -297,15 +308,16 @@ class ProjectDetailView(ProjectMixin, DetailView):
 
     def get_context_data(self, **kwargs):
         data = super().get_context_data(**kwargs)
-        if self.request.is_admin:
+        if self.request.is_admin:  # type: ignore
-            #data['project_link'] = signed_url(f'{self.request.path}?nonce={self.project.nonce}')
+            # data['project_link'] = signed_url(f'{self.request.path}?nonce={self.project.nonce}')
-            data['project_link'] = self.request.path + "?auth=" + self.project.auth()
+            data["project_link"] = self.request.path + "?auth=" + self.project.auth()
         return data
 
+
 class ProjectUpdateView(ProjectMixin, UpdateView):
     admin_required = True
     template_name = "interface/default_form.html"
-    pk_url_kwarg = 'project'
+    pk_url_kwarg = "project"
     form_class = forms.ProjectForm
 
     def get_object(self):
@@ -313,26 +325,26 @@ class ProjectUpdateView(ProjectMixin, UpdateView):
 
     def get_initial(self):
         data = super().get_initial()
-        data['modules'] = self.object.active_modules
+        data["modules"] = self.object.active_modules
         print(data)
         return data
 
     def form_valid(self, form):
         self.object = form.save()
         current = set(self.object.active_modules)
-        desired = set(form.cleaned_data['modules'])
+        desired = set(form.cleaned_data["modules"])
         self.object.modules.exclude(name__in=desired).delete()
-        for module in desired-current:
+        for module in desired - current:
             self.object.modules.create(name=module)
-        return redirect('project_detail', self.kwargs['project'])
+        return redirect("project_detail", self.kwargs["project"])
 
     @property
     def cancel_url(self):
-        return resolve_url('project_detail', self.kwargs['project'])
+        return resolve_url("project_detail", self.kwargs["project"])
 
 
 # Old Makefile from submission module
-#class ProjectMakefileView(EnsembleMixin, DetailView):
+# class ProjectMakefileView(EnsembleMixin, DetailView):
 #    template_name = 'interface/project_submissions.mk'
 #    content_type = 'text/plain'
 #
@@ -359,29 +371,32 @@ class ProjectUpdateView(ProjectMixin, UpdateView):
 
 """ WIKI VIEWS """
 
+
 class WikiView(ProjectMixin, DetailView):
-    template_name = 'interface/wiki.html'
+    template_name = "interface/wiki.html"
     model = models.WikiPage
 
     def get_context_data(self, **kwargs):
         data = super().get_context_data(**kwargs)
-        data['wiki_html'] = markdown(self.object.markdown)
+        data["wiki_html"] = markdown(self.object.markdown)
         return data
 
+
 class WikiCreateView(ProjectMixin, CreateView):
     admin_required = True
     model = models.WikiPage
-    template_name = 'interface/default_form.html'
+    template_name = "interface/default_form.html"
     form_class = forms.WikiForm
 
     def cancel_url(self):
-        return resolve_url('project_detail', self.kwargs['project'])
+        return resolve_url("project_detail", self.kwargs["project"])
 
     def form_valid(self, form):
         self.object = form.save(commit=False)
         self.object.project = self.project
         self.object.save()
-        return redirect('wiki', project=self.object.project_id, pk=self.object.pk)
+        return redirect("wiki", project=self.object.project_id, pk=self.object.pk)
+
 
 class WikiEditView(ProjectMixin, UpdateView):
     admin_required = True
@@ -389,46 +404,53 @@ class WikiEditView(ProjectMixin, UpdateView):
     form_class = forms.WikiForm
 
     def cancel_url(self):
-        return resolve_url('wiki', self.kwargs['project'], self.kwargs['pk'])
+        return resolve_url("wiki", self.kwargs["project"], self.kwargs["pk"])
+
 
 """ RESOURCE VIEWS """
 
+
 class ResourceCreateView(ProjectMixin, CreateView):
     admin_required = True
     model = models.Resource
     form_class = forms.ResourceForm
-    template_name = 'interface/default_form.html'
+    template_name = "interface/default_form.html"
     title = "Add a new resource"
 
     def form_valid(self, form):
         self.object = form.save(commit=False)
         self.object.project = self.project
         self.object.save()
-        return redirect('resource_upload', project=self.object.project_id, pk=self.object.pk)
+        return redirect(
+            "resource_upload", project=self.object.project_id, pk=self.object.pk
+        )
+
 
 class ResourceUploadView(ProjectMixin, UpdateView):
     admin_required = True
     model = models.Resource
-    fields = ['file']
+    fields = ["file"]
-    template_name = 'interface/default_form.html'
+    template_name = "interface/default_form.html"
 
     def get_success_url(self):
-        return resolve_url('resource_list', project=self.kwargs['project'])
+        return resolve_url("resource_list", project=self.kwargs["project"])
+
 
 class ResourceListView(ProjectMixin, ListView):
     model = models.Resource
 
     def get_queryset(self):
         qs = super().get_queryset()
-        if not self.request.is_admin:
+        if not self.request.is_admin:  # type: ignore
             qs = qs.filter(visible=True)
         return qs
 
+
 class ResourceEditView(ProjectMixin, UpdateView):
     admin_required = True
     model = models.Resource
-    fields = ['name', 'description', 'visible']
+    fields = ["name", "description", "visible"]
-    template_name = 'interface/default_form.html'
+    template_name = "interface/default_form.html"
 
     def get_success_url(self):
-        return resolve_url('resource_list', project=self.kwargs['project'])
+        return resolve_url("resource_list", project=self.kwargs["project"])

app/library/admin.py

@@ -2,60 +2,77 @@ from django.contrib import admin
 
 from . import models
 
+
 class EnsembleAccessInline(admin.StackedInline):
     model = models.EnsembleAccess
     extra = 0
 
+
 class CollectionAdmin(admin.ModelAdmin):
-    list_display = ['name', 'location', 'storage', 'prefix']
+    list_display = ["name", "location", "storage", "prefix"]
     inlines = [EnsembleAccessInline]
 
+
 admin.site.register(models.Collection, CollectionAdmin)
 
+
 class ItemInline(admin.TabularInline):
     model = models.ProjectItem
     extra = 0
 
+
 class DocInline(admin.TabularInline):
     model = models.Document
     extra = 0
 
+
 class MetaInline(admin.TabularInline):
     model = models.WorkMeta
     extra = 0
 
+
 class WorkAdmin(admin.ModelAdmin):
-    list_display = ['name', 'composer', 'edition', 'identifier', 'running_time']
+    list_display = ["name", "composer", "edition", "identifier", "running_time"]
-    list_filter = ['collection']
+    list_filter = ["collection"]
-    search_fields = ['name', 'composer']
+    search_fields = ["name", "composer"]
     inlines = [MetaInline, DocInline, ItemInline]
 
+
 admin.site.register(models.Work, WorkAdmin)
 
+
 class SectionInline(admin.TabularInline):
     model = models.Section
-    fields = ['tag', 'start', 'end', 'page']
+    fields = ["tag", "start", "end", "page"]
+
 
 class DocumentAdmin(admin.ModelAdmin):
-    list_display = ['work', '__str__']
+    list_display = ["work", "__str__"]
-    list_filter = ['work__collection']
+    list_filter = ["work__collection"]
     inlines = [SectionInline]
 
+
 admin.site.register(models.Document, DocumentAdmin)
 
+
 class ItemAdmin(admin.ModelAdmin):
-    list_display = ['project', 'work', 'order']
+    list_display = ["project", "work", "order"]
-    list_filter = ['project']
+    list_filter = ["project"]
+
 
 admin.site.register(models.ProjectItem, ItemAdmin)
 
+
 class EnsembleAccessAdmin(admin.ModelAdmin):
-    list_display = ['ensemble', 'collection', 'access_type']
+    list_display = ["ensemble", "collection", "access_type"]
-    list_filter = ['ensemble']
+    list_filter = ["ensemble"]
+
 
 admin.site.register(models.EnsembleAccess, EnsembleAccessAdmin)
 
+
 class OrchestrationAdmin(admin.ModelAdmin):
-    list_display = ['name', 'instruments']
+    list_display = ["name", "instruments"]
+
 
 admin.site.register(models.Orchestration, OrchestrationAdmin)

app/library/apps.py

@@ -2,4 +2,4 @@ from django.apps import AppConfig
 
 
 class LibraryConfig(AppConfig):
-    name = 'library'
+    name = "library"

app/library/forms.py

@@ -5,29 +5,50 @@ from interface.forms import BaseForm
 
 
 class WorkCreateForm(forms.ModelForm, BaseForm):
-
     class Meta:
         model = Work
-        fields = ['name', 'composer', 'edition', 'code', 'orchestration', 'licence', 'running_time', 'notes']    
+        fields = [
+            "name",
+            "composer",
+            "edition",
+            "code",
+            "orchestration",
+            "licence",
+            "running_time",
+            "notes",
+        ]
+
 
 class PlaylistAddForm(forms.Form):
     work = forms.ModelChoiceField(queryset=Work.objects.all())
 
     def __init__(self, instance, *args, **kwargs):
-         super(PlaylistAddForm, self).__init__(*args, **kwargs)
+        super(PlaylistAddForm, self).__init__(*args, **kwargs)
 
-         existing = [ x[0] for x in instance.works.values_list('pk') ]
+        existing = [x[0] for x in instance.works.values_list("pk")]
 
-         qs = Work.objects.filter(ensemble_id=instance.ensemble_id).exclude(id__in=existing)
+        qs = Work.objects.filter(ensemble_id=instance.ensemble_id).exclude(
-         self.fields['work'].queryset = qs
+            id__in=existing
-         self.instance = instance
+        )
+        self.fields["work"].queryset = qs
+        self.instance = instance
 
     def save(self):
-        self.instance.works.add(self.cleaned_data['work'])
+        self.instance.works.add(self.cleaned_data["work"])
+
 
 class ProjectEnsembleChoiceField(forms.ModelChoiceField):
     def label_from_instance(self, obj):
         return f"{obj.ensemble.name} - {obj.name}"
 
+
 class ProjectSelectForm(BaseForm):
     project = ProjectEnsembleChoiceField(queryset=Project.objects.all())
+
+
+class DocumentLinkForm(BaseForm):
+    link = forms.URLField(help_text="Paste the direct link relevant to this storage")
+
+
+class DocumentBulkForm(BaseForm):
+    folder_link = forms.URLField(help_text="Paste the folder link for this storage")

app/library/gdrive/__init__.py

@@ -0,0 +1,75 @@
+from library.models import Collection, Work, WorkMeta, Document
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def sync_work(work: Work):
+    logger.info("Syncing '%s'", work.name)
+    folder_id = work.meta_info.get(name="folderid").value
+
+    storage = work.collection.storage.instance()
+    prefix = work.collection.storage.name
+    _, files = storage.listdir(folder_id)
+
+    existing = set(
+        [
+            storage.parse_id(x.partition(":")[2])
+            for x in work.docs.values_list("upload", flat=True)
+        ]
+    )
+    logger.debug("%d existing documents", len(existing))
+
+    for file in files:
+        if file.id in existing:
+            logger.debug("%30s: Skipping existing (%s)", file.name, file.id)
+            existing.discard(file.id)
+            continue
+
+        if not file.name.lower().endswith(".pdf"):
+            logger.debug("%40s: Not a PDF", file.name)
+            continue
+
+        logger.info("%40s: Adding", file.name)
+        doc = work.docs.create(upload=f"{prefix}:{file}", doctype=Document.DOCTYPE_PDF)
+        doc.auto_tag()
+
+    for uri in existing:
+        logger.warning("Local entry not in folder: %s", uri)
+
+
+def sync_collection(collection: Collection, sync_existing: bool = False):
+    logger.info("Syncing '%s'", collection)
+
+    if not collection.storage.storage.endswith("GDriveLinkStorage"):
+        raise RuntimeError("Not a gdrive storage")
+
+    if not collection.prefix:
+        raise KeyError("Prefix must store folder id")
+
+    existing = dict(
+        WorkMeta.objects.filter(
+            work__collection=collection, name="folderid"
+        ).values_list("value", "work_id")
+    )
+
+    storage = collection.storage.instance()
+    folders, _ = storage.listdir(collection.prefix)
+
+    for folder in folders:
+        if folder.id in existing:
+            if sync_existing:
+                logger.info("%40s: Syncing (%s)", folder.name, folder.id[:12])
+                sync_work(Work.objects.get(pk=existing[folder.id]))
+            del existing[folder.id]
+            continue
+
+        logger.info("%40s: Adding", folder.name)
+        work = Work(name=folder.name, collection=collection)
+        work.save()
+        work.meta_info.create(name="folderid", value=folder.id)
+        sync_work(work)
+
+    for folderid, work in existing:
+        logger.warning("Folder for work %d no longer in drive (%s)", work, folderid)

app/library/gdrive/storage.py

@@ -0,0 +1,120 @@
+from django.core.files.storage import Storage
+from collections import namedtuple
+import requests
+import re
+from gzip import GzipFile
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+SHARED_FOLDER = re.compile(r"https://drive.google.com/drive/folders/(\w+)")
+SHARED_FILE = re.compile(r"https://drive.google.com/file/d/([\w\-]+)")
+
+FILES_API = "https://www.googleapis.com/drive/v3/files"
+
+
+class DriveObject(namedtuple("DriveObject", ("id", "name"))):
+    @classmethod
+    def from_string(cls, s):
+        return cls(*s.split("/", 1))
+
+    def __str__(self):
+        return f"{self.id}/{self.name}"
+
+
+class GDriveLinkStorage(Storage):
+    is_writable = False
+
+    def __init__(self, api_key):
+        self.api_key = api_key
+        super().__init__()
+
+    def parse_id(self, name):
+        parts = name.split("/")
+        return parts[0]
+
+    def extract_id(self, url, *patterns):
+        logger.debug("EXTRACT_ID: %r", url)
+        for pattern in patterns:
+            match = pattern.match(url)
+            if match:
+                return match.groups()[0]
+        raise FileNotFoundError(f"Not a valid url: {url}")
+
+    def get_json(self, url):
+        logger.debug("GET_JSON: %s", url)
+        response = requests.get(url)
+        data = response.json()
+        logger.debug("Data: %r", data)
+        return data
+
+    def listdir(self, path) -> tuple[list[str], list[str]]:
+
+        # used to test for valid connection parameters - should do something to validate API key here
+        if path == "":
+            return [], []
+
+        logger.debug("LISTDIR: %s", path)
+        folder_id = self.parse_id(path)
+        url = f"{FILES_API}?q='{folder_id}'+in+parents&key={self.api_key}"
+        data = self.get_json(url)
+        files = []
+        folders = []
+        for x in data["files"]:
+            if x["mimeType"] == "application/vnd.google-apps.folder":
+                # folders.append(f"{x['id']}/{x['name']}")
+                folders.append(DriveObject(x["id"], x["name"]))
+            else:
+                # files.append(f"{x['id']}/{x['name']}")
+                files.append(DriveObject(x["id"], x["name"]))
+
+        return folders, files
+
+    def get_meta(self, name):
+        file_id = self.parse_id(name)
+        url = f"{FILES_API}/{file_id}?key={self.api_key}"
+        return self.get_json(url)
+
+    def open(self, name, mode="rb"):
+        file_id = self.parse_id(name)
+        url = f"{FILES_API}/{file_id}?alt=media&key={self.api_key}"
+        logger.info("URL: %s", url)
+        response = requests.get(url, stream=True)
+        return GzipFile(name, "rb", 9, response.raw)
+
+    def size(self, name):
+        raise NotImplementedError()
+
+    def delete(self, name):
+        pass
+
+    def url(self, name):
+        logger.debug("URL: %r", name)
+        file_id = self.parse_id(name)
+        return f"https://drive.usercontent.google.com/download?export=download&id={file_id}&confirm=yes"
+
+    def get_folder_id(self, url):
+        try:
+            return self.extract_id(url, SHARED_FOLDER)
+        except FileNotFoundError:
+            return None
+
+    def get_file_id(self, url):
+        try:
+            return self.extract_id(url, SHARED_FILE)
+        except FileNotFoundError:
+            return None
+
+
+"""
+    def import_link(self, url) -> str:
+        file_id = self.extract_id(url, SHARED_FILE)
+        meta = self.get_meta(file_id)
+        return f"{file_id}/{meta['name']}"
+
+    def folder_import(self, url) -> list[str]:
+        folder_id = self.extract_id(url, SHARED_FOLDER)
+        _, files = self.listdir(folder_id)
+        return files
+"""

app/library/gdrive/views.py

@@ -0,0 +1,60 @@
+from django.shortcuts import resolve_url, redirect
+from django.views.generic import FormView
+from django.views.generic.detail import SingleObjectMixin
+from library.views import CollectionMixin
+from library.models import Work, Document
+from library import forms
+
+
+class WorkGDriveView(CollectionMixin, SingleObjectMixin, FormView):
+    model = Work
+    template_name = "library/gdrive.html"
+    form_class = forms.DocumentLinkForm
+
+    @property
+    def cancel_url(self):
+        return resolve_url("work_detail", self.collection.pk, self.kwargs["pk"])
+
+    def get_context_data(self, *args, **kwargs):
+        self.object = self.get_object()
+        data = super().get_context_data(*args, **kwargs)
+
+        data["meta"] = dict(self.object.meta_info.values_list("name", "value"))
+        print(data["meta"])
+
+        return data
+
+    def form_valid(self, form):
+        link = form.cleaned_data["link"]
+
+        storage = self.collection.storage.instance()
+        self.object = self.get_object()
+
+        try:
+            folderid = storage.get_folder_id(link)
+            self.object.meta_info.update_or_create(
+                name="folderid", defaults={"value": folderid}
+            )
+            return redirect("work_detail", self.collection.pk, self.kwargs["pk"])
+        except FileNotFoundError:
+            pass  # not a folder id
+
+        try:
+            link = self.collection.storage.instance().import_link(link)
+        except AttributeError:
+            pass
+        except FileNotFoundError as e:
+            form.add_error("link", str(e))
+            return self.form_invalid(form)
+
+        work = self.collection.works.get(pk=self.kwargs["pk"])
+
+        doc = Document(
+            work=work,
+            upload=f"{self.collection.storage.name}:{link}",
+            doctype=Document.DOCTYPE_PDF,
+        )
+        doc.save()
+        doc.auto_tag()
+
+        return redirect("work_detail", self.collection.pk, self.kwargs["pk"])

app/library/indexer/__init__.py

@@ -0,0 +1,103 @@
+from typing import Protocol, Any, Iterable, Generator
+from django.conf import settings
+from django.utils.module_loading import import_module
+from django.db.models import QuerySet
+
+from library.models import Work, Collection
+
+instance = getattr(settings, "INDEXER", "library.indexer.whoosh")
+
+
+class Indexer(Protocol):
+    def create_index(self) -> None: ...
+
+    def get_index(self) -> None: ...
+
+    def reset_index(self) -> None: ...
+
+    def search(
+        self,
+        query: str,
+        collections: list[int],
+        page: int = 1,
+        pagesize: int = 20,
+    ) -> tuple[list[dict], dict[str, Any]]: ...
+
+    def index_docs(self, docs: Iterable[dict]): ...
+
+
+class PartialResultSet(object):
+    """
+    Implements enough of QuerySet to fool the Paginator!
+    """
+
+    def __init__(self, works, meta):
+        self.works = works
+        self.meta = meta
+
+        self.start = (meta["page"] - 1) * meta["pagesize"]
+
+    def __getitem__(self, key):
+        if isinstance(key, slice):
+            if key.start != self.start:
+                raise KeyError(f"Expected {self.start}, got {key.start}")
+
+            return self.works
+
+        return self.works[key]
+
+    def count(self):
+        return self.meta["total"]
+
+
+# make the given module available as indexer
+indexer: Indexer = import_module(instance)  # type: ignore
+
+
+def work_to_doc(work: Work) -> dict[str, str]:
+    tags = ",".join(work.meta_info.filter(name="tag").values_list("value", flat=True))  # type: ignore
+    meta = ", ".join(work.meta_info.values_list("value", flat=True))  # type: ignore
+
+    description = f"""{work.name}
+{work.composer}
+{work.edition}
+{work.notes}
+{meta}
+"""
+
+    return dict(
+        work=str(work.pk),
+        collection=str(work.collection_id),  # type: ignore
+        name=work.name,
+        composer=work.composer,
+        edition=work.edition,
+        tag=tags.lower(),
+        text=description,
+    )
+
+
+def doc_set(works: QuerySet) -> Generator:
+    for work in works:
+        yield work_to_doc(work)
+
+
+def index_works(works: QuerySet):
+    indexer.index_docs(doc_set(works))
+
+
+def model_search(
+    query: str, collections: list[int], page: int = 1, pagesize: int = 20
+) -> PartialResultSet:
+    hits, meta = indexer.search(query.lower(), collections, page, pagesize)
+    meta["pagesize"] = pagesize
+    meta["page"] = page
+
+    works = [Work(**hit) for hit in hits]
+
+    collection_names = dict(Collection.objects.values_list("pk", "name"))
+    for work in works:
+        work.collection = Collection(  # type: ignore
+            pk=work.collection_id, name=collection_names[work.collection_id]
+        )
+
+    return PartialResultSet(works, meta)

app/library/indexer/whoosh.py

@@ -0,0 +1,91 @@
+from whoosh.index import create_in, open_dir, EmptyIndexError, Index
+from whoosh.analysis import StemmingAnalyzer, CharsetFilter
+from whoosh.support.charset import accent_map
+from whoosh.fields import Schema, TEXT, KEYWORD, NUMERIC
+from whoosh.qparser import QueryParser
+from whoosh.query import Term, NullQuery
+
+
+from typing import Any
+from django.conf import settings
+import os.path
+import shutil
+
+stemming_analyzer = StemmingAnalyzer() | CharsetFilter(accent_map)
+
+schema = Schema(
+    work=NUMERIC(stored=True, unique=True),
+    collection=NUMERIC(stored=True),
+    name=TEXT(stored=True),
+    composer=TEXT(stored=True),
+    edition=TEXT(stored=True),
+    tag=KEYWORD(commas=True),
+    text=TEXT(analyzer=stemming_analyzer),
+)
+
+
+index_path = os.path.join(os.path.dirname(settings.BASE_DIR), "index")
+
+
+def create_index() -> Index:
+    if not os.path.exists(index_path):
+        os.mkdir(index_path)
+
+    ix = create_in(index_path, schema)
+    return ix
+
+
+def get_index() -> Index:
+    try:
+        return open_dir(index_path)
+    except EmptyIndexError:
+        return create_index()
+
+
+def reset_index() -> Index:
+    shutil.rmtree(index_path)
+    return create_index()
+
+
+def index_docs(works: list[dict]):
+    ix = get_index()
+    with ix.writer() as writer:
+        for work in works:
+            writer.update_document(**work)
+
+
+def search(
+    query: str,
+    collections: list[int] = [],
+    page: int = 1,
+    pagesize: int = 20,
+) -> tuple[list[dict], dict[str, Any]]:
+    meta = {}
+
+    qp = QueryParser("text", schema=schema)
+    q = qp.parse(query.lower())
+    meta["query"] = str(q)
+
+    terms = NullQuery
+    for c in collections:
+        terms = terms | Term("collection", c)
+    q = q & terms
+
+    hits = []
+    ix = get_index()
+    with ix.searcher() as searcher:
+        results = searcher.search_page(q, page, pagesize)
+        for result in results:
+            hits.append(
+                dict(
+                    pk=result["work"],
+                    name=result["name"],
+                    composer=result["composer"],
+                    edition=result["edition"],
+                    collection_id=int(result["collection"]),
+                )
+            )
+
+    meta["total"] = len(results)
+
+    return hits, meta

app/library/management/commands/import_works.py

@@ -1,20 +1,23 @@
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand
 import argparse
 import csv
 
 from library import models
 
+
 class Command(BaseCommand):
-    help = 'Imports works from a csv file'
+    help = "Imports works from a csv file"
 
     def add_arguments(self, parser):
-        parser.add_argument('collection', type=int, help="Collection ID")
+        parser.add_argument("collection", type=int, help="Collection ID")
-        parser.add_argument('source', type=argparse.FileType('r'), help="Source CSV")
+        parser.add_argument("source", type=argparse.FileType("r"), help="Source CSV")
 
     def handle(self, *args, **options):
 
-        collection = models.Collection.objects.get(pk=options['collection'])
+        collection = models.Collection.objects.get(pk=options["collection"])
 
-        reader = csv.DictReader(options['source'])
+        reader = csv.DictReader(options["source"])
         for row in reader:
-            collection.works.create(name=row['Piece'], composer=row['Composer'], notes=row['Notes'])
+            collection.works.create(
+                name=row["Piece"], composer=row["Composer"], notes=row["Notes"]
+            )

app/library/management/commands/index.py

@@ -0,0 +1,36 @@
+from django.core.management.base import BaseCommand
+
+from library import models
+from library.indexer import model_search, index_works, indexer
+
+FORMATTER = "{w.name:50s} {w.edition:15s} {w.collection.name:15s}"
+
+
+class Command(BaseCommand):
+    help = "Imports works from a csv file"
+
+    def add_arguments(self, parser):
+        parser.add_argument("action", choices=("run", "search", "terms"))
+        parser.add_argument("query", nargs="*")
+        parser.add_argument("--collection", "-c", nargs="*", type=int)
+        # parser.add_argument("collection", type=int, help="Collection ID")
+        # parser.add_argument("source", type=argparse.FileType("r"), help="Source CSV")
+
+    def handle(self, action, query, *args, **options):
+        try:
+            method = getattr(self, f"handle_{action}")
+        except AttributeError:
+            raise RuntimeError(f"Unknown handler: {action}")
+        return method("".join(query), options["collection"] or [])
+
+    def handle_run(self, query, collections=[]):
+        index_works(models.Work.objects.all())
+
+    def handle_search(self, query, collections=[]):
+        for result in model_search(query, collections):
+            print(FORMATTER.format(w=result))
+
+    def handle_terms(self, query, collections=[]):
+        ix = indexer.get_index()
+        with ix.searcher() as searcher:
+            print(b", ".join(searcher.lexicon(query or "text")))

app/library/management/commands/sync.py

@@ -0,0 +1,25 @@
+from django.core.management.base import BaseCommand, CommandError
+from library.models import Work, Collection
+from library.gdrive import sync_work, sync_collection
+
+
+class Command(BaseCommand):
+    help = "Synchronizes folders and works"
+
+    def add_arguments(self, parser):
+        parser.add_argument("type", type=str, choices=["work", "collection"])
+        parser.add_argument("pk", nargs="?", type=int)
+
+    def handle(self, *args, **options):
+
+        if options["type"] == "work":
+            work = Work.objects.get(pk=options["pk"])
+            sync_work(work)
+            return
+
+        if options["type"] == "collection":
+            collection = Collection.objects.get(pk=options["pk"])
+            sync_collection(collection)
+            return
+
+        raise CommandError("Unknown object type")

app/library/migrations/0015_collection_settings_alter_document_doctype.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.0.6 on 2024-06-18 05:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('library', '0014_auto_20230223_1422'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='collection',
+            name='settings',
+            field=models.JSONField(default=dict, help_text='Storage specific settings'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='doctype',
+            field=models.PositiveSmallIntegerField(choices=[(1, 'PDF'), (2, 'Audio'), (3, 'Video'), (4, 'Misc')], default=1),
+        ),
+    ]

app/library/models.py

@@ -1,12 +1,9 @@
-from os import SCHED_OTHER
-from django.conf import settings
 from django.shortcuts import resolve_url
 from django.db import models
 from django.utils.text import slugify
 from django.utils.timezone import now
 from django.utils.functional import cached_property
-from django.core.files.storage import get_storage_class
+from django.db.models import Count, Min, Max
-from django.db.models import Q, Count, Min, Max
 
 import re
 import os.path
@@ -14,49 +11,59 @@ import os.path
 from byostorage.user import BYOStorage
 from byostorage.cached import CachedStorage
 
-from library.music_tags import MusicTag
+from library.music_tags import MusicTag, auto_tag
 from interface.utils import sign_data
 
 import logging
 
-#from polyphonic.settings import LIBRARY_STORAGE
+# from polyphonic.settings import LIBRARY_STORAGE
 logger = logging.getLogger(__name__)
 
-#try:
+# try:
 #    library_storage = get_storage_class(settings.LIBRARY_STORAGE)()
-#except (ImportError, AttributeError):
+# except (ImportError, AttributeError):
 #    logger.exception("Failed to load library storage")
 #    library_storage = get_storage_class()()
-#logger.info("Library storage: %s", library_storage.__class__.__name__)
+# logger.info("Library storage: %s", library_storage.__class__.__name__)
 
 # FIXME: move back to settings
 library_storage = CachedStorage(BYOStorage())
 
+
 class Orchestration(models.Model):
     """
     Stores a list of instrument codes as a single entry (space delimited).
     Can be global or ensemble specific
     """
-    collection = models.ForeignKey('Collection', on_delete=models.CASCADE, related_name="custom_orchestrations", null=True, blank=True)
+
+    collection = models.ForeignKey(
+        "Collection",
+        on_delete=models.CASCADE,
+        related_name="custom_orchestrations",
+        null=True,
+        blank=True,
+    )
     name = models.CharField(max_length=100)
     instruments = models.TextField()
 
     def as_list(self):
-        tags = [ t.strip() for t in self.instruments.split(' ') ]
+        tags = [t.strip() for t in self.instruments.split(" ")]
-        return [ (t, MusicTag.from_tag(t)) for t in tags if t ]
+        return [(t, MusicTag.from_tag(t)) for t in tags if t]
 
     def tag_order(self):
-        tags = [ t.strip() for t in self.instruments.split(' ') if t ]
+        tags = [t.strip() for t in self.instruments.split(" ") if t]
-        order = {'score': 0}
+        order = {"score": 0}
         for i, t in enumerate(tags):
-            order.setdefault(t.strip('-0123456789'), i*2+1)
+            order.setdefault(t.strip("-0123456789"), i * 2 + 1)
 
         return order
 
     def sorter(self):
         tag_order = self.tag_order()
+
         def f(x):
-            return (tag_order.get(x[0].strip('-0123456789'), 1000), x[0])
+            return (tag_order.get(x[0].strip("-0123456789"), 1000), x[0])
+
         return f
 
     def save(self):
@@ -66,74 +73,101 @@ class Orchestration(models.Model):
     def __str__(self):
         return self.name
 
+
 class ProjectItem(models.Model):
     """
     ProjectItem represents a Work attached to a Project e.g. item in set list or programme
     It also allows works to be shared from one ensemble to another on a per-project basis.
     """
-    project = models.ForeignKey('interface.Project', on_delete=models.CASCADE, related_name='items')
+
-    work = models.ForeignKey('Work', on_delete=models.CASCADE, related_name='project_items')
+    project = models.ForeignKey(
+        "interface.Project", on_delete=models.CASCADE, related_name="items"
+    )
+    work = models.ForeignKey(
+        "Work", on_delete=models.CASCADE, related_name="project_items"
+    )
     checkout = models.DateTimeField()
     due = models.DateTimeField(null=True, blank=True)
     returned = models.DateTimeField(null=True, blank=True)
-    approved_by = models.ForeignKey('auth.User', on_delete=models.CASCADE)
+    approved_by = models.ForeignKey("auth.User", on_delete=models.CASCADE)
     order = models.SmallIntegerField(default=0)
     section = models.CharField(max_length=100, blank=True)
 
-
     class Meta:
-        ordering = ['order', 'work']
+        ordering = ["order", "work"]
 
     def __str__(self):
         return f"<{self.project_id}:{slugify(self.work.name)}>"
 
+
 class Collection(models.Model):
     """
     A logical collection of works, typically owned by an organisation or person (physical or virtual)
     """
-    name = models.CharField(max_length=255,
+
-            help_text="Name of the collection")
+    name = models.CharField(max_length=255, help_text="Name of the collection")
-    prefix = models.SlugField(max_length=30, default="default",
+    prefix = models.CharField(
-            help_text="Folder to store works in")
+        max_length=255, default="default", help_text="Folder to store works in"
-    administrators = models.ManyToManyField('auth.User', related_name="collections", 
+    )
-            help_text="Administrators for this collection")
+    administrators = models.ManyToManyField(
-    location = models.CharField(max_length=100, 
+        "auth.User",
-            help_text="Physical location (institution, town...)", blank=True)
+        related_name="collections",
-    storage = models.ForeignKey('byostorage.UserStorage', on_delete=models.CASCADE, null=True, blank=True, 
+        help_text="Administrators for this collection",
-            help_text="User storage for documents")
+    )
-    notes = models.TextField(blank=True, 
+    location = models.CharField(
-            help_text="Publicly visible notes about collection and loans policy (markdown format)")
+        max_length=100, help_text="Physical location (institution, town...)", blank=True
-    nonce = models.SmallIntegerField(default=1,
+    )
-            help_text="Increment this to reset the authentication links")
+    storage = models.ForeignKey(
+        "byostorage.UserStorage",
+        on_delete=models.CASCADE,
+        null=True,
+        blank=True,
+        help_text="User storage for documents",
+    )
+    notes = models.TextField(
+        blank=True,
+        help_text="Publicly visible notes about collection and loans policy (markdown format)",
+    )
+    settings = models.JSONField(
+        default=dict, blank=True, help_text="Storage specific settings"
+    )
+    nonce = models.SmallIntegerField(
+        default=1, help_text="Increment this to reset the authentication links"
+    )
 
     def meta(self, name):
-        items = WorkMeta.objects.filter(work__collection=self.pk, name=name).values_list('value', flat=True).distinct()
+        items = (
+            WorkMeta.objects.filter(work__collection=self.pk, name=name)
+            .values_list("value", flat=True)
+            .distinct()
+        )
         return items
 
     @property
     def tags(self):
-        return self.meta('tag')
+        return self.meta("tag")
 
     @property
     def genres(self):
-        return self.meta('genre')    
+        return self.meta("genre")
 
     def has_administrator(self, user):
         if not user.is_authenticated:
             return False
         if user.is_superuser:
             return True
-        return user.pk in self.administrators.values_list('pk', flat=True)
+        return user.pk in self.administrators.values_list("pk", flat=True)
 
     def get_absolute_url(self):
-        return resolve_url('collection_work_list', self.pk)
+        return resolve_url("collection_work_list", self.pk)
 
     def auth(self):
-        return sign_data(f'{self.pk}-{self.nonce}', 12)
+        return sign_data(f"{self.pk}-{self.nonce}", 12)
 
     def __str__(self):
         return self.name
 
+
 class EnsembleAccess(models.Model):
     """
     Can have different access levels to a collection
@@ -143,12 +177,18 @@ class EnsembleAccess(models.Model):
     ACCESS_APPROVED = 2
 
     ACCESS_TYPES = (
-        (ACCESS_UNLIMITED, 'Unlimited'),
+        (ACCESS_UNLIMITED, "Unlimited"),
-        (ACCESS_APPROVED, 'Approval required'),
+        (ACCESS_APPROVED, "Approval required"),
     )
 
-    ensemble = models.ForeignKey('interface.Ensemble', on_delete=models.CASCADE, related_name="allowed_collections")
+    ensemble = models.ForeignKey(
-    collection = models.ForeignKey(Collection, on_delete=models.CASCADE, related_name="allowed_ensembles")
+        "interface.Ensemble",
+        on_delete=models.CASCADE,
+        related_name="allowed_collections",
+    )
+    collection = models.ForeignKey(
+        Collection, on_delete=models.CASCADE, related_name="allowed_ensembles"
+    )
     access_type = models.PositiveSmallIntegerField(choices=ACCESS_TYPES, default=2)
 
     class Meta:
@@ -159,6 +199,7 @@ class Work(models.Model):
     """
     A musical work 'owned' by a collection from a licencing perspective.
     """
+
     LICENCE_PUBLIC = 2
     LICENCE_EXPIRED = 4
     LICENCE_RECORDING = 5
@@ -167,37 +208,73 @@ class Work(models.Model):
     LICENCE_NONE = 10
 
     LICENCE_TYPES = (
-        (LICENCE_PUBLIC, 'Public Domain'),
+        (LICENCE_PUBLIC, "Public Domain"),
-        (LICENCE_EXPIRED, 'Copyright Expired'),
+        (LICENCE_EXPIRED, "Copyright Expired"),
-        (LICENCE_RECORDING, 'Recording Licence'),
+        (LICENCE_RECORDING, "Recording Licence"),
-        (LICENCE_PERFORMANCE, 'Performance Licence'),
+        (LICENCE_PERFORMANCE, "Performance Licence"),
-        (LICENCE_PERUSAL, 'Perusal Licence'),
+        (LICENCE_PERUSAL, "Perusal Licence"),
-        (LICENCE_NONE, 'Internal use only'),
+        (LICENCE_NONE, "Internal use only"),
     )
 
     name = models.CharField(max_length=255, help_text="Original name of the work")
-    edition = models.CharField(max_length=255, blank=True,
+    edition = models.CharField(
-            help_text="Edition details to distinguish multiple versions")
+        max_length=255,
-    parent = models.ForeignKey('Work', null=True, blank=True, on_delete=models.SET_NULL, related_name="related_works",
+        blank=True,
-            help_text="Arrangement of another work or part of an anthology")
+        help_text="Edition details to distinguish multiple versions",
-    composer = models.CharField(max_length=255, default='Anon',
+    )
-            help_text="Composer or compilation editor. Use <b>Surname, Initial</b> for easy searching")
+    parent = models.ForeignKey(
+        "Work",
+        null=True,
+        blank=True,
+        on_delete=models.SET_NULL,
+        related_name="related_works",
+        help_text="Arrangement of another work or part of an anthology",
+    )
+    composer = models.CharField(
+        max_length=255,
+        default="Anon",
+        help_text="Composer or compilation editor. Use <b>Surname, Initial</b> for easy searching",
+    )
 
-    orchestration = models.ForeignKey(Orchestration, on_delete=models.SET_DEFAULT, default=1, help_text="Orchestration for the work")
+    orchestration = models.ForeignKey(
-    original_parts = models.JSONField(default=dict, blank=True, help_text="Original printed parts (IMSLP format)")
+        Orchestration,
+        on_delete=models.SET_DEFAULT,
+        default=1,
+        help_text="Orchestration for the work",
+    )
+    original_parts = models.JSONField(
+        default=dict, blank=True, help_text="Original printed parts (IMSLP format)"
+    )
 
     # Collection details
-    collection = models.ForeignKey(Collection, on_delete=models.CASCADE, related_name="works")
+    collection = models.ForeignKey(
-    code = models.CharField(max_length=100, blank=True, help_text="Collection specific code or number. Will be auto-generated if not supplied")
+        Collection, on_delete=models.CASCADE, related_name="works"
-    licence = models.PositiveSmallIntegerField(choices=LICENCE_TYPES, default=6, help_text="Copyright status")
+    )
-    max_projects = models.IntegerField(default=1, help_text="How many active projects can this work be attached to")
+    code = models.CharField(
+        max_length=100,
+        blank=True,
+        help_text="Collection specific code or number. Will be auto-generated if not supplied",
+    )
+    licence = models.PositiveSmallIntegerField(
+        choices=LICENCE_TYPES, default=6, help_text="Copyright status"
+    )
+    max_projects = models.IntegerField(
+        default=1, help_text="How many active projects can this work be attached to"
+    )
 
     # Extra info
-    running_time = models.DurationField(null=True, blank=True, help_text="Running time in mm:ss format")
+    running_time = models.DurationField(
+        null=True, blank=True, help_text="Running time in mm:ss format"
+    )
     notes = models.TextField(blank=True)
 
     # Allocation to projects
-    projects = models.ManyToManyField('interface.Project', through='ProjectItem', related_name="works", help_text="Current usage")
+    projects = models.ManyToManyField(
+        "interface.Project",
+        through="ProjectItem",
+        related_name="works",
+        help_text="Current usage",
+    )
 
     @property
     def folder(self):
@@ -205,19 +282,20 @@ class Work(models.Model):
 
     def tagged_sections(self, *tags):
         qs = self.docs.filter(sections__tag__in=tags)
-        qs = qs.annotate(Count('sections'), end=Min('sections__end'), start=Max('sections__start')) \
+        qs = qs.annotate(
-            .filter(sections__count=len(tags))
+            Count("sections"), end=Min("sections__end"), start=Max("sections__start")
+        ).filter(sections__count=len(tags))
         return qs
 
     def list_sections(self, *tags):
-        return list(self.tagged_sections(*tags).values_list('upload', 'start', 'end'))
+        return list(self.tagged_sections(*tags).values_list("upload", "start", "end"))
 
     @property
     def digital_parts(self):
-        sections = [ (s.tag, s.name) for s in Section.objects.filter(doc__work=self.pk) ]
+        sections = [(s.tag, s.name) for s in Section.objects.filter(doc__work=self.pk)]
         sections.sort(key=self.orchestration.sorter())
-        #return [ s[1] for s in sections ]
+        # return [ s[1] for s in sections ]
-        sections = list(dict(sections).items()) # primitive unique()
+        sections = list(dict(sections).items())  # primitive unique()
         return sections
 
     def pdfs(self):
@@ -229,19 +307,21 @@ class Work(models.Model):
             return []
         parts = list(self.original_parts.items())
         parts.sort(key=self.orchestration.sorter())
-        return [ (MusicTag.from_tag(x[0]), x[1]) for x in parts ]
+        return [(MusicTag.from_tag(x[0]), x[1]) for x in parts]
 
     @property
     def tags(self):
-        return self.meta_info.filter(name='tag').values_list('value', flat=True)
+        return self.meta_info.filter(name="tag").values_list("value", flat=True)
 
     @property
     def meta(self):
-        return self.meta_info.exclude(name='tag')
+        return self.meta_info.exclude(name="tag")
 
     @property
     def current_loans(self):
-        return self.project_items.filter(checkout__lte=now(), returned=None).select_related('project')
+        return self.project_items.filter(
+            checkout__lte=now(), returned=None
+        ).select_related("project")
 
     @cached_property
     def loans(self):
@@ -259,29 +339,30 @@ class Work(models.Model):
     @property
     def available(self):
         if self.max_projects < 0:
-            return 'Unlimited'
+            return "Unlimited"
         a = self.max_projects - self.loans
-        return '{0} of {1}'.format(max(a, 0), self.max_projects)
+        return "{0} of {1}".format(max(a, 0), self.max_projects)
 
     @property
     def identifier(self):
-
         if self.code:
-            return self.code;
+            return self.code
 
         composer = self.composer or "Anon"
-        composer = re.sub('[^\w]', '', composer)
+        composer = re.sub(r"[^\w]", "", composer)
         words = self.name.split()
         work = words[0][:3]
 
         return f"{composer[:4]}-{work}-{self.pk:05d}".upper()
 
     def assigned_instruments(self):
-        return Section.objects.filter(doc__work_id=self.pk).values_list('tag', flat=True)
+        return Section.objects.filter(doc__work_id=self.pk).values_list(
+            "tag", flat=True
+        )
 
     def unassigned_instruments(self):
         assigned = set(self.assigned_instruments())
-        return [ x for x in self.orchestration.as_list() if not x[0] in assigned ]
+        return [x for x in self.orchestration.as_list() if x[0] not in assigned]
 
     def music_tags(self):
         tags = dict(self.orchestration.as_list())
@@ -290,31 +371,33 @@ class Work(models.Model):
 
         return tags.items()
 
-
     def __str__(self):
         return f"{self.name} ({self.composer})"
 
-class WorkMeta(models.Model):
 
+class WorkMeta(models.Model):
     META_CHOICES = (
-        ('tag', 'Tag'),
+        ("tag", "Tag"),
-        ('arr', 'Arranger'),
+        ("arr", "Arranger"),
-        ('lyrics', 'Lyracist'),
+        ("lyrics", "Lyracist"),
-        ('genre', 'Genre'),
+        ("genre", "Genre"),
-        ('style', 'Style'),
+        ("style", "Style"),
-        ('orchestration', 'Orchestration'),
+        ("orchestration", "Orchestration"),
+        ("folderid", "GDrive"),
     )
 
-    work = models.ForeignKey(Work, on_delete=models.CASCADE, related_name='meta_info')
+    work = models.ForeignKey(Work, on_delete=models.CASCADE, related_name="meta_info")
     name = models.SlugField(max_length=20, choices=META_CHOICES)
     value = models.CharField(max_length=255)
 
+
 def doc_upload_filename(doc, filename):
     collection = doc.work.collection
     storage = collection.storage
     if not storage:
         raise RuntimeError("Collection has no storage attached")
-    return f'{storage}:library/{collection.prefix}/{doc.work.folder}/{filename}'
+    return f"{storage}:library/{collection.prefix}/{doc.work.folder}/{filename}"
+
 
 class Document(models.Model):
     """
@@ -324,27 +407,35 @@ class Document(models.Model):
     DOCTYPE_PDF = 1
     DOCTYPE_AUDIO = 2
     DOCTYPE_VIDEO = 3
-    DOCTYPE_MISC= 4
+    DOCTYPE_MISC = 4
 
     DOCTYPES = (
-        (DOCTYPE_PDF, 'PDF'),
+        (DOCTYPE_PDF, "PDF"),
-        (DOCTYPE_AUDIO, 'Audio'),
+        (DOCTYPE_AUDIO, "Audio"),
-        (DOCTYPE_VIDEO, 'Video'),
+        (DOCTYPE_VIDEO, "Video"),
-        (DOCTYPE_MISC, 'Misc'),
+        (DOCTYPE_MISC, "Misc"),
     )
 
     DOCTYPE_MAP = {
-        '.pdf': DOCTYPE_PDF,
+        ".pdf": DOCTYPE_PDF,
-        '.mp3': DOCTYPE_AUDIO,
+        ".mp3": DOCTYPE_AUDIO,
-        '.mp4': DOCTYPE_VIDEO,
+        ".mp4": DOCTYPE_VIDEO,
     }
 
-    work = models.ForeignKey('Work', on_delete=models.CASCADE, related_name="docs")
+    work = models.ForeignKey("Work", on_delete=models.CASCADE, related_name="docs")
     doctype = models.PositiveSmallIntegerField(choices=DOCTYPES, default=DOCTYPE_PDF)
     upload = models.FileField(upload_to=doc_upload_filename, storage=library_storage)
     created = models.DateTimeField(auto_now_add=True)
     version = models.CharField(max_length=30, blank=True)
 
+    def auto_tag(self):
+        if self.doctype != self.DOCTYPE_PDF:
+            return
+        filename = os.path.basename(str(self.upload))
+        inst = auto_tag(filename)
+        if inst:
+            self.sections.get_or_create(tag=inst.abbreviate())
+
     def delete(self, *args, **kwargs):
         self.upload.delete(save=False)
         return super().delete(*args, **kwargs)
@@ -355,6 +446,7 @@ class Document(models.Model):
     def __str__(self):
         return self.upload.name
 
+
 class Section(models.Model):
     """
     Section is a tagged portion of a Document
@@ -365,20 +457,21 @@ class Section(models.Model):
     PAGE_RIGHT = 2
 
     PAGE_PREFERENCE = (
-        (PAGE_AUTO, 'auto'),
+        (PAGE_AUTO, "auto"),
-        (PAGE_LEFT, 'left'),
+        (PAGE_LEFT, "left"),
-        (PAGE_RIGHT, 'right'),
+        (PAGE_RIGHT, "right"),
     )
 
-
     doc = models.ForeignKey(Document, on_delete=models.CASCADE, related_name="sections")
     tag = models.CharField(max_length=50, blank=True)
     start = models.SmallIntegerField(null=True, blank=True)
     end = models.SmallIntegerField(null=True, blank=True)
-    page = models.SmallIntegerField(default=PAGE_AUTO, choices=PAGE_PREFERENCE) # NOT CURRENTLY USED
+    page = models.SmallIntegerField(
+        default=PAGE_AUTO, choices=PAGE_PREFERENCE
+    )  # NOT CURRENTLY USED
 
     class Meta:
-        ordering = ['doc', 'start', 'pk']
+        ordering = ["doc", "start", "pk"]
 
     @property
     def music_tag(self):
@@ -390,11 +483,11 @@ class Section(models.Model):
 
     @property
     def bulma_class(self):
-        return "success" if self.music_tag.is_general else 'info'
+        return "success" if self.music_tag.is_general else "info"
 
     @property
     def filename(self):
-        return slugify(f'{self.doc.work.name} - {self.name}').title() + '.pdf'
+        return slugify(f"{self.doc.work.name} - {self.name}").title() + ".pdf"
 
     @property
     def pagerange(self):

app/library/music_tags.py

@@ -1,5 +1,5 @@
-
 from collections import namedtuple
+import re
 
 GENERAL = """
 mvmt    Movement
@@ -158,20 +158,21 @@ zith 	Zither
 MUSIC_TAGS = []
 GENERAL_TAGS = set()
 for i, abbreviations in enumerate((GENERAL, INSTRUMENTS)):
-    for line in abbreviations.split('\n'):
+    for line in abbreviations.split("\n"):
         parts = line.strip().split(maxsplit=1)
-        if len(parts) < 2: continue
+        if len(parts) < 2:
-        name, _, _ = parts[1].partition('(')
+            continue
+        name, _, _ = parts[1].partition("(")
         MUSIC_TAGS.append((parts[0], name))
         if i == 0:
             GENERAL_TAGS.add(parts[0])
 
 
 MUSIC_NAME_BY_TAG = dict(MUSIC_TAGS)
-MUSIC_TAG_BY_NAME = dict( ( (x[1].lower(), x[0]) for x in MUSIC_TAGS ) )
+MUSIC_TAG_BY_NAME = dict(((x[1].lower(), x[0]) for x in MUSIC_TAGS))
 
-class MusicTag(namedtuple('MusicTag', ('name', 'variant'), defaults=[None])):
 
+class MusicTag(namedtuple("MusicTag", ("name", "variant"), defaults=[None])):
     @classmethod
     def from_tag(cls, tag):
         """
@@ -186,7 +187,7 @@ class MusicTag(namedtuple('MusicTag', ('name', 'variant'), defaults=[None])):
         >>> MusicTag.from_tag('pce-A2')
         MusicTag(name='Piece', variant='A2')
         """
-        abbr, _, variant = tag.partition('-')
+        abbr, _, variant = tag.partition("-")
         name = MUSIC_NAME_BY_TAG.get(abbr.lower(), abbr)
 
         if variant:
@@ -195,8 +196,8 @@ class MusicTag(namedtuple('MusicTag', ('name', 'variant'), defaults=[None])):
 
     @property
     def tag(self):
-        l = self.name.lower()
+        lc = self.name.lower()
-        return MUSIC_TAG_BY_NAME.get(l, l)
+        return MUSIC_TAG_BY_NAME.get(lc, lc)
 
     @property
     def is_general(self):
@@ -231,11 +232,16 @@ class MusicTag(namedtuple('MusicTag', ('name', 'variant'), defaults=[None])):
             return f"{self.name} {self.variant}"
         return self.name
 
-import re
+
-PATTERNS = [re.compile('([A-Za-z]+)[_\- ]*(\d+)'), re.compile('([A-Za-z]+)()')]
+PATTERNS = [
+    re.compile(r"(?P<inst>[A-Za-z]+)[_\- ]*(?P<ord>\d+)"),
+    re.compile(r"(?P<ord>\d+)(st|nd|rd|th)[_\- ]*(?P<inst>[A-Za-z]+)"),
+    re.compile(r"(?P<inst>[A-Za-z]+)()"),
+]
+
 
 def auto_tag(filename):
-    '''
+    """
 
     >>> auto_tag('Ode to Joy - Violin 1.pdf')
     MusicTag(name='Violin', variant=1)
@@ -247,19 +253,27 @@ def auto_tag(filename):
     MusicTag(name='Viola', variant=None)
     >>> auto_tag('Ode to Joy - fl-2 (piccolo).pdf')
     MusicTag(name='Flute', variant=2)
-    '''
+    >>> auto_tag('1st Violin - Ode to Joy.pdf')
+    MusicTag(name='Violin', variant=1)
+    >>> auto_tag('Ode to Joy - 2nd Violin.pdf')
+    MusicTag(name='Violin', variant=2)
+
+    """
 
     for pattern in PATTERNS:
-        for inst, ordinal in pattern.findall(filename):
+        for m in pattern.finditer(filename):
-            inst = inst.lower()
+            inst = m["inst"].lower()
-            ordinal = int(ordinal) if ordinal else None
+            try:
+                ordinal = int(m["ord"])
+            except IndexError:
+                ordinal = None
             if inst in MUSIC_TAG_BY_NAME:
                 return MusicTag(inst.title(), ordinal)
             if inst in MUSIC_NAME_BY_TAG:
                 return MusicTag(MUSIC_NAME_BY_TAG[inst], ordinal)
 
 
-
 if __name__ == "__main__":
     import doctest
+
     print(doctest.testmod())

app/library/pdf_utils.py

@@ -5,22 +5,23 @@ import string
 
 SAFECHARS = string.ascii_letters + string.digits + " _-"
 
+
 def extract_pages(source, bookmark, start=None, end=None, count=1):
 
     return extract_and_concat([(source, bookmark, start, end, count)])
 
+
 def extract_and_concat(items):
 
     # create a temporary directory for our sections
     d = tempfile.TemporaryDirectory(prefix="polyphonic_")
-    pdfmarks = os.path.join(d.name, 'pdfmarks.txt')
+    pdfmarks = os.path.join(d.name, "pdfmarks.txt")
-    marks = open(pdfmarks, 'w')
+    marks = open(pdfmarks, "w")
 
     sections = []
     current_page = 1
 
     for i, (source, bookmark, start, end, count) in enumerate(items):
-
         if count == 0:
             continue
 
@@ -28,23 +29,34 @@ def extract_and_concat(items):
             sections.append(source)
 
         else:
-
             if not end:
                 end = start
 
-            dest = os.path.join(d.name, f'section_{i}.pdf')
+            dest = os.path.join(d.name, f"section_{i}.pdf")
 
-            cmd = ['gs', '-sDEVICE=pdfwrite', '-dBATCH', '-dNOPAUSE', 
+            cmd = [
-                f'-dFirstPage={start}', f'-dLastPage={end}', 
+                "gs",
-                f'-sOutputFile={dest}',
+                "-sDEVICE=pdfwrite",
-                source]
+                "-dBATCH",
+                "-dNOPAUSE",
+                f"-dFirstPage={start}",
+                f"-dLastPage={end}",
+                f"-sOutputFile={dest}",
+                source,
+            ]
 
             bookmark = "".join(filter(lambda c: c in SAFECHARS, bookmark))
 
-            marks.write(f'[/Title ({bookmark}) /Page {current_page} /OUT pdfmark\n')
+            marks.write(f"[/Title ({bookmark}) /Page {current_page} /OUT pdfmark\n")
 
             p = subprocess.run(cmd, check=True, capture_output=True)
-            pages = len([ x for x in p.stdout.splitlines() if x.decode('utf8').startswith('Page ')])
+            pages = len(
+                [
+                    x
+                    for x in p.stdout.splitlines()
+                    if x.decode("utf8").startswith("Page ")
+                ]
+            )
             for j in range(count):
                 sections.append(dest)
                 current_page += pages
@@ -52,10 +64,9 @@ def extract_and_concat(items):
     marks.close()
 
     # concat the items
-    output = tempfile.NamedTemporaryFile(prefix="polyphonic_", suffix='.pdf')
+    output = tempfile.NamedTemporaryFile(prefix="polyphonic_", suffix=".pdf")
 
-    cmd = ['gs', '-sDEVICE=pdfwrite', '-q', '-dBATCH', '-dNOPAUSE',
+    cmd = ["gs", "-sDEVICE=pdfwrite", "-q", "-dBATCH", "-dNOPAUSE", "-sOutputFile=-"]
-           '-sOutputFile=-']
     cmd.extend(sections)
     cmd.append(pdfmarks)
 

app/library/templates/library/collection_list.html

@@ -7,7 +7,7 @@
 <form method="GET" action="{% url 'work_list' %}">
     <div class="field has-addons">
         <div class="control is-expanded">
-            <input class="input" name="filter" type="text" placeholder="Find a work" value="{{ request.GET.filter }}"/>
+            <input class="input" name="q" type="text" placeholder="Find a work" value="{{ request.GET.filter }}"/>
         </div>
         <div class="control">
             <a class="button" href="?"><i class="fas fa-times"></i></a>
@@ -32,10 +32,10 @@
                 </p>
                 <p>
                     {% for tag in collection.tags %}
-                    <a href="{% url 'collection_work_list' collection.pk %}?filter=tag:{{ tag }}" class="tag is-success">{{ tag }}</a>
+                    <a href="{% url 'collection_work_list' collection.pk %}?q=tag:{{ tag }}" class="tag is-success">{{ tag }}</a>
                     {% endfor %}
                     {% for genre in collection.genres %}
-                    <a href="{% url 'collection_work_list' collection.pk %}?filter=genre:{{ genre }}" class="tag is-warning">{{ genre }}</a>
+                    <a href="{% url 'collection_work_list' collection.pk %}?q=genre:{{ genre }}" class="tag is-warning">{{ genre }}</a>
                     {% endfor %}
                 </p>
             </div>

app/library/templates/library/document_annotate.html

@@ -139,7 +139,7 @@
   crossorigin="anonymous"></script>
 {{ json_data|json_script:"data" }}
 <script type="text/javascript">
-  let url = "{{ document.upload.url|safe }}";
+  let url = "{% url 'document_download' collection.pk document.pk %}";
 
   // Loaded via <script> tag, create shortcut to access PDF.js exports.
   let pdfjsLib = window['pdfjs-dist/build/pdf'];

app/library/templates/library/gdrive.html

@@ -0,0 +1,41 @@
+{% extends "interface/project_base.html" %}
+
+{% block admin %}
+<a href="{% url 'work_detail' collection=collection.pk pk=object.pk %}" class="button is-link is-light">
+  <span>Back to work</span>
+</a>
+{% endblock %}
+
+{% block page %}
+<h3 class="subtitle"><a href="{% url 'work_detail' collection.pk object.pk %}">{{ object.name }}</a></h3>
+<div class="m-3">
+<p>This page lets you link a work to a google drive folder.  You can either paste a public link to a folder to enable syncing or a file to add individually</p>
+</div>
+<div class="m-3">
+<p>
+    {% if meta.folderid %}
+This work is currently linked to <b>{{ meta.folderid }}</b>.  Pasting a new folder link will overwrite this.
+    {% else %}
+There is currently no shared drive folder linked to this work - paste one here to enable syncing.
+    {% endif %}
+</p>
+</div>
+<div>
+<form method="post">
+  <div class="field">
+    <div class="control">
+      <input name="link" class="input is-expanded" type="text" placeholder="Shared link">
+    </div>
+    {% for error in form.errors.link %}
+    <p class="help is-danger">{{ error }}</p>
+    {% endfor %}
+  </div>
+  <div class="field">
+    <div class="control">
+      <button class="button is-info" type="submit">Add</button>
+    </div>
+  </div>
+  {% csrf_token %}
+</form>
+</div>
+{% endblock %}

app/library/templates/library/storage_browser.html

@@ -0,0 +1,12 @@
+{% load path_filters %}
+<div>
+  <h3>Listing for {{ collection }}</h3>
+  <ul>
+    {% for item in folders %}
+      <li><a href="{% url 'storage_browser_folder' collection=collection.pk folder=item %}">{{ item|basename }}</a></li>
+    {% endfor %}
+    {% for item in files %}
+      <li>{{ item|basename }}</li>
+    {% endfor %}
+  </ul>
+</div>

app/library/templates/library/work_detail.html

@@ -25,46 +25,46 @@
     {% endfor %}
 </h3>
 <p class="subtitle">{% firstof work.composer "Unattributed" %}{% if work.edition %} - {{ work.edition }}{% endif %}</p>
-<section class="block">
-    <p class="block">{{ work.notes }}</p>
-
-    <p class="block">
-    <table class="table">
-        <tr>
-            <th>Location:</th><td><a href="{% url 'collection_work_list' work.collection.pk %}">{{ work.collection }}</a> [{{ work.identifier }}]</td>
-            <th>Orchestration:</th><td>{{ work.orchestration }}</td>
-        </tr><tr>
-            <th>Running time:</th><td>{% firstof work.duration 'Unknown' %}</td>
-            <th>Licence:</th><td>{{ work.get_licence_display }}</td>
-        </tr><tr>
-            <td colspan="4">
-    {% for meta in work.meta %}
-        <a href="{% url 'collection_work_list' work.collection.pk %}?filter={{ meta.name}}:{{ meta.value }}" class="tag" >
-            {{ meta.get_name_display }}:
-            {{ meta.value }}
-        </a>
-    {% endfor %}
-            </td>
-        </tr>
-    </table>
-
-    {% if work.parent %}
-    <p>From <a href="{% url 'work_detail' work.parent.collection.pk work.parent.pk %}">{{ work.parent.name }} - {{ work.parent.composer }}</a>
-    </p>
-    {% endif %}
-
-    {% if work.related_works.count %}
-    <h3>Related</h3>
-    <ul>
-        {% for related in work.related_works.all %}
-        <li><a href="{% url 'work_detail' related.collection.pk related.pk %}">{{ related.name }} - {{ related.composer }}</a></li>
-        {% endfor %}
-    </ul>
-    {% endif %}
-</section>
 <section class="block">
 
     <div class="columns">
+        <div class="column is-half">
+        <p class="block">{{ work.notes }}</p>
+
+        <p class="block">
+        <table class="table">
+            <tr>
+                <th>Location:</th><td><a href="{% url 'collection_work_list' work.collection.pk %}">{{ work.collection }}</a> [{{ work.identifier }}]</td>
+                <th>Orchestration:</th><td>{{ work.orchestration }}</td>
+            </tr><tr>
+                <th>Running time:</th><td>{% firstof work.duration 'Unknown' %}</td>
+                <th>Licence:</th><td>{{ work.get_licence_display }}</td>
+            </tr><tr>
+                <td colspan="4">
+        {% for meta in work.meta %}
+            <a href="{% url 'collection_work_list' work.collection.pk %}?filter={{ meta.name}}:{{ meta.value }}" class="tag" >
+                {{ meta.get_name_display }}:
+                {{ meta.value }}
+            </a>
+        {% endfor %}
+                </td>
+            </tr>
+        </table>
+
+        {% if work.parent %}
+        <p>From <a href="{% url 'work_detail' work.parent.collection.pk work.parent.pk %}">{{ work.parent.name }} - {{ work.parent.composer }}</a>
+        </p>
+        {% endif %}
+
+        {% if work.related_works.count %}
+        <h3>Related</h3>
+        <ul>
+            {% for related in work.related_works.all %}
+            <li><a href="{% url 'work_detail' related.collection.pk related.pk %}">{{ related.name }} - {{ related.composer }}</a></li>
+            {% endfor %}
+        </ul>
+        {% endif %}
+        </div>
         <div class="column is-half">
             <div class="box">
                 <h4 class="subtitle is-size-4">
@@ -79,8 +79,6 @@
                     {% endfor %}
                 </div>
             </div>
-        </div>
-        <div class="column is-half">
             <div class="box">
                 <h4 class="subtitle is-size-4">
                     <span class="icon"><i class="fas fa-print"></i></span>
@@ -130,10 +128,17 @@
             </div>
             {% if request.is_admin %}
             <div class="column is-one-quarter">
-                <h4 class="is-size-5">Upload files</h4>
+                <h4 class="is-size-5">Add Files</h4>
+                {% if "gdrive" in methods %}
+                <div class="has-text-centered mt-3">
+                  <a class="button button-primary" href="{% url 'work_gdrive' collection.pk object.pk %}">Link Google Drive Files</a><br/>
+                </div>
+                {% endif %}
+                {% if "upload" in methods %}
                 <form action="{% url 'document_add' collection.pk object.pk %}" class="dropzone" id="doc-upload" style="-moz-user-select: none">
                     {% csrf_token %}
                 </form>
+                {% endif %}
             </div>
             {% endif %}
         </div>

app/library/templates/library/work_list.html

@@ -15,7 +15,7 @@
 <form method="GET">
     <div class="field has-addons">
         <div class="control is-expanded">
-            <input class="input" name="filter" type="text" placeholder="Filter" value="{{ request.GET.filter }}"/>
+            <input class="input" name="q" type="text" placeholder="Filter" value="{{ request.GET.q }}"/>
         </div>
         <div class="control">
             <a class="button" href="?"><i class="fas fa-times"></i></a>
@@ -23,6 +23,15 @@
     </div>
 </form>
 
+{% if letters %}
+<div class="has-text-centered">
+<a href="?">_</a>
+{% for letter in letters %}
+<a href="?start={{ letter }}">{{ letter }}</a>&nbsp;
+{% endfor %}
+</div>
+{% endif %}
+
 <table class="table is-striped is-fullwidth">
     <thead>
         <tr>
@@ -35,7 +44,9 @@
     <tbody>
         {% for work in object_list %}
             <tr>
-                <td><a href="{% url 'work_detail' collection=work.collection.pk pk=work.pk %}">{{ work.name }}</a></td>
+                <td>
+                  <a href="{% url 'work_detail' collection=work.collection_id pk=work.pk %}">{{ work.name }}</a>
+                </td>
                 <td title="{{ work.composer }}">{{ work.composer|truncatewords:3 }}</td>
                 <td class="is-hidden-mobile" title="{{ work.edition }}">{{ work.edition|truncatewords:2 }}</td>
                 {% if not collection %}<td class="is-hidden-touch">{{ work.collection.name }}</td>{% endif %}
@@ -46,20 +57,41 @@
     </tbody>
 </table>
 
+<footer>
 <nav class="pagination is-centered" role="navigation" aria-label="pagination">
-    {% if page_obj.has_previous %}
+    <a class="pagination-previous" 
-    <a class="pagination-previous" href="{% url_update page=page_obj.prev_page_number %}">Previous</a>
+      {% if page_obj.has_previous %}
-    {% endif %}
+        href="{% url_update page=page_obj.previous_page_number %}"
-    {% if page_obj.has_next %}
+      {% else %}
-    <a class="pagination-next" href="{% url_update page=page_obj.next_page_number %}">Next page</a>
+        disabled
-    {% endif %}
+    {% endif %}>
+      Previous</a>
+    <a class="pagination-next"
+      {% if page_obj.has_next %}
+        href="{% url_update page=page_obj.next_page_number %}"
+      {% else %}
+        disabled
+      {% endif %}>
+      Next
+    </a>
     <ul class="pagination-list">
-      {% for page in page_obj.paginator.page_range %}
+      {% for page in page_range %}
         <li>
-            <a class="pagination-link {% if forloop.counter == page_obj.number %}is-current{% endif %}" href="{% url_update page=forloop.counter %}" aria-label="Goto page {{ forloop.counter }}">{{ forloop.counter }}</a>
+            {% if page == '...' %}
+              {{ page }}
+            {% else %}
+              <a class="pagination-link {% if page == page_obj.number %}is-current{% endif %}" href="{% url_update page=page %}" aria-label="Goto page {{ page }}">{{ page }}</a>
+            {% endif %}
         </li>
       {% endfor %}
     </ul>
 </nav>
 
+{% if meta %}
+<div class="has-text-right has-text-grey-light is-size-7">
+Query="{{ meta.query }}"
+</div>
+{% endif %}
+</footer>
+
 {% endblock %}

app/library/templates/library/work_parts_fragment.html

@@ -1,10 +1,11 @@
 <h3 class="subtitle">{{ work.name }}</h3>
-<div class="block">
+<div class="block tags">
 {% for tag, name in work.digital_parts %}
-<a class="tag is-info" href="{% url 'work_download' work.collection_id work.pk %}?tag={{ tag }}">{{ name }}</a>
+<a class="tag is-info" href="{% url 'work_download' work.collection_id work.pk %}?tag={{ tag }}" target="polyphonic_parts">{{ name }}</a>
 {% endfor %}
 </div>
 
+<!--
 <h3 class="subtitle">Files</h3>
 <table class="table is-narrow is-fullwidth">
 <tbody>
@@ -19,6 +20,7 @@
     {% endfor %}
 </tbody>
 </table>
+-->
 
 <p>
 <a href="{% url 'work_detail' work.collection_id work.pk %}">More details...</a>

app/library/tests.py

@@ -1,176 +0,0 @@
-from interface.tests import AccessTestCase 
-
-from byostorage.user import UserStorage
-from . import models
-from .views.api import WorkSerializer
-
-import tempfile
-import json
-
-class LibraryTestCase(AccessTestCase):
-
-    USERS = (
-        {'username': 'admin', 'password': 'secret', 'is_superuser': True, 'is_staff': True},
-        {'username': 'homer', 'password': 'maggie'},
-    )
-
-    ENSEMBLES = (
-        {'name': 'The Be Sharps', 'slug': 'be-sharps', 'admins': ['homer']},
-        {'name': 'Lisa & the Bleeding Gums', 'slug': 'bleeding-gums'},
-        {'name': 'Party Posse'},
-    )
-
-    PROJECTS = (
-        {'name': 'Baker St', 'ensemble': 'bleeding-gums', 'when': -12},
-        {'name': 'Navy Recruitment Day', 'ensemble': 'party-posse', 'when': 6},
-        {'name': 'Barbershop Contest', 'ensemble': 'be-sharps', 'when': 28},
-        {'name': 'Open Mic Night', 'ensemble': 'bleeding-gums', 'when': 1 },
-    )
-
-    COLLECTIONS = (
-        {'name': 'Springfield Elementary Library', 'prefix': 'sel'},
-        {'name': 'Neds Library', 'prefix': 'ned', 'admins': ['homer']},
-    )
-
-    WORKS = (
-        {'name': 'Baby on Board', 'collection': 'ned', 'docs': [{'upload': 'local:baby_on_board.pdf'}]},
-        {'name': 'Star Spangled Banner', 'collection': 'sel'},
-    )
-
-    PROTECTED_URLS = (
-        '/collections/1',
-        '/collections/1/add',
-        '/collections/2/works/1',
-        '/collections/2/works/1/edit',
-        '/collections/2/works/1/partset',
-        '/collections/2/works/1/add_to_project',
-        '/collections/2/works/1/upload',
-        '/collections/2/docs/1/annotate',
-
-        # Need to add storage before we can test these
-        '/api/collections/2',
-        '/api/collections/2/works/1',
-
-        '/admin/library/collection/',
-        '/admin/library/document/',
-        '/admin/library/ensembleaccess/',
-        '/admin/library/orchestration/',
-        '/admin/library/projectitem/',
-        '/admin/library/work/',
-    )
-
-    @classmethod
-    def setUpTestData(cls):
-        super().setUpTestData()
-        
-        cls.temp_dir = tempfile.TemporaryDirectory()
-        cls.storage = UserStorage.objects.create(name='local', storage='django.core.files.storage.FileSystemStorage',
-                                            settings_data=json.dumps({'location': cls.temp_dir.name, 'base_url': 'file://' + cls.temp_dir.name}))
-        cls.collections = {}
-        for details in cls.COLLECTIONS:
-            admins = details.pop('admins', [])
-            obj = models.Collection.objects.create(storage=cls.storage, **details)
-            for admin in admins:
-                obj.administrators.add(cls.users[admin])
-            cls.collections[details['prefix']] = obj
-
-        cls.works = {}
-        for details in cls.WORKS:
-            collection = cls.collections[details.pop('collection')]
-            #details.setdefault('docs', [])
-            #details.setdefault('meta_info', [])
-            #s = WorkSerializer(data=details)
-            #assert s.is_valid(), s.errors
-            #s.save(collection_id=collection.pk)
-            docs = details.pop('docs', [])
-            obj = models.Work.objects.create(collection=collection, **details)
-            for doc in docs:
-                obj.docs.create(**doc)
-            cls.works[details['name']] = obj
-
-
-    def setUp(self):
-        pass
-
-    @classmethod
-    def tearDownClass(cls):
-        cls.temp_dir.cleanup()
-        
-
-    def test_integration(self):
-        pass
-
-    def test_superuser_access(self):
-        self.login('admin', 'secret')
-        self.assertAccess({
-            '/collections': True,
-            '/collections/1': True,
-            '/collections/2/works/1': True,
-        })
-
-    def test_administrator_access(self):
-        self.login('homer', 'maggie')
-        self.assertAccess({
-            '/collections': True,
-            '/collections/1': False,
-            '/collections/2': True,
-            '/collections/2/works/1': True,
-        })
-
-    def test_link_access(self):
-        self.assertAccess({
-            '/collections': True,
-            '/collections/1': False,
-            '/collections/2': False,
-            '/collections/2/works/1': False,
-        })
-
-        self.authorize(models.Collection, pk=2)
-        self.assertAccess({
-            '/collections': True,
-            '/collections/1': False,
-            '/collections/2': True,
-            '/collections/2/works/1': True,
-        })
-
-    def test_anon_access(self):
-        self.assertAccess({
-            '/collections': True,
-            '/collections/1': False,
-            '/collections/2': False,
-            '/collections/2/works/1': False,
-        })
-
-    def test_export_and_import(self):
-        self.login('admin', 'secret')
-        data = self.client.get('/api/collections/1/works/2', HTTP_ACCEPT="application/json").json()
-        response = self.client.post('/api/collections/2/import', data, "application/json")
-        self.assertEqual(response.status_code, 201) 
-
-    def test_movement_from_large_work(self):
-        '''
-        Will be common to store a work which has several movements, but the project is only going to play one.
-        This also should give us the ability to store an anthology as one Work have Project reference 'no:23'
-        '''
-
-        work = self.collections['sel'].works.create(name="Some Quartet", composer="Beethoven")
-        for g in ('vl-1', 'vl-2', 'vla', 'vc'):
-            doc = work.docs.create(upload=f'sel/beethoven/some_quartet/some_quartet_{g}.pdf')
-            doc.sections.create(tag='mvmt-1', start=1, end=3)
-            doc.sections.create(tag='mvmt-2', start=4, end=8)
-            doc.sections.create(tag='mvmt-3', start=9, end=12)
-            doc.sections.create(tag=g)
-
-        # no tags - get nothing (should it be everything?)
-        self.assertEqual(work.list_sections(), [])
-
-        # single tag - should get just that range
-        self.assertEqual(work.list_sections('vl-1'), [('sel/beethoven/some_quartet/some_quartet_vl-1.pdf', None, None)])
-
-        # single tag - returns all documents with that range
-        result = work.list_sections('mvmt-2')
-        self.assertEqual(len(result), 4)
-
-        # multiple tags - returns the overlapping portion of all documents that have all tags
-        self.assertEqual(work.list_sections('vl-1', 'mvmt-2'), [('sel/beethoven/some_quartet/some_quartet_vl-1.pdf', 4, 8)])
-        self.assertEqual(work.list_sections('vl-1', 'vl-2'), [])

app/library/tests/__init__.py

@@ -0,0 +1,212 @@
+from interface.tests import AccessTestCase
+
+from byostorage.user import UserStorage
+from library import models
+
+import tempfile
+import json
+
+
+class LibraryTestCase(AccessTestCase):
+    USERS = (
+        {
+            "username": "admin",
+            "password": "secret",
+            "is_superuser": True,
+            "is_staff": True,
+        },
+        {"username": "homer", "password": "maggie"},
+    )
+
+    ENSEMBLES = (
+        {"name": "The Be Sharps", "slug": "be-sharps", "admins": ["homer"]},
+        {"name": "Lisa & the Bleeding Gums", "slug": "bleeding-gums"},
+        {"name": "Party Posse"},
+    )
+
+    PROJECTS = (
+        {"name": "Baker St", "ensemble": "bleeding-gums", "when": -12},
+        {"name": "Navy Recruitment Day", "ensemble": "party-posse", "when": 6},
+        {"name": "Barbershop Contest", "ensemble": "be-sharps", "when": 28},
+        {"name": "Open Mic Night", "ensemble": "bleeding-gums", "when": 1},
+    )
+
+    COLLECTIONS = (
+        {"name": "Springfield Elementary Library", "prefix": "sel"},
+        {"name": "Neds Library", "prefix": "ned", "admins": ["homer"]},
+    )
+
+    WORKS = (
+        {
+            "name": "Baby on Board",
+            "collection": "ned",
+            "docs": [{"upload": "local:baby_on_board.pdf"}],
+        },
+        {"name": "Star Spangled Banner", "collection": "sel"},
+    )
+
+    PROTECTED_URLS = (
+        "/collections/1",
+        "/collections/1/add",
+        "/collections/2/works/1",
+        "/collections/2/works/1/edit",
+        "/collections/2/works/1/partset",
+        "/collections/2/works/1/add_to_project",
+        "/collections/2/works/1/upload",
+        "/collections/2/docs/1/annotate",
+        # Need to add storage before we can test these
+        "/api/collections/2",
+        "/api/collections/2/works/1",
+        "/admin/library/collection/",
+        "/admin/library/document/",
+        "/admin/library/ensembleaccess/",
+        "/admin/library/orchestration/",
+        "/admin/library/projectitem/",
+        "/admin/library/work/",
+    )
+
+    @classmethod
+    def setUpTestData(cls):
+        super().setUpTestData()
+
+        cls.temp_dir = tempfile.TemporaryDirectory()
+        cls.storage = UserStorage.objects.create(
+            name="local",
+            storage="django.core.files.storage.FileSystemStorage",
+            settings_data=json.dumps(
+                {
+                    "location": cls.temp_dir.name,
+                    "base_url": "file://" + cls.temp_dir.name,
+                }
+            ),
+        )
+        cls.collections = {}
+        for details in cls.COLLECTIONS:
+            admins = details.pop("admins", [])
+            obj = models.Collection.objects.create(storage=cls.storage, **details)
+            for admin in admins:
+                obj.administrators.add(cls.users[admin])
+            cls.collections[details["prefix"]] = obj
+
+        cls.works = {}
+        for details in cls.WORKS:
+            collection = cls.collections[details.pop("collection")]
+            # details.setdefault('docs', [])
+            # details.setdefault('meta_info', [])
+            # s = WorkSerializer(data=details)
+            # assert s.is_valid(), s.errors
+            # s.save(collection_id=collection.pk)
+            docs = details.pop("docs", [])
+            obj = models.Work.objects.create(collection=collection, **details)
+            for doc in docs:
+                obj.docs.create(**doc)
+            cls.works[details["name"]] = obj
+
+    def setUp(self):
+        pass
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.temp_dir.cleanup()
+
+    def test_integration(self):
+        pass
+
+    def test_superuser_access(self):
+        self.login("admin", "secret")
+        self.assertAccess(
+            {
+                "/collections": True,
+                "/collections/1": True,
+                "/collections/2/works/1": True,
+            }
+        )
+
+    def test_administrator_access(self):
+        self.login("homer", "maggie")
+        self.assertAccess(
+            {
+                "/collections": True,
+                "/collections/1": False,
+                "/collections/2": True,
+                "/collections/2/works/1": True,
+            }
+        )
+
+    def test_link_access(self):
+        self.assertAccess(
+            {
+                "/collections": True,
+                "/collections/1": False,
+                "/collections/2": False,
+                "/collections/2/works/1": False,
+            }
+        )
+
+        self.authorize(models.Collection, pk=2)
+        self.assertAccess(
+            {
+                "/collections": True,
+                "/collections/1": False,
+                "/collections/2": True,
+                "/collections/2/works/1": True,
+            }
+        )
+
+    def test_anon_access(self):
+        self.assertAccess(
+            {
+                "/collections": True,
+                "/collections/1": False,
+                "/collections/2": False,
+                "/collections/2/works/1": False,
+            }
+        )
+
+    def test_export_and_import(self):
+        self.login("admin", "secret")
+        data = self.client.get(
+            "/api/collections/1/works/2", HTTP_ACCEPT="application/json"
+        ).json()
+        response = self.client.post(
+            "/api/collections/2/import", data, "application/json"
+        )
+        self.assertEqual(response.status_code, 201)
+
+    def test_movement_from_large_work(self):
+        """
+        Will be common to store a work which has several movements, but the project is only going to play one.
+        This also should give us the ability to store an anthology as one Work have Project reference 'no:23'
+        """
+
+        work = self.collections["sel"].works.create(
+            name="Some Quartet", composer="Beethoven"
+        )
+        for g in ("vl-1", "vl-2", "vla", "vc"):
+            doc = work.docs.create(
+                upload=f"sel/beethoven/some_quartet/some_quartet_{g}.pdf"
+            )
+            doc.sections.create(tag="mvmt-1", start=1, end=3)
+            doc.sections.create(tag="mvmt-2", start=4, end=8)
+            doc.sections.create(tag="mvmt-3", start=9, end=12)
+            doc.sections.create(tag=g)
+
+        # no tags - get nothing (should it be everything?)
+        self.assertEqual(work.list_sections(), [])
+
+        # single tag - should get just that range
+        self.assertEqual(
+            work.list_sections("vl-1"),
+            [("sel/beethoven/some_quartet/some_quartet_vl-1.pdf", None, None)],
+        )
+
+        # single tag - returns all documents with that range
+        result = work.list_sections("mvmt-2")
+        self.assertEqual(len(result), 4)
+
+        # multiple tags - returns the overlapping portion of all documents that have all tags
+        self.assertEqual(
+            work.list_sections("vl-1", "mvmt-2"),
+            [("sel/beethoven/some_quartet/some_quartet_vl-1.pdf", 4, 8)],
+        )
+        self.assertEqual(work.list_sections("vl-1", "vl-2"), [])

app/library/tests/test_modules.py

@@ -0,0 +1,8 @@
+from doctest import DocTestSuite
+
+from library import music_tags
+
+
+def load_tests(loader, tests, ignore):
+    tests.addTests(DocTestSuite(music_tags))
+    return tests

app/library/urls.py

@@ -1,44 +1,129 @@
-from django.urls import path, include
+from django.urls import path
-from django.contrib.auth import views as auth_views
-from rest_framework import routers
 
 from . import views
+from .gdrive import views as gdrive_views
 
 from library.views import api
 
-#router = routers.DefaultRouter()
+# router = routers.DefaultRouter()
-#router.register(r'collection', external.CollectionViewSet, basename="collection")
+# router.register(r'collection', external.CollectionViewSet, basename="collection")
-#router.register(r'work', external.WorkViewSet, basename="work")
+# router.register(r'work', external.WorkViewSet, basename="work")
 
 urlpatterns = [
-    
+    path(
-    path('projects/<int:project>/items', views.ProjectItemListView.as_view(), name="item_list"),
+        "projects/<int:project>/items",
-    path('projects/<int:project>/items/manage', views.ProjectItemManageView.as_view(), name="item_list_manage"),
+        views.ProjectItemListView.as_view(),
-    path('projects/<int:project>/items/append', views.ProjectItemAddView.as_view(), name="item_list_append"),
+        name="item_list",
-    
+    ),
-    path('library', views.LibraryWorkListView.as_view(), name="work_list"),
+    path(
-
+        "projects/<int:project>/items/manage",
-    path('collections', views.CollectionListView.as_view(), name="collection_list"),
+        views.ProjectItemManageView.as_view(),
-    path('collections/<int:collection>', views.CollectionWorkListView.as_view(), name="collection_work_list"),
+        name="item_list_manage",
-    path('collections/<int:collection>/add', views.WorkAddView.as_view(), name="work_add"),
+    ),
-    
+    path(
-    path('collections/<int:collection>/works/<int:pk>', views.WorkDetailView.as_view(), name="work_detail"),
+        "projects/<int:project>/items/append",
-    path('collections/<int:collection>/works/<int:pk>/edit', views.WorkUpdateView.as_view(), name="work_edit"),
+        views.ProjectItemAddView.as_view(),
-    path('collections/<int:collection>/works/<int:pk>/partset', views.WorkPartSetView.as_view(), name="work_partset"),
+        name="item_list_append",
-    path('collections/<int:collection>/works/<int:pk>/parts', views.WorkPartsView.as_view(), name="work_parts"),
+    ),
-    path('collections/<int:collection>/works/<int:pk>/add_to_project', views.WorkAddToProject.as_view(), name="work_add_to_project"),
+    path("library", views.LibraryWorkListView.as_view(), name="work_list"),
-    path('collections/<int:collection>/works/<int:pk>/upload', views.WorkAddDocumentView.as_view(), name="document_add"),
+    path("collections", views.CollectionListView.as_view(), name="collection_list"),
-    path('collections/<int:collection>/works/<int:pk>/download', views.WorkDownloadView.as_view(), name="work_download"),
+    path(
-
+        "collections/<int:collection>",
-    path('collections/<int:collection>/docs/<int:pk>/delete', views.DocumentDeleteView.as_view(), name="document_delete"),
+        views.CollectionWorkListView.as_view(),
-    path('collections/<int:collection>/docs/<int:pk>/download', views.DocumentDownloadView.as_view(), name="document_download"),
+        name="collection_work_list",
-    path('collections/<int:collection>/docs/<int:pk>/annotate', views.DocumentAnnotateView.as_view(), name="document_annotate"),
+    ),
-
+    path(
-    path('collections/<int:collection>/download/<int:section>/<str:filename>', views.PartDownloadView.as_view(), name="part_download"),
+        "collections/<int:collection>/add", views.WorkAddView.as_view(), name="work_add"
-
+    ),
-    #path('api/', include(router.urls))
+    path(
-    path('api/collections/<int:pk>', api.CollectionExportView.as_view(), name="collection_export"),
+        "collections/<int:collection>/works/<int:pk>",
-    path('api/collections/<int:collection>/works/<int:pk>', api.WorkExportView.as_view(), name="work_export"),
+        views.WorkDetailView.as_view(),
-    path('api/collections/<int:collection>/import', api.WorkImportView.as_view(), name="work_import"),
+        name="work_detail",
-    path('api/collections/<int:collection>/bulk_import', api.CollectionImportView.as_view(), name="collection_import"),
+    ),
+    path(
+        "collections/<int:collection>/works/<int:pk>/edit",
+        views.WorkUpdateView.as_view(),
+        name="work_edit",
+    ),
+    path(
+        "collections/<int:collection>/works/<int:pk>/partset",
+        views.WorkPartSetView.as_view(),
+        name="work_partset",
+    ),
+    path(
+        "collections/<int:collection>/works/<int:pk>/parts",
+        views.WorkPartsView.as_view(),
+        name="work_parts",
+    ),
+    path(
+        "collections/<int:collection>/works/<int:pk>/add_to_project",
+        views.WorkAddToProject.as_view(),
+        name="work_add_to_project",
+    ),
+    path(
+        "collections/<int:collection>/works/<int:pk>/upload",
+        views.WorkAddDocumentView.as_view(),
+        name="document_add",
+    ),
+    path(
+        "collections/<int:collection>/works/<int:pk>/gdrive",
+        gdrive_views.WorkGDriveView.as_view(),
+        name="work_gdrive",
+    ),
+    path(
+        "collections/<int:collection>/works/<int:pk>/download",
+        views.WorkDownloadView.as_view(),
+        name="work_download",
+    ),
+    path(
+        "collections/<int:collection>/docs/<int:pk>/delete",
+        views.DocumentDeleteView.as_view(),
+        name="document_delete",
+    ),
+    path(
+        "collections/<int:collection>/docs/<int:pk>/download",
+        views.DocumentDownloadView.as_view(),
+        name="document_download",
+    ),
+    path(
+        "collections/<int:collection>/docs/<int:pk>/annotate",
+        views.DocumentAnnotateView.as_view(),
+        name="document_annotate",
+    ),
+    path(
+        "collections/<int:collection>/download/<int:section>/<str:filename>",
+        views.PartDownloadView.as_view(),
+        name="part_download",
+    ),
+    path(
+        "collections/<int:collection>/browse",
+        views.StorageBrowserView.as_view(),
+        name="storage_browser",
+    ),
+    path(
+        "collections/<int:collection>/browse/<path:folder>",
+        views.StorageBrowserView.as_view(),
+        name="storage_browser_folder",
+    ),
+    # path('api/', include(router.urls))
+    path(
+        "api/collections/<int:pk>",
+        api.CollectionExportView.as_view(),
+        name="collection_export",
+    ),
+    path(
+        "api/collections/<int:collection>/works/<int:pk>",
+        api.WorkExportView.as_view(),
+        name="work_export",
+    ),
+    path(
+        "api/collections/<int:collection>/import",
+        api.WorkImportView.as_view(),
+        name="work_import",
+    ),
+    path(
+        "api/collections/<int:collection>/bulk_import",
+        api.CollectionImportView.as_view(),
+        name="collection_import",
+    ),
 ]

app/library/views/__init__.py

@@ -1,79 +1,93 @@
 from django.shortcuts import get_object_or_404, redirect, resolve_url
+from django.http import HttpRequest
+from django.views.generic import TemplateView
 from django.views.generic.detail import DetailView, SingleObjectMixin, View
-from django.views.generic.list import ListView, MultipleObjectMixin
+from django.views.generic.list import ListView
 from django.views.generic.edit import CreateView, FormView, UpdateView, DeleteView
 from django.http import FileResponse, HttpResponse, JsonResponse
-from django.db import IntegrityError
+from django.db.models import Q, Sum
-from django.db.models import Q, Count, Sum
 from django.db import transaction
 from django.utils.timezone import now
-from django.urls import reverse
 from django.template.loader import render_to_string
 from django.core.exceptions import SuspiciousOperation
+from django.core.paginator import Paginator
 from django.http import Http404, HttpResponseRedirect
 
 import json
 import os.path
-import re
+import string
 
-from interface.views import EnsembleMixin, ProjectMixin, AuthorizedResourceMixin
+from interface.views import ProjectMixin, AuthorizedResourceMixin
-from interface.models import Project
+from interface.utils import signed_url
 from library.models import Collection, Work, Document, Section
-from library.music_tags import MUSIC_TAGS, MusicTag, auto_tag
+from library.music_tags import MUSIC_TAGS, MusicTag
 from library import forms, models
 from library.pdf_utils import extract_pages, extract_and_concat
+from library.indexer import indexer, model_search
 
 import logging
+
 logger = logging.getLogger(__name__)
 
+
 class ProjectItemListView(ProjectMixin, ListView):
     template_name = "library/item_list.html"
     model = models.ProjectItem
 
-    def post(self, request, **kwargs):
+    def post(self, request: HttpRequest, **kwargs):
 
         project_works = self.project.works.all()
 
-        instruments = request.POST.getlist('instruments')
+        instruments = request.POST.getlist("instruments")
-        works = request.POST.getlist('works')
+        works = request.POST.getlist("works")
 
-        self.request.session['part'] = request.POST.get('part', '')
+        request.session["part"] = request.POST.get("part", "")
-        self.request.session['instrument'] = request.POST.get('instrument')
+        request.session["instrument"] = request.POST.get("instrument")
 
-        valid_pks = [ x.pk for x in project_works ]
+        valid_pks = [x.pk for x in project_works]
 
         sections = []
 
         for i, pk in enumerate(works):
-
             if int(pk) not in valid_pks:
                 raise Exception(f"Not a valid work pk: {pk}")
 
             tag = instruments[i]
-            if tag == '-':
+            if tag == "-":
                 continue
 
-            part = Section.objects.filter(tag=tag, doc__work=pk).select_related('doc').get()
+            part = (
-            sections.append((part.doc.upload.path, part.doc.work.name, part.start, part.end, 1))
+                Section.objects.filter(tag=tag, doc__work=pk)
+                .select_related("doc")
+                .get()
+            )
+            sections.append(
+                (part.doc.upload.path, part.doc.work.name, part.start, part.end, 1)
+            )
 
         result = extract_and_concat(sections)
 
-        download_name = f'{self.project.name}.pdf'
+        download_name = f"{self.project.name}.pdf"
 
         response = FileResponse(result, content_type="application/pdf")
-        response['Content-Disposition'] = f'inline; filename="{download_name}"'
+        response["Content-Disposition"] = f'inline; filename="{download_name}"'
         return response
 
-
     def get_queryset(self):
-        return super(ProjectItemListView, self).get_queryset().select_related('project', 'work')
+        return (
+            super(ProjectItemListView, self)
+            .get_queryset()
+            .select_related("project", "work")
+        )
 
     def get_context_data(self, **kwargs):
         data = super(ProjectItemListView, self).get_context_data(**kwargs)
-        data['instruments'] = MUSIC_TAGS
+        data["instruments"] = MUSIC_TAGS
-        data['instrument'] = self.request.session.get('instrument', 'Score')
+        data["instrument"] = self.request.session.get("instrument", "Score")
-        data['part'] = self.request.session.get('part', '0')
+        data["part"] = self.request.session.get("part", "0")
-        data['running_time'] = self.get_queryset().aggregate(Sum('work__running_time'))['work__running_time__sum']
+        data["running_time"] = self.get_queryset().aggregate(Sum("work__running_time"))[
+            "work__running_time__sum"
+        ]
         return data
 
 
@@ -93,31 +107,37 @@ class ProjectItemManageView(ProjectMixin, ListView):
             if order == -1:
                 q.filter(pk=pk).delete()
             else:
-                i = q.filter(pk=pk).update(order=order)
+                q.filter(pk=pk).update(order=order)
 
         return HttpResponse(status=204)
 
     def get_queryset(self):
-        return super(ProjectItemManageView, self).get_queryset().select_related('project', 'work')
+        return (
+            super(ProjectItemManageView, self)
+            .get_queryset()
+            .select_related("project", "work")
+        )
+
 
 class ProjectItemAddView(ProjectMixin, UpdateView):
     form_class = forms.PlaylistAddForm
     template_name = "interface/default_form.html"
 
     def get_success_url(self):
-        return resolve_url('item_list_manage', project=self.kwargs['project'])
+        return resolve_url("item_list_manage", project=self.kwargs["project"])
 
-    def get_object(self):
+    def get_object(self, queryset=None):
         return self.get_project()
 
+
 """ COLLECTION VIEWS """
 
-class CollectionMixin(AuthorizedResourceMixin):
 
-    collection = None
+class CollectionMixin(AuthorizedResourceMixin):
+    collection: Collection | None = None
 
     def is_authorized(self):
-        collection_id = self.kwargs['collection']
+        collection_id = self.kwargs["collection"]
         self.collection = get_object_or_404(models.Collection, pk=collection_id)
 
         if super().is_authorized():
@@ -127,7 +147,7 @@ class CollectionMixin(AuthorizedResourceMixin):
             self.request.is_admin = True
             return True
 
-        if self.is_authorized_key('collection', collection_id, self.collection.nonce):
+        if self.is_authorized_key("collection", collection_id, self.collection.nonce):
             return True
 
         return False
@@ -135,7 +155,7 @@ class CollectionMixin(AuthorizedResourceMixin):
     def get_context_data(self, **kwargs):
         data = super().get_context_data(**kwargs)
         if self.collection:
-            data['collection'] = self.collection
+            data["collection"] = self.collection
         return data
 
     def get_queryset(self):
@@ -146,7 +166,7 @@ class CollectionListView(ListView):
     paginate_by = 20
 
     def get_queryset(self):
-        collections = models.Collection.objects.order_by('name')
+        collections = models.Collection.objects.order_by("name")
 
         if self.request.user.is_anonymous:
             return models.Collection.objects.none()
@@ -154,66 +174,110 @@ class CollectionListView(ListView):
         if self.request.user.is_staff:
             return collections
 
-        return collections.filter(Q(administrators=self.request.user) | Q(allowed_ensembles__ensemble__admins=self.request.user))
+        return collections.filter(
+            Q(administrators=self.request.user)
+            | Q(allowed_ensembles__ensemble__admins=self.request.user)
+        )
 
-class WorkListView(CollectionMixin, ListView):
+
+class WorkListView(CollectionMixin, TemplateView):
     paginate_by = 20
+    template_name = "library/work_list.html"
 
     def get_context_data(self, *args, **kwargs):
         data = super(WorkListView, self).get_context_data(*args, **kwargs)
-        #data['title'] = f'Music available to {self.ensemble.name}' 
+        data["title"] = "My Library"
-        data['title'] = "My Library"
+        data["collection"] = self.collection
+
+        data["query"] = self.request.GET.get("q", "")
+        page = int(self.request.GET.get("page", 1))  # type: ignore
+        if data["query"]:
+            qs = self.get_results(data["query"], page)
+            data["meta"] = qs.meta
+            # data["page_range"] = data["page_obj"]["paginator"]
+        else:
+            qs = self.get_queryset()
+
+            start = self.request.GET.get("start")
+            if start:
+                start = start.upper()
+                qs = qs.filter(name__gte=start, name__lt=start + "~")
+                data["start"] = start
+
+            data["letters"] = string.ascii_uppercase
+
+        paginator = Paginator(qs, self.paginate_by)
+        data["page_obj"] = paginator.get_page(page)
+        data["object_list"] = data["page_obj"]
+        data["page_range"] = paginator.get_elided_page_range(page)
+
         return data
 
+    def get_works(self):
+        raise NotImplementedError
+
+    def get_collections(self):
+        raise NotImplementedError
+
     def get_queryset(self):
         works = self.get_works()
+        return works.order_by("name", "composer", "edition", "pk").distinct()
 
-        q = self.request.GET.get('filter')
+    def get_results(self, query, page):
-        if q:
+        try:
-            if ":" in q:
+            collections = self.get_collections()
-                name, _, value = q.partition(":")
+        except IndexError:
-                works = works.filter(meta_info__name=name, meta_info__value__contains=value)
+            return []
-            else:
+
-                works = works.filter(Q(name__contains=q) | Q(composer__contains=q) | Q(meta_info__value__contains=q))
+        return model_search(query, collections, page, self.paginate_by)
 
-        return works.order_by('name', 'composer', 'edition', 'pk').distinct()
 
 class LibraryWorkListView(WorkListView):
-    
     def is_authorized(self):
         return True
 
-    def get_works(self):
+    def get_collections(self):
         collections = models.Collection.objects.all()
+        if self.request.user.is_superuser:
+            return []
 
-        if not self.request.user.is_superuser:
+        collections = models.Collection.objects.filter(administrators=self.request.user)
-            collections = collections.filter(administrators=self.request.user)
+        return collections.values_list("pk", flat=True)
+
+    def get_works(self):
+        if self.request.user.is_superuser:
+            return Work.objects.all()
+
+        return Work.objects.filter(collection_id__in=self.get_collections())
 
-        return  Work.objects.filter(collection__in=collections).select_related('collection')
 
 class CollectionWorkListView(WorkListView):
-
     def request_denied(self):
-        if 'auth' in self.request.GET:
+        if "auth" in self.request.GET:
-            if self.request.GET['auth'] != self.collection.auth():
+            if self.request.GET["auth"] != self.collection.auth():
                 raise SuspiciousOperation("Bad collection link")
-            self.add_authorized_key('collection', self.collection.pk, self.collection.nonce)
+            self.add_authorized_key(
+                "collection", self.collection.pk, self.collection.nonce
+            )
             return HttpResponseRedirect(self.request.path)
         return super().request_denied()
 
-    def get_works(self):
+    def get_collections(self):
-        works = Work.objects.filter(collection=self.kwargs['collection'])
+        return [int(self.kwargs["collection"])]
 
-        #if self.request.is_admin:
+    def get_works(self):
+        works = Work.objects.filter(collection=self.kwargs["collection"])
+        # if self.request.is_admin:
         #    loan_count = Count('project_items', Q(project_items__checkout__lte=now(), project_items__returned=None))
         #    works = works.annotate(loan_count=loan_count)
         return works
 
     def get_context_data(self, *args, **kwargs):
         data = super(CollectionWorkListView, self).get_context_data(*args, **kwargs)
-        data['title'] = self.collection.name 
+        data["title"] = self.collection.name
         return data
 
+
 class WorkAddView(CollectionMixin, FormView):
     template_name = "interface/default_form.html"
     form_class = forms.WorkCreateForm
@@ -222,38 +286,64 @@ class WorkAddView(CollectionMixin, FormView):
 
     def get_form(self, form_class=None):
         form = super().get_form(form_class)
-        qs = models.Orchestration.objects.filter(Q(collection=None) | Q(collection=self.collection))
+        qs = models.Orchestration.objects.filter(
-        form.fields['orchestration'].queryset = qs.order_by('-collection_id', 'pk')
+            Q(collection=None) | Q(collection=self.collection)
+        )
+        form.fields["orchestration"].queryset = qs.order_by("-collection_id", "pk")
         return form
 
     def form_valid(self, form):
         work = form.save(commit=False)
-        #work.ensemble_id = self.request.ensemble_id
+        # work.ensemble_id = self.request.ensemble_id
         work.collection_id = self.collection.pk
         work.save()
 
         # handle the files
-        uploads = self.request.FILES.getlist('uploads')
+        uploads = self.request.FILES.getlist("uploads")
         docs = []
         for f in uploads:
             docs.append(work.docs.create(upload=f).pk)
 
+        ix = indexer.get_index()
+        indexer.index_works(ix, [work])
+
         if len(docs) == 1:
-            return redirect('document_annotate', docs[0])
+            return redirect("document_annotate", docs[0])
         else:
-            return redirect('work_detail', collection=self.collection.pk, pk=work.pk)
+            return redirect("work_detail", collection=self.collection.pk, pk=work.pk)
+
 
 class WorkDetailView(CollectionMixin, DetailView):
     model = models.Work
 
+    def get_context_data(self, *args, **kwargs):
+        context = super().get_context_data(*args, **kwargs)
+
+        methods = set("upload")
+        match self.collection.storage.storage:
+            case "library.storage.GDriveLinkStorage":
+                methods.discard("upload")
+                methods.add("gdrive")
+        context["methods"] = methods
+
+        return context
+
+
 class WorkUpdateView(CollectionMixin, UpdateView):
     model = models.Work
     form_class = forms.WorkCreateForm
-    template_name = 'interface/default_form.html'
+    template_name = "interface/default_form.html"
+
+    def form_valid(self, form):
+        response = super().form_valid(form)
+
+        ix = indexer.get_index()
+        indexer.index_works(ix, [self.object])
+
+        return response
 
     def get_success_url(self):
-        return resolve_url('work_detail', self.collection.pk, self.kwargs['pk'])
+        return resolve_url("work_detail", self.collection.pk, self.kwargs["pk"])
-
 
 
 class WorkAddToProject(CollectionMixin, FormView):
@@ -263,11 +353,11 @@ class WorkAddToProject(CollectionMixin, FormView):
     title = "Select project to add work to"
 
     def get_object(self):
-        return Work.objects.get(pk=self.kwargs['pk'])
+        return Work.objects.get(pk=self.kwargs["pk"])
 
     def get_form(self):
         f = super(WorkAddToProject, self).get_form()
-        qs = f.fields['project'].queryset.select_related('ensemble') 
+        qs = f.fields["project"].queryset.select_related("ensemble")
 
         # Limit to projects for ensembles where we are an admin and they haven't occured yet
         qs = qs.for_user(self.request.user).current()
@@ -276,59 +366,69 @@ class WorkAddToProject(CollectionMixin, FormView):
         work = self.get_object()
         qs = qs.exclude(pk__in=work.projects.all())
 
-        f.fields['project'].queryset = qs.order_by('ensemble__name', 'name')
+        f.fields["project"].queryset = qs.order_by("ensemble__name", "name")
         return f
 
     def form_valid(self, form):
         work = self.get_object()
-        project = form.cleaned_data['project']
+        project = form.cleaned_data["project"]
-        work.project_items.create(project=project, approved_by=self.request.user, checkout=now())
+        work.project_items.create(
-        return redirect('item_list', project=project.pk)
+            project=project, approved_by=self.request.user, checkout=now()
+        )
+        return redirect("item_list", project=project.pk)
+
 
 class WorkPartsView(CollectionMixin, DetailView):
     model = models.Work
     template_name = "library/work_parts_fragment.html"
 
+
 class WorkPartSetView(CollectionMixin, DetailView):
     template_name = "library/work_partset.html"
 
     def post(self, request, *args, **kwargs):
-
         work = self.get_object()
 
-        parts = request.POST.getlist('parts')
+        parts = request.POST.getlist("parts")
-        copies = request.POST.getlist('copies')
+        copies = request.POST.getlist("copies")
 
         sections = []
 
         for i, tag in enumerate(parts):
             c = int(copies[i])
             if c > 0:
-                for part in models.Section.objects.select_related('doc').filter(tag=tag, doc__work=work):
+                for part in models.Section.objects.select_related("doc").filter(
-                    sections.append((part.doc.upload.path, part.name, part.start, part.end, c))
+                    tag=tag, doc__work=work
+                ):
+                    sections.append(
+                        (part.doc.upload.path, part.name, part.start, part.end, c)
+                    )
 
         result = extract_and_concat(sections)
 
-        download_name = f'{work.name}.pdf'
+        download_name = f"{work.name}.pdf"
 
         response = FileResponse(result, content_type="application/pdf")
-        response['Content-Disposition'] = f'inline; filename="{download_name}"'
+        response["Content-Disposition"] = f'inline; filename="{download_name}"'
         return response
 
     def get_queryset(self):
         works = Work.objects.all()
 
         if not self.request.is_admin:
-            works = works.filter(collection__allowed_ensembles__ensemble=self.request.ensemble_id)
+            works = works.filter(
+                collection__allowed_ensembles__ensemble=self.request.ensemble_id
+            )
         return works
 
+
 class WorkDownloadView(CollectionMixin, SingleObjectMixin, View):
     model = models.Work
 
     def get(self, request, *args, **kwargs):
         self.object = self.get_object()
 
-        tags = request.GET.getlist('tag')
+        tags = request.GET.getlist("tag")
 
         if not tags:
             raise Http404("No tags given")
@@ -344,84 +444,126 @@ class WorkDownloadView(CollectionMixin, SingleObjectMixin, View):
             logger.debug("Redirecting to url")
             return redirect(sections[0].upload.url)
 
-        result = extract_and_concat([ (s.upload.path, s.upload.name, s.start, s.end, 1) for s in sections ])
+        result = extract_and_concat(
+            [(s.upload.path, s.upload.name, s.start, s.end, 1) for s in sections]
+        )
 
-        tag_names = " - ".join([ str(MusicTag.from_tag(tag)) for tag in tags ])
+        tag_names = " - ".join([str(MusicTag.from_tag(tag)) for tag in tags])
 
-        download_name = f'{self.object.name} - {tag_names}.pdf'
+        download_name = f"{self.object.name} - {tag_names}.pdf"
 
         response = FileResponse(result, content_type="application/pdf")
-        response['Content-Disposition'] = f'inline; filename="{download_name}"'
+        response["Content-Disposition"] = f'inline; filename="{download_name}"'
         return response
 
+
 class WorkAddDocumentView(CollectionMixin, CreateView):
     template_name = "interface/default_form.html"
     model = Document
-    fields = ['upload']
+    fields = ["upload"]
 
     def title(self):
-        work = Work.objects.get(pk=self.kwargs['pk'])
+        work = Work.objects.get(pk=self.kwargs["pk"])
         return f"Add a document to {work.name}"
 
     def form_invalid(self, form):
-        if self.request.headers['Accept'] == 'application/json':
+        if self.request.headers["Accept"] == "application/json":
             return HttpResponse(status=400)
         return super().form_invalid(form)
 
     def form_valid(self, form):
-        orig_name, ext = os.path.splitext(form.cleaned_data['upload'].name)
+        orig_name, ext = os.path.splitext(form.cleaned_data["upload"].name)
         logger.info("Uploaded: %s", orig_name)
 
         doc = form.save(commit=False)
-        doc.doctype = models.Document.DOCTYPE_MAP.get(ext.lower(), models.Document.DOCTYPE_MISC)
+        doc.doctype = models.Document.DOCTYPE_MAP.get(
-        doc.work_id = self.kwargs['pk']
+            ext.lower(), models.Document.DOCTYPE_MISC
+        )
+        doc.work_id = self.kwargs["pk"]
         doc.save()
 
         # auto tag the document
 
-        #name, ext = os.path.splitext(os.path.basename(doc.upload.name))
+        # name, ext = os.path.splitext(os.path.basename(doc.upload.name))
-        if doc.doctype == models.Document.DOCTYPE_PDF:
+        doc.auto_tag()
-            inst = auto_tag(orig_name)
-            if inst:
-                doc.sections.create(tag=inst.abbreviate())
 
-        if self.request.headers['Accept'] == 'application/json':
+        if self.request.headers["Accept"] == "application/json":
-            filename = os.path.basename(doc.upload.name)
+            # filename = os.path.basename(doc.upload.name)
-            return JsonResponse({
+            return JsonResponse(
-                "message": "created",
+                {
-                "id": doc.pk,
+                    "message": "created",
-                "entry": render_to_string('library/document_entry.html',
+                    "id": doc.pk,
-                    {'collection': self.collection, 'doc': doc, 'request': self.request}
+                    "entry": render_to_string(
-                )
+                        "library/document_entry.html",
-            }, status=201)
+                        {
+                            "collection": self.collection,
+                            "doc": doc,
+                            "request": self.request,
+                        },
+                    ),
+                },
+                status=201,
+            )
+
+        return redirect("document_annotate", self.collection.pk, doc.pk)
+
+
+class WorkAddDocumentBulkView(CollectionMixin, FormView):
+    template_name = "interface/default_form.html"
+    form_class = forms.DocumentBulkForm
+
+    @property
+    def cancel_url(self):
+        return resolve_url("work_detail", self.collection.pk, self.kwargs["pk"])
+
+    def form_valid(self, form):
+        folder_link = form.cleaned_data["folder_link"]
+
+        work = self.collection.works.get(pk=self.kwargs["pk"])
+        current = set(work.docs.values_list("upload", flat=True))
+
+        logger.info("Current documents: %r", current)
+
+        for link in self.collection.storage.instance().folder_import(folder_link):
+            uri = f"{self.collection.storage.name}:{link}"
+            if uri not in current:
+                doc = work.docs.create(upload=uri, doctype=Document.DOCTYPE_PDF)
+                doc.auto_tag()
+
+        return redirect("work_detail", self.collection.pk, self.kwargs["pk"])
 
-        return redirect('document_annotate', self.collection.pk, doc.pk)
 
 class DocumentMixin(CollectionMixin):
     model = models.Document
 
     def get_queryset(self):
-        qs = models.Document.objects.select_related('work')
+        qs = models.Document.objects.select_related("work")
         if self.request.is_admin:
             return qs
         return qs.filter(work__collection=self.collection)
 
+
 class DocumentDetailView(DocumentMixin, DetailView):
     pass
 
-class DocumentDownloadView(DocumentMixin, SingleObjectMixin, View):
 
+class DocumentDownloadView(DocumentMixin, SingleObjectMixin, View):
     def get(self, request, **args):
         self.request = request
         self.args = args
         self.object = self.get_object()
 
-        #response = FileResponse(self.object.upload, content_type="application/pdf")
+        if request.GET.get("method") == "direct":
-        #return response
+            return redirect(self.object.upload.url)
-        return redirect(self.object.upload.url)
+
+        response = FileResponse(
+            self.object.upload.open("rb"), content_type="application/pdf"
+        )
+        return response
+
 
 class DocumentAnnotateView(DocumentMixin, DetailView):
-    template_name = 'library/document_annotate.html'
+    template_name = "library/document_annotate.html"
 
     def post(self, request, **args):
         self.request = request
@@ -433,9 +575,9 @@ class DocumentAnnotateView(DocumentMixin, DetailView):
         with transaction.atomic():
             self.object.sections.all().delete()
             for tag, start, end in data:
-                #pages.sort()
+                # pages.sort()
-                #end = pages[-1] if len(pages) > 1 else None
+                # end = pages[-1] if len(pages) > 1 else None
-                o = self.object.sections.create(tag=tag, start=start, end=end)
+                self.object.sections.create(tag=tag, start=start, end=end)
 
         return HttpResponse(status=204)
 
@@ -443,23 +585,29 @@ class DocumentAnnotateView(DocumentMixin, DetailView):
         data = super(DocumentAnnotateView, self).get_context_data(**kwargs)
 
         pages = []
-        for part in data['document'].sections.all():
+        for part in data["document"].sections.all():
             pages.append((part.tag, part.start, part.end))
 
-        data['json_data'] = {'pageTags': pages, 'instruments': dict(MUSIC_TAGS)}
+        data["url"] = signed_url(
+            "document_download",
+            collection=data["collection"].pk,
+            pk=data["document"].pk,
+        )
+
+        data["json_data"] = {"pageTags": pages, "instruments": dict(MUSIC_TAGS)}
         return data
 
-class DocumentDeleteView(DocumentMixin, DeleteView):
 
-    #def get_template_names(self):
+class DocumentDeleteView(DocumentMixin, DeleteView):
+    # def get_template_names(self):
     #   return ["interface/default_form.html"]
 
     def get_success_url(self):
-        return resolve_url('work_detail', self.collection.pk, self.object.work_id)
+        return resolve_url("work_detail", self.collection.pk, self.object.work_id)
+
 
 class PartDownloadView(CollectionMixin, SingleObjectMixin, View):
-
+    pk_url_kwarg = "section"
-    pk_url_kwarg = 'section'
 
     def get(self, request, **args):
         self.request = request
@@ -469,14 +617,34 @@ class PartDownloadView(CollectionMixin, SingleObjectMixin, View):
         if self.object.start is None:
             return redirect(self.object.doc.upload.url)
 
-        result = extract_pages(self.object.doc.upload.path, self.object.doc.work.name, self.object.start, self.object.end)
+        result = extract_pages(
+            self.object.doc.upload.path,
+            self.object.doc.work.name,
+            self.object.start,
+            self.object.end,
+        )
 
-        #download_name = f'{self.object.doc.work.name}_{self.object.instrument}.pdf'
+        # download_name = f'{self.object.doc.work.name}_{self.object.instrument}.pdf'
 
         response = FileResponse(result, content_type="application/pdf")
-        response['Content-Disposition'] = f'inline; filename="{self.args["filename"]}"'
+        response["Content-Disposition"] = f'inline; filename="{self.args["filename"]}"'
         return response
 
     def get_object(self):
+        return (
+            Section.objects.filter(doc__work__collection=self.collection)
+            .select_related("doc", "doc__work")
+            .get(pk=self.kwargs["section"])
+        )
 
-        return Section.objects.filter(doc__work__collection=self.collection).select_related('doc', 'doc__work').get(pk=self.kwargs['section'])
+
+class StorageBrowserView(CollectionMixin, TemplateView):
+    template_name = "library/storage_browser.html"
+
+    def get_context_data(self, **kwargs):
+        data = super().get_context_data(**kwargs)
+        folder = self.kwargs.get("folder") or data["collection"].prefix
+        data["folders"], data["files"] = (
+            data["collection"].storage.instance().listdir(folder)
+        )
+        return data

app/library/views/api.py

@@ -1,6 +1,24 @@
+from interface.views import AuthorizedResourceMixin
+
+from rest_framework import serializers
+from rest_framework.exceptions import APIException
+from rest_framework import generics
+
+from library.models import Collection, Work, Document, Section, WorkMeta
+
+
+import requests
+import urllib
+import shutil
+import os.path
+
+from django.db import transaction
+from django.core.files.uploadedfile import TemporaryUploadedFile
+
 """
 Views relating to importing and exporting collection items
 """
+
 """
 from interface.views import EnsembleMixin
 from library.views import WorkMixin
@@ -32,39 +50,24 @@ class WorkExportView(EnsembleMixin, WorkMixin, View):
 
 """
 
-from interface.views import AuthorizedResourceMixin
-
-from rest_framework import routers, serializers, viewsets
-from rest_framework.exceptions import APIException
-
-from library.models import Collection, Work, Document, Section, WorkMeta
-
-
-import requests
-from io import BytesIO
-import urllib
-import shutil
-import os.path
-
-from django.db import transaction
-from django.core.files.uploadedfile import TemporaryUploadedFile
 
 class WorkMetaSerializer(serializers.ModelSerializer):
     class Meta:
         model = WorkMeta
-        exclude = ['id', 'work']
+        exclude = ["id", "work"]
 
     def to_representation(self, instance):
         return f"{instance.name}:{instance.value}"
 
     def to_internal_value(self, data):
-        name, _, value = data.partition(':')
+        name, _, value = data.partition(":")
-        return super().to_internal_value({'name': name, 'value': value})
+        return super().to_internal_value({"name": name, "value": value})
+
 
 class SectionSerializer(serializers.ModelSerializer):
     class Meta:
         model = Section
-        exclude = ['id', 'doc']
+        exclude = ["id", "doc"]
 
     def to_representation(self, instance):
         start = instance.start or 0
@@ -79,14 +82,14 @@ class SectionSerializer(serializers.ModelSerializer):
             start = None
         if end < 1:
             end = None
-        return super().to_internal_value({'tag': tag, 'start': start, 'end': end})
+        return super().to_internal_value({"tag": tag, "start": start, "end": end})
+
 
 class DocumentSerializer(serializers.ModelSerializer):
-
     upload = serializers.URLField()
     sections = SectionSerializer(many=True)
 
-    #def to_internal_value(self, data):
+    # def to_internal_value(self, data):
     #    r = requests.get(data['upload'], stream=True)
     #    with tempfile.NamedTemporaryFile('wb') as f:
     #        shutil.copyfileobj(r.raw, f)
@@ -96,7 +99,7 @@ class DocumentSerializer(serializers.ModelSerializer):
 
     def to_representation(self, instance):
         data = super().to_representation(instance)
-        data['upload'] = instance.upload.url
+        data["upload"] = instance.upload.url
         return data
 
     def create(self, validated_data):
@@ -115,35 +118,40 @@ class DocumentSerializer(serializers.ModelSerializer):
         model = Document
         exclude = ["id", "work", "version", "created"]
 
+
 # Serializers define the API representation.
 class WorkSerializer(serializers.ModelSerializer):
-
     docs = DocumentSerializer(many=True)
     meta_info = WorkMetaSerializer(many=True)
 
     class Meta:
         model = Work
-        exclude = ['id', 'collection', 'projects', 'parent']
+        exclude = ["id", "collection", "projects", "parent"]
 
     def create(self, validated):
         with transaction.atomic():
-            docs = validated.pop('docs', [])
+            docs = validated.pop("docs", [])
-            meta = validated.pop('meta_info', [])
+            meta = validated.pop("meta_info", [])
             work = Work.objects.create(**validated)
 
             for d in docs:
-                sections = d.pop('sections', [])
+                sections = d.pop("sections", [])
 
-                url = urllib.parse.urlparse(d['upload'])
+                url = urllib.parse.urlparse(d["upload"])
                 filename = os.path.basename(url.path)
 
-                r = requests.get(d['upload'], stream=True)
+                r = requests.get(d["upload"], stream=True)
                 if r.status_code != 200:
                     raise APIException("Failed to download file")
-                f = TemporaryUploadedFile(filename, r.headers['content-type'], r.headers.get('content-length'), r.encoding)
+                f = TemporaryUploadedFile(
+                    filename,
+                    r.headers["content-type"],
+                    r.headers.get("content-length"),
+                    r.encoding,
+                )
                 shutil.copyfileobj(r.raw, f.file)
                 r.close()
-                d['upload'] = f
+                d["upload"] = f
                 doc = Document.objects.create(work_id=work.pk, **d)
 
                 for s in sections:
@@ -154,22 +162,21 @@ class WorkSerializer(serializers.ModelSerializer):
 
             return work
 
+
 class CollectionSerializer(serializers.Serializer):
     works = WorkSerializer(many=True)
 
     def create(self, validated):
         s = WorkSerializer()
         print(validated)
-        collection = validated['collection_id']
+        collection = validated["collection_id"]
         with transaction.atomic():
-            for work in validated['works']:
+            for work in validated["works"]:
-                work['collection_id'] = collection
+                work["collection_id"] = collection
                 s.create(work)
         return Collection.objects.get(pk=collection)
 
 
-from rest_framework import generics
-
 class CollectionExportView(AuthorizedResourceMixin, generics.RetrieveAPIView):
     serializer_class = CollectionSerializer
 
@@ -178,23 +185,26 @@ class CollectionExportView(AuthorizedResourceMixin, generics.RetrieveAPIView):
             return Collection.objects.all()
         return Collection.objects.filter(administrators=self.request.user)
 
+
 class WorkExportView(AuthorizedResourceMixin, generics.RetrieveAPIView):
     serializer_class = WorkSerializer
 
     def get_queryset(self):
-        works = Work.objects.filter(collection=self.kwargs['collection'])
+        works = Work.objects.filter(collection=self.kwargs["collection"])
         if self.request.user.is_superuser:
             return works
         return works.filter(collection__administrators=self.request.user)
 
+
 class WorkImportView(AuthorizedResourceMixin, generics.CreateAPIView):
     serializer_class = WorkSerializer
 
     def perform_create(self, serializer):
-        serializer.save(collection_id=self.kwargs['collection'])
+        serializer.save(collection_id=self.kwargs["collection"])
+
 
 class CollectionImportView(AuthorizedResourceMixin, generics.CreateAPIView):
     serializer_class = CollectionSerializer
 
     def perform_create(self, serializer):
-        serializer.save(collection_id=self.kwargs['pk'])
+        serializer.save(collection_id=self.kwargs["pk"])

app/polyphonic/default_settings.py

@@ -16,7 +16,7 @@ from os import environ
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
 
-SECRET_KEY=environ.get('SECRET_KEY')
+SECRET_KEY = environ.get("SECRET_KEY")
 
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/
@@ -25,90 +25,89 @@ SECRET_KEY=environ.get('SECRET_KEY')
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = False
 
-ALLOWED_HOSTS = ['localhost']
+ALLOWED_HOSTS = ["localhost"]
+INTERNAL_IPS = ["127.0.0.1"]
 
 # Application definition
 
-POLYPHONIC_MODULES = [
+POLYPHONIC_MODULES = ["library"]
-    'library'
-]
 
 INSTALLED_APPS = [
-    'django.contrib.admin',
+    "django.contrib.admin",
-    'django.contrib.auth',
+    "django.contrib.auth",
-    'django.contrib.contenttypes',
+    "django.contrib.contenttypes",
-    'django.contrib.sessions',
+    "django.contrib.sessions",
-    'django.contrib.messages',
+    "django.contrib.messages",
-    'django.contrib.staticfiles',
+    "django.contrib.staticfiles",
-    'django_markdown2',
+    "django_markdown2",
-    'rest_framework',
+    "rest_framework",
-    'crispy_forms',
+    "crispy_forms",
-    'crispy_bulma',
+    "crispy_bulma",
-    'byostorage',
+    "byostorage",
-    'interface',
+    "interface",
 ]
 
 CRISPY_ALLOWED_TEMPLATE_PACKS = ("bulma",)
 CRISPY_TEMPLATE_PACK = "bulma"
 
 MIDDLEWARE = [
-    'django.middleware.security.SecurityMiddleware',
+    "django.middleware.security.SecurityMiddleware",
-    'django.contrib.sessions.middleware.SessionMiddleware',
+    "django.contrib.sessions.middleware.SessionMiddleware",
-    'django.middleware.common.CommonMiddleware',
+    "django.middleware.common.CommonMiddleware",
-    'django.middleware.csrf.CsrfViewMiddleware',
+    "django.middleware.csrf.CsrfViewMiddleware",
-    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    "django.contrib.auth.middleware.AuthenticationMiddleware",
-    'django.contrib.messages.middleware.MessageMiddleware',
+    "django.contrib.messages.middleware.MessageMiddleware",
-    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    "django.middleware.clickjacking.XFrameOptionsMiddleware",
 ]
 
-ROOT_URLCONF = 'polyphonic.urls'
+ROOT_URLCONF = "polyphonic.urls"
 
 TEMPLATES = [
     {
-        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        "BACKEND": "django.template.backends.django.DjangoTemplates",
-        'DIRS': [],
+        "DIRS": [],
-        'APP_DIRS': True,
+        "APP_DIRS": True,
-        'OPTIONS': {
+        "OPTIONS": {
-            'context_processors': [
+            "context_processors": [
-                'django.template.context_processors.debug',
+                "django.template.context_processors.debug",
-                'django.template.context_processors.request',
+                "django.template.context_processors.request",
-                'django.contrib.auth.context_processors.auth',
+                "django.contrib.auth.context_processors.auth",
-                'django.contrib.messages.context_processors.messages',
+                "django.contrib.messages.context_processors.messages",
             ],
         },
     },
 ]
 
-WSGI_APPLICATION = 'polyphonic.wsgi.application'
+WSGI_APPLICATION = "polyphonic.wsgi.application"
 
 
 # Database
 # https://docs.djangoproject.com/en/3.1/ref/settings/#databases
 
 DATABASES = {
-    'default': {
+    "default": {
-        'ENGINE': 'django.db.backends.sqlite3',
+        "ENGINE": "django.db.backends.sqlite3",
-        'NAME': BASE_DIR / 'db.sqlite3',
+        "NAME": BASE_DIR / "db.sqlite3",
     }
 }
 
-DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
+DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
 
 # Password validation
 # https://docs.djangoproject.com/en/3.1/ref/settings/#auth-password-validators
 
 AUTH_PASSWORD_VALIDATORS = [
     {
-        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
     },
     {
-        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
     },
     {
-        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+        "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
     },
     {
-        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+        "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
     },
 ]
 LOGIN_REDIRECT_URL = "/"
@@ -117,8 +116,8 @@ LOGIN_REDIRECT_URL = "/"
 # https://docs.djangoproject.com/en/3.1/topics/i18n/
 
 # Localisation (localization?)
-LANGUAGE_CODE = 'en-us'
+LANGUAGE_CODE = "en-us"
-TIME_ZONE = 'Australia/Melbourne'
+TIME_ZONE = "Australia/Melbourne"
 
 USE_I18N = True
 
@@ -130,9 +129,8 @@ USE_TZ = True
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/3.1/howto/static-files/
 
-STATIC_URL = '/static/'
+STATIC_URL = "/static/"
-STATIC_ROOT = 'static'
+STATIC_ROOT = "static"
 
 # Need to set this
-AWS_BUCKET = ''
+AWS_BUCKET = ""
-

app/polyphonic/settings.py

@@ -1,7 +1,6 @@
 try:
-    from .local_settings import *
+    from .local_settings import *  # noqa
 except ImportError:
-    from .default_settings import *
+    from .default_settings import *  # noqa
 
-
+INSTALLED_APPS += POLYPHONIC_MODULES  # type: ignore #noqa
-INSTALLED_APPS += POLYPHONIC_MODULES

app/polyphonic/urls.py

@@ -13,18 +13,20 @@ Including another URLconf
     1. Import the include() function: from django.urls import include, path
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
+
 from django.contrib import admin
-from django.urls import path, re_path, include
+from django.urls import path, include
 
 urlpatterns = [
-    path('admin/', admin.site.urls),
+    path("admin/", admin.site.urls),
-    path('', include('interface.urls')),
+    path("", include("interface.urls")),
-    #path('', include('submissions.urls')),
+    # path('', include('submissions.urls')),
-    path('', include('library.urls')),
+    path("", include("library.urls")),
 ]
 
 try:
     import debug_toolbar
-    urlpatterns.append(path('__debug__', include(debug_toolbar.urls)))
+
+    urlpatterns.append(path("__debug__", include(debug_toolbar.urls)))
 except ImportError:
     pass

app/requirements.txt

@@ -1,12 +1,11 @@
-asgiref==3.4.1
+asgiref==3.8.1
 boto3==1.18.34
 botocore==1.21.34
 certifi==2022.12.7
 charset-normalizer==2.1.1
 crispy-bulma==0.8.0
-Django==3.2.7
+Django==5.1
-#-e git+ssh://git@gitea.tfconsulting.com.au/tris/django-byostorage.git@227ce5fccb5864657f080c25528263b655c2b6b7#egg=django_byostorage
+django-byostorage @ git+https://gitea.tfconsulting.com.au/tris/django-byostorage.git@9903bb00888f20dfd2d39754e5ee22eeb5f36298
-git+https://gitea.tfconsulting.com.au/tris/django-byostorage.git
 django-crispy-forms==1.14.0
 django-markdown2==0.3.1
 django-rest-framework==0.1.0
@@ -14,11 +13,14 @@ django-storages==1.13.1
 django_debug_toolbar==3.8.1
 djangorestframework==3.14.0
 djantic==0.7.0
+greenlet==3.0.3
 idna==3.4
 jmespath==0.10.0
 markdown2==2.4.1
+msgpack==1.0.8
 pydantic==1.10.2
 pydantic-django==0.1.1
+pynvim==0.5.0
 python-dateutil==2.8.2
 pytz==2021.1
 requests==2.28.1

pyproject.toml

@@ -0,0 +1,37 @@
+[project]
+name = "polyphonic"
+version = "0.8.3"
+description = "Polyphonic Ensemble Manager"
+authors = [
+    {name = "Tris Forster",email = "tris@tfconsulting.com.au"}
+]
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "django (==6.0.4)",
+    "django-crispy-forms",
+    "django-markdown2",
+    "django-rest-framework",
+    "crispy-bulma (>=0.12.0,<0.13.0)",
+    "django-byostorage @ git+https://gitea.tfconsulting.com.au/tris/django-byostorage.git@9903bb00888f20dfd2d39754e5ee22eeb5f36298",
+    "requests (>=2.32.5,<3.0.0)",
+    "django-storages (>=1.14.6,<2.0.0)",
+    "boto3 (>=1.40.20,<2.0.0)",
+    "whoosh (>=2.7.4,<3.0.0)"
+]
+
+[tool.poetry]
+packages = [{include = "*", from="app"}]
+
+[tool.poetry.group.dev.dependencies]
+django-debug-toolbar = "5.2"
+ruff = "^0.15.12"
+coverage = "^7.14.0"
+
+[tool.poetry.scripts]
+manage = "manage:main"
+
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"