Merge pull request 'whoosh_search' (#11) from whoosh_search into master

Reviewed-on: #11

.gitignore

@@ -1,10 +1,13 @@
 __pycache__
 *.pyc
 *.sqlite3
+*.swp
 credentials.json
 credentials
 local_settings.py
 .coverage
+Session.vim
+poetry.lock
 /env
 /data
 /old
@@ -13,3 +16,4 @@ local_settings.py
 /cache
 /local_storage
 /media
+/index

README.md

@@ -6,20 +6,23 @@ No registration required for ensemble participants - just a one time code and pa
 
 ### Library App
 
-Store all your scores on your own cloud account (Amazon S3, Google Files etc).  Tag up the scores so you can generate
+Store all your scores on your own cloud account (Amazon S3, Google Files etc).
-custom part sets and assign them to projects so people can easily print just their parts.
+Tag up the scores so you can generate custom part sets and assign them to
+projects so people can easily print just their parts.
 
 ### Submissions App
 
-Accept video/audio submissions direct to your cloud storage.  Was developed and used during 2020 lockdown period for
+Accept video/audio submissions direct to your cloud storage.  Was developed and
-virtual choirs/orchestras but could have more uses.
+used during 2020 lockdown period for virtual choirs/orchestras but could have more uses.
 
 ### S3 Setup
 
 #### Bucket setup [virtual-orchestra]
+
 Default block public access
 
 Permissions -> CORS
+
 ```xml
 <?xml version="1.0" encoding="UTF-8"?>
 <CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
@@ -40,6 +43,7 @@ Permissions -> CORS
 
 User
 Create with programatic access (copy keys) and an inline policy for the bucket.
+
 ```json
 {
     "Version": "2012-10-17",
@@ -63,4 +67,5 @@ Create with programatic access (copy keys) and an inline policy for the bucket.
 }
 ```
 
-3. 
+3.
+

TODO.md

@@ -0,0 +1,19 @@
+## Polyphonic TODO
+
+## Core interface
+
+* Shift from crispy forms to native component templates
+* Make long running calls async (Django 5)
+* Deprecate Django 4 portions
+
+### Library App
+
+* Remove music tags and replace with strings vn1 -> 'Violin 1'
+* GDrive selector
+* Move upload to modal from 'Upload' button
+* Tagging app - migrate to AlpineJS
+* Allow other tags (movements, sections, pieces)
+
+### Submissions App
+
+* None currently pending

app/interface/templates/base.html

@@ -55,4 +55,4 @@
   {% block scripts %}
   {% endblock %} 
   </body>
-</html>
+</html>

app/interface/views.py

@@ -1,17 +1,15 @@
-"""
+""" """
-
-"""
 
+# pyright: basic
 
 from django.shortcuts import get_object_or_404, redirect, resolve_url
 from django.views.generic import RedirectView
 from django.views.generic.detail import DetailView, SingleObjectMixin
 from django.views.generic.list import ListView
-from django.views.generic.edit import CreateView, UpdateView, FormMixin
+from django.views.generic.edit import CreateView, UpdateView
 from django.core.exceptions import SuspiciousOperation
 from django.http import Http404, HttpResponseRedirect
-from django.db.models import Q
+from django.http.request import HttpRequest
-from django.utils import timezone
 
 from markdown2 import markdown
 
@@ -19,8 +17,10 @@ from . import models, forms
 from interface.utils import check_signed_url
 
 import logging
+
 logger = logging.getLogger(__name__)
 
+
 class AuthorizedResourceMixin(object):
     """
     Handles these parts of the permission system:
@@ -29,47 +29,51 @@ class AuthorizedResourceMixin(object):
         * Admin enforcing
     """
 
-    SESSION_KEY = 'authorized'
+    request: HttpRequest
+    kwargs: dict
+    _authorized: dict
+
+    SESSION_KEY = "authorized"
     admin_required = False
 
     def is_authorized(self):
         "By default check if superuser or a signed request"
-        if self.request.is_admin:
+        if self.request.is_admin:  # type: ignore
-            #logger.debug("is_authorized: superuser")
+            # logger.debug("is_authorized: superuser")
             return True
-        
+
-        if 'sig' in self.request.GET:
+        if "sig" in self.request.GET:
             check_signed_url(self.request.get_full_path())
             self.on_signed_request()
             return True
 
         return False
-    
+
     def on_signed_request(self):
         pass
 
     def is_authorized_key(self, resource, key, auth):
         current = self.get_authorized_keys(resource).get(str(key), None)
         if current is None:
-            #logger.debug("is_authorized_key: %s %s not in session", resource, key)
+            # logger.debug("is_authorized_key: %s %s not in session", resource, key)
             return False
         if auth == current:
             return True
 
-        #logger.info("Authorisation revoked")
+        # logger.info("Authorisation revoked")
         self.del_authorized_key(resource, key)
         return False
 
     def get_authorized_keys(self, resource):
-        'Returns a set of authorized keys for this resource'
+        "Returns a set of authorized keys for this resource"
         return self._authorized.get(resource, {})
-    
+
     def add_authorized_key(self, resource, key, auth):
-        'Adds a key to the authorized list for this resource'
+        "Adds a key to the authorized list for this resource"
         current = self.get_authorized_keys(resource)
         current[str(key)] = auth
         self._authorized[resource] = current
-        self.request.session[self.SESSION_KEY] = self._authorized
+        self.request.session[self.SESSION_KEY] = self._authorized  # type: ignore
 
     def del_authorized_key(self, resource, key):
         logger.info("Revoking authorization for %s %s", resource, key)
@@ -80,39 +84,41 @@ class AuthorizedResourceMixin(object):
             self._authorized[resource] = current
         else:
             self._authorized.pop(resource)
-        self.request.session[self.SESSION_KEY] = self._authorized
+        self.request.session[self.SESSION_KEY] = self._authorized  # type: ignore
         return True
 
     def request_denied(self):
-        raise Http404("Either the given resource doesn't exist or you dont have access to it.")
+        raise Http404(
+            "Either the given resource doesn't exist or you dont have access to it."
+        )
 
     def dispatch(self, request, *args, **kwargs):
-
+        self._authorized = request.session.get("authorized", {})
-        self._authorized = request.session.get('authorized', {})
         request.is_admin = request.user.is_superuser
 
         if not self.is_authorized():
             return self.request_denied()
-        
+
         if self.admin_required and not request.is_admin:
             return self.request_denied()
 
-        return super().dispatch(request, *args, **kwargs)
+        return super().dispatch(request, *args, **kwargs)  # type: ignore
+
 
 # TODO: RevokeResourceView - increment nonce
 
+
 class ForgetResourceView(AuthorizedResourceMixin, RedirectView):
-    
     def is_authorized(self):
         return True
 
     def get_redirect_url(self, resource, key):
         self.del_authorized_key(resource, key)
-        return "/" 
+        return "/"
+
 
 class EnsembleMixin(AuthorizedResourceMixin):
-
+    ensemble_slug_kwarg = "ensemble"
-    ensemble_slug_kwarg = 'ensemble'
     limited_project_access = False
 
     def is_authorized(self):
@@ -121,16 +127,16 @@ class EnsembleMixin(AuthorizedResourceMixin):
 
         if super().is_authorized():
             return True
-        
+
-        if self.ensemble.has_admin(self.request.user):
+        if self.ensemble.has_admin(self.request.user):  # type: ignore
-            self.request.is_admin = True
+            self.request.is_admin = True  # type: ignore
             return True
 
-        if self.is_authorized_key('ensemble', ensemble_slug, self.ensemble.nonce):
+        if self.is_authorized_key("ensemble", ensemble_slug, self.ensemble.nonce):
             return True
-        
+
-        authorized = set([ int(x) for x in self.get_authorized_keys('project').keys() ])
+        authorized = set([int(x) for x in self.get_authorized_keys("project").keys()])
-        projects = set(self.ensemble.projects.values_list('pk', flat=True))
+        projects = set(self.ensemble.projects.values_list("pk", flat=True))
         logger.debug("is_authorized: %r & %r", authorized, projects)
         if authorized & projects:
             self.limited_project_access = True
@@ -138,131 +144,131 @@ class EnsembleMixin(AuthorizedResourceMixin):
             return True
         return False
 
-    def get_object(self):
+    def get_object(self, queryset=None):
         return self.ensemble
 
 
 class ProjectMixin(AuthorizedResourceMixin):
-
+    project_kwarg = "project"
-    project_kwarg = 'project'
 
     def is_authorized(self):
         project_id = self.kwargs[self.project_kwarg]
-        self.project = get_object_or_404(models.Project.objects.select_related('ensemble'), pk=project_id)
+        self.project = get_object_or_404(
+            models.Project.objects.select_related("ensemble"), pk=project_id
+        )
 
         if super().is_authorized():
             return True
-        
+
-        
         # check if the current user is an admin on the ensemble
-        if self.project.ensemble.has_admin(self.request.user):
+        if self.project.ensemble.has_admin(self.request.user):  # type: ignore
             logger.debug("is_authorized: ensemble admin for project")
-            self.request.is_admin = True
+            self.request.is_admin = True  # type: ignore
-            return True
-        
-        if self.is_authorized_key('ensemble', self.project.ensemble.pk, self.project.ensemble.nonce):
-            logger.debug('is_authorized: has ensemble link for project')
             return True
 
-        if self.is_authorized_key('project', project_id, self.project.nonce):
+        if self.is_authorized_key(
-            logger.debug('is_authorized: has project link')
+            "ensemble", self.project.ensemble.pk, self.project.ensemble.nonce
+        ):
+            logger.debug("is_authorized: has ensemble link for project")
+            return True
+
+        if self.is_authorized_key("project", project_id, self.project.nonce):
+            logger.debug("is_authorized: has project link")
             return True
 
         return False
 
     # filter any generated querysets
     def get_queryset(self):
-        return super().get_queryset().filter(project=self.project)
+        return super().get_queryset().filter(project=self.project)  # type: ignore
 
     def get_context_data(self, **kwargs):
-        context = super().get_context_data(**kwargs)
+        context = super().get_context_data(**kwargs)  # type: ignore
-        if 'project' in self.kwargs:
+        if "project" in self.kwargs:
-            context['project'] = self.project
+            context["project"] = self.project
-            context['modules'] = self.project.active_modules
+            context["modules"] = self.project.active_modules
         return context
 
-class CrispyFormMixin(object):
 
+class CrispyFormMixin(object):
     cancel_url = None
 
     def get_cancel_url(self):
         return self.cancel_url
 
+
 """ ENSEMBLE VIEWS """
 
-class EnsembleListView(AuthorizedResourceMixin, ListView):
 
+class EnsembleListView(AuthorizedResourceMixin, ListView):
     def is_authorized(self):
         return True
-    
+
     def get_queryset(self):
-        return models.Ensemble.objects.for_user(self.request.user,
+        return models.Ensemble.objects.for_user(
-                                                self.get_authorized_keys('ensemble').keys(),
+            self.request.user,  # type: ignore
-                                                self.get_authorized_keys('project').keys())
+            self.get_authorized_keys("ensemble").keys(),
-        #ensembles = models.Ensemble.objects.all()
+            self.get_authorized_keys("project").keys(),
+        )
 
-        #if self.request.is_admin:
-        #    return ensembles
-
-        # limit to registered ensembles
-        #f = Q(slug__in=self.get_authorized_keys('ensemble').keys()) | Q(projects__in=self.get_authorized_keys('project').keys())
-        
-        # or ensembles where the user is admin
-        #if self.request.user.is_authenticated:
-        #    f |= Q(admins=self.request.user.pk)
-
-        #return ensembles.filter(f).distinct()
 
 class EnsembleDetailView(EnsembleMixin, DetailView):
-
     def request_denied(self):
-        if 'auth' in self.request.GET:
+        if "auth" in self.request.GET:
-            if self.request.GET['auth'] != self.ensemble.auth():
+            if self.request.GET["auth"] != self.ensemble.auth():
-                raise SuspiciousOperation("Bad ensemble link")    
+                raise SuspiciousOperation("Bad ensemble link")
-            self.add_authorized_key('ensemble', self.ensemble.slug, self.ensemble.nonce)
+            self.add_authorized_key("ensemble", self.ensemble.slug, self.ensemble.nonce)
-            return HttpResponseRedirect(resolve_url('ensemble_detail', self.ensemble.slug))
+            return HttpResponseRedirect(
-        
+                resolve_url("ensemble_detail", self.ensemble.slug)
+            )
+
         return super().request_denied()
-    
+
     def get_context_data(self, **kwargs):
         data = super().get_context_data(**kwargs)
-        inactive = 'inactive' in self.request.GET and self.request.is_admin
+        inactive = "inactive" in self.request.GET and self.request.is_admin  # type: ignore
 
         projects = self.ensemble.projects.all()
         if self.limited_project_access:
-            projects = projects.filter(pk__in=self.get_authorized_keys('project').keys())
+            projects = projects.filter(
+                pk__in=self.get_authorized_keys("project").keys()
+            )
 
         if inactive:
-            projects = projects.order_by('-pk')
+            projects = projects.order_by("-pk")
         else:
             projects = projects.active().current()
 
-        data['inactive'] = inactive
+        data["inactive"] = inactive
-        data['object_list'] = projects
+        data["object_list"] = projects
-        if self.request.is_admin:
+        if self.request.is_admin:  # type: ignore
-            data['ensemble_link'] = self.request.path + "?auth=" + self.ensemble.auth()
+            data["ensemble_link"] = self.request.path + "?auth=" + self.ensemble.auth()
         return data
 
-class EnsembleRevokeView(SingleObjectMixin, RedirectView):
 
+class EnsembleRevokeView(SingleObjectMixin, RedirectView):
     def get_redirect_url(self):
-        return 
+        return
+
 
 """ PROJECT VIEWS """
 
-class ProjectListView(ProjectMixin, ListView):
 
+class ProjectListView(ProjectMixin, ListView):
     def is_authorized(self):
         return True
-    
+
     def get_project_queryset(self):
-        return models.Project.objects.for_user(self.request.user,
+        return models.Project.objects.for_user(
-                                       self.get_authorized_keys('project'),
+            self.request.user,  # type: ignore
-                                       self.get_authorized_keys('ensemble'))
+            self.get_authorized_keys("project"),
+            self.get_authorized_keys("ensemble"),
+        )
 
     def get_queryset(self):
         return self.get_project_queryset().current().active()
 
+
 class ProjectCreateView(EnsembleMixin, CreateView):
     admin_required = True
     model = models.Project
@@ -270,26 +276,25 @@ class ProjectCreateView(EnsembleMixin, CreateView):
     title = "Add a new project"
     form_class = forms.ProjectForm
 
-
     def form_valid(self, form):
         self.object = form.save(commit=False)
         self.object.ensemble = self.ensemble
-        self.object.owner = self.request.user
+        self.object.owner = self.request.user  # type: ignore
         self.object.save()
 
-        for module in form.cleaned_data['modules']:
+        for module in form.cleaned_data["modules"]:
             self.object.modules.create(name=module)
 
-        return redirect('project_detail', project=self.object.pk)
+        return redirect("project_detail", project=self.object.pk)
+
 
 class ProjectDetailView(ProjectMixin, DetailView):
-
     def request_denied(self):
-        if 'auth' in self.request.GET:
+        if "auth" in self.request.GET:
-            if self.request.GET['auth'] != self.project.auth():
+            if self.request.GET["auth"] != self.project.auth():
                 raise SuspiciousOperation("Bad project link")
-            self.add_authorized_key('project', self.project.pk, self.project.nonce)
+            self.add_authorized_key("project", self.project.pk, self.project.nonce)
-            return HttpResponseRedirect(resolve_url('project_detail', self.project.pk))
+            return HttpResponseRedirect(resolve_url("project_detail", self.project.pk))
         return super().request_denied()
 
     def get_object(self, queryset=None):
@@ -297,15 +302,16 @@ class ProjectDetailView(ProjectMixin, DetailView):
 
     def get_context_data(self, **kwargs):
         data = super().get_context_data(**kwargs)
-        if self.request.is_admin:
+        if self.request.is_admin:  # type: ignore
-            #data['project_link'] = signed_url(f'{self.request.path}?nonce={self.project.nonce}')
+            # data['project_link'] = signed_url(f'{self.request.path}?nonce={self.project.nonce}')
-            data['project_link'] = self.request.path + "?auth=" + self.project.auth()
+            data["project_link"] = self.request.path + "?auth=" + self.project.auth()
         return data
 
+
 class ProjectUpdateView(ProjectMixin, UpdateView):
     admin_required = True
     template_name = "interface/default_form.html"
-    pk_url_kwarg = 'project'
+    pk_url_kwarg = "project"
     form_class = forms.ProjectForm
 
     def get_object(self):
@@ -313,26 +319,26 @@ class ProjectUpdateView(ProjectMixin, UpdateView):
 
     def get_initial(self):
         data = super().get_initial()
-        data['modules'] = self.object.active_modules
+        data["modules"] = self.object.active_modules
         print(data)
         return data
-    
+
     def form_valid(self, form):
         self.object = form.save()
         current = set(self.object.active_modules)
-        desired = set(form.cleaned_data['modules'])
+        desired = set(form.cleaned_data["modules"])
         self.object.modules.exclude(name__in=desired).delete()
-        for module in desired-current:
+        for module in desired - current:
             self.object.modules.create(name=module)
-        return redirect('project_detail', self.kwargs['project'])
+        return redirect("project_detail", self.kwargs["project"])
 
     @property
     def cancel_url(self):
-        return resolve_url('project_detail', self.kwargs['project'])
+        return resolve_url("project_detail", self.kwargs["project"])
 
 
 # Old Makefile from submission module
-#class ProjectMakefileView(EnsembleMixin, DetailView):
+# class ProjectMakefileView(EnsembleMixin, DetailView):
 #    template_name = 'interface/project_submissions.mk'
 #    content_type = 'text/plain'
 #
@@ -359,29 +365,32 @@ class ProjectUpdateView(ProjectMixin, UpdateView):
 
 """ WIKI VIEWS """
 
+
 class WikiView(ProjectMixin, DetailView):
-    template_name = 'interface/wiki.html'
+    template_name = "interface/wiki.html"
     model = models.WikiPage
 
     def get_context_data(self, **kwargs):
         data = super().get_context_data(**kwargs)
-        data['wiki_html'] = markdown(self.object.markdown)
+        data["wiki_html"] = markdown(self.object.markdown)
         return data
 
+
 class WikiCreateView(ProjectMixin, CreateView):
     admin_required = True
     model = models.WikiPage
-    template_name = 'interface/default_form.html'
+    template_name = "interface/default_form.html"
     form_class = forms.WikiForm
 
     def cancel_url(self):
-        return resolve_url('project_detail', self.kwargs['project'])
+        return resolve_url("project_detail", self.kwargs["project"])
 
     def form_valid(self, form):
         self.object = form.save(commit=False)
         self.object.project = self.project
         self.object.save()
-        return redirect('wiki', project=self.object.project_id, pk=self.object.pk)
+        return redirect("wiki", project=self.object.project_id, pk=self.object.pk)
+
 
 class WikiEditView(ProjectMixin, UpdateView):
     admin_required = True
@@ -389,46 +398,53 @@ class WikiEditView(ProjectMixin, UpdateView):
     form_class = forms.WikiForm
 
     def cancel_url(self):
-        return resolve_url('wiki', self.kwargs['project'], self.kwargs['pk'])
+        return resolve_url("wiki", self.kwargs["project"], self.kwargs["pk"])
+
 
 """ RESOURCE VIEWS """
 
+
 class ResourceCreateView(ProjectMixin, CreateView):
     admin_required = True
     model = models.Resource
     form_class = forms.ResourceForm
-    template_name = 'interface/default_form.html'
+    template_name = "interface/default_form.html"
     title = "Add a new resource"
 
     def form_valid(self, form):
         self.object = form.save(commit=False)
         self.object.project = self.project
         self.object.save()
-        return redirect('resource_upload', project=self.object.project_id, pk=self.object.pk)
+        return redirect(
+            "resource_upload", project=self.object.project_id, pk=self.object.pk
+        )
+
 
 class ResourceUploadView(ProjectMixin, UpdateView):
     admin_required = True
     model = models.Resource
-    fields = ['file']
+    fields = ["file"]
-    template_name = 'interface/default_form.html'
+    template_name = "interface/default_form.html"
 
     def get_success_url(self):
-        return resolve_url('resource_list', project=self.kwargs['project'])
+        return resolve_url("resource_list", project=self.kwargs["project"])
+
 
 class ResourceListView(ProjectMixin, ListView):
     model = models.Resource
 
     def get_queryset(self):
         qs = super().get_queryset()
-        if not self.request.is_admin:
+        if not self.request.is_admin:  # type: ignore
             qs = qs.filter(visible=True)
         return qs
 
+
 class ResourceEditView(ProjectMixin, UpdateView):
     admin_required = True
     model = models.Resource
-    fields = ['name', 'description', 'visible']
+    fields = ["name", "description", "visible"]
-    template_name = 'interface/default_form.html'
+    template_name = "interface/default_form.html"
 
     def get_success_url(self):
-        return resolve_url('resource_list', project=self.kwargs['project'])
+        return resolve_url("resource_list", project=self.kwargs["project"])

app/library/indexer/__init__.py

@@ -0,0 +1,103 @@
+from typing import Protocol, Any, Iterable, Generator
+from django.conf import settings
+from django.utils.module_loading import import_module
+from django.db.models import QuerySet
+
+from library.models import Work, Collection
+
+instance = getattr(settings, "INDEXER", "library.indexer.whoosh")
+
+
+class Indexer(Protocol):
+    def create_index(self) -> None: ...
+
+    def get_index(self) -> None: ...
+
+    def reset_index(self) -> None: ...
+
+    def search(
+        self,
+        query: str,
+        collections: list[int],
+        page: int = 1,
+        pagesize: int = 20,
+    ) -> tuple[list[dict], dict[str, Any]]: ...
+
+    def index_docs(self, docs: Iterable[dict]): ...
+
+
+class PartialResultSet(object):
+    """
+    Implements enough of QuerySet to fool the Paginator!
+    """
+
+    def __init__(self, works, meta):
+        self.works = works
+        self.meta = meta
+
+        self.start = (meta["page"] - 1) * meta["pagesize"]
+
+    def __getitem__(self, key):
+        if isinstance(key, slice):
+            if key.start != self.start:
+                raise KeyError(f"Expected {self.start}, got {key.start}")
+
+            return self.works
+
+        return self.works[key]
+
+    def count(self):
+        return self.meta["total"]
+
+
+# make the given module available as indexer
+indexer: Indexer = import_module(instance)  # type: ignore
+
+
+def work_to_doc(work: Work) -> dict[str, str]:
+    tags = ",".join(work.meta_info.filter(name="tag").values_list("value", flat=True))  # type: ignore
+    meta = ", ".join(work.meta_info.values_list("value", flat=True))  # type: ignore
+
+    description = f"""{work.name}
+{work.composer}
+{work.edition}
+{work.notes}
+{meta}
+"""
+
+    return dict(
+        work=str(work.pk),
+        collection=str(work.collection_id),  # type: ignore
+        name=work.name,
+        composer=work.composer,
+        edition=work.edition,
+        tag=tags.lower(),
+        text=description,
+    )
+
+
+def doc_set(works: QuerySet) -> Generator:
+    for work in works:
+        yield work_to_doc(work)
+
+
+def index_works(works: QuerySet):
+    indexer.index_docs(doc_set(works))
+
+
+def model_search(
+    query: str, collections: list[int], page: int = 1, pagesize: int = 20
+) -> PartialResultSet:
+    hits, meta = indexer.search(query.lower(), collections, page, pagesize)
+    meta["pagesize"] = pagesize
+    meta["page"] = page
+
+    works = [Work(**hit) for hit in hits]
+
+    collection_names = dict(Collection.objects.values_list("pk", "name"))
+    for work in works:
+        work.collection = Collection(  # type: ignore
+            pk=work.collection_id, name=collection_names[work.collection_id]
+        )
+
+    return PartialResultSet(works, meta)

app/library/indexer/whoosh.py

@@ -0,0 +1,93 @@
+from whoosh.index import create_in, open_dir, EmptyIndexError, Index
+from whoosh.analysis import StemmingAnalyzer, CharsetFilter
+from whoosh.support.charset import accent_map
+from whoosh.fields import Schema, ID, TEXT, KEYWORD, STORED, NUMERIC
+from whoosh.qparser import QueryParser
+from whoosh.query import Term, NullQuery
+
+from library.models import Work
+
+from typing import Any
+from django.conf import settings
+from django.db.models import QuerySet
+import os.path
+import shutil
+
+stemming_analyzer = StemmingAnalyzer() | CharsetFilter(accent_map)
+
+schema = Schema(
+    work=NUMERIC(stored=True, unique=True),
+    collection=NUMERIC(stored=True),
+    name=TEXT(stored=True),
+    composer=TEXT(stored=True),
+    edition=TEXT(stored=True),
+    tag=KEYWORD(commas=True),
+    text=TEXT(analyzer=stemming_analyzer),
+)
+
+
+index_path = os.path.join(os.path.dirname(settings.BASE_DIR), "index")
+
+
+def create_index() -> Index:
+    if not os.path.exists(index_path):
+        os.mkdir(index_path)
+
+    ix = create_in(index_path, schema)
+    return ix
+
+
+def get_index() -> Index:
+    try:
+        return open_dir(index_path)
+    except EmptyIndexError:
+        return create_index()
+
+
+def reset_index() -> Index:
+    shutil.rmtree(index_path)
+    return create_index()
+
+
+def index_docs(works: list[dict]):
+    ix = get_index()
+    with ix.writer() as writer:
+        for work in works:
+            writer.update_document(**work)
+
+
+def search(
+    query: str,
+    collections: list[int] = [],
+    page: int = 1,
+    pagesize: int = 20,
+) -> tuple[list[dict], dict[str, Any]]:
+    meta = {}
+
+    qp = QueryParser("text", schema=schema)
+    q = qp.parse(query.lower())
+    meta["query"] = str(q)
+
+    terms = NullQuery
+    for c in collections:
+        terms = terms | Term("collection", c)
+    q = q & terms
+
+    hits = []
+    ix = get_index()
+    with ix.searcher() as searcher:
+        results = searcher.search_page(q, page, pagesize)
+        for result in results:
+            hits.append(
+                dict(
+                    pk=result["work"],
+                    name=result["name"],
+                    composer=result["composer"],
+                    edition=result["edition"],
+                    collection_id=int(result["collection"]),
+                )
+            )
+
+    meta["total"] = len(results)
+
+    return hits, meta

app/library/management/commands/index.py

@@ -0,0 +1,36 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from library import models
+from library.indexer import model_search, index_works, indexer
+
+FORMATTER = "{w.name:50s} {w.edition:15s} {w.collection.name:15s}"
+
+
+class Command(BaseCommand):
+    help = "Imports works from a csv file"
+
+    def add_arguments(self, parser):
+        parser.add_argument("action", choices=("run", "search", "terms"))
+        parser.add_argument("query", nargs="*")
+        parser.add_argument("--collection", "-c", nargs="*", type=int)
+        # parser.add_argument("collection", type=int, help="Collection ID")
+        # parser.add_argument("source", type=argparse.FileType("r"), help="Source CSV")
+
+    def handle(self, action, query, *args, **options):
+        try:
+            method = getattr(self, f"handle_{action}")
+        except AttributeError:
+            raise RuntimeError(f"Unknown handler: {action}")
+        return method("".join(query), options["collection"] or [])
+
+    def handle_run(self, query, collections=[]):
+        index_works(models.Work.objects.all())
+
+    def handle_search(self, query, collections=[]):
+        for result in model_search(query, collections):
+            print(FORMATTER.format(w=result))
+
+    def handle_terms(self, query, collections=[]):
+        ix = indexer.get_index()
+        with ix.searcher() as searcher:
+            print(b", ".join(searcher.lexicon(query or "text")))

app/library/migrations/0015_collection_settings_alter_document_doctype.py

@@ -0,0 +1,23 @@
+# Generated by Django 5.0.6 on 2024-06-18 05:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('library', '0014_auto_20230223_1422'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='collection',
+            name='settings',
+            field=models.JSONField(default=dict, help_text='Storage specific settings'),
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='doctype',
+            field=models.PositiveSmallIntegerField(choices=[(1, 'PDF'), (2, 'Audio'), (3, 'Video'), (4, 'Misc')], default=1),
+        ),
+    ]

app/library/models.py

@@ -5,7 +5,6 @@ from django.db import models
 from django.utils.text import slugify
 from django.utils.timezone import now
 from django.utils.functional import cached_property
-from django.core.files.storage import get_storage_class
 from django.db.models import Q, Count, Min, Max
 
 import re
@@ -44,13 +43,13 @@ class Orchestration(models.Model):
     def as_list(self):
         tags = [ t.strip() for t in self.instruments.split(' ') ]
         return [ (t, MusicTag.from_tag(t)) for t in tags if t ]
-    
+
     def tag_order(self):
         tags = [ t.strip() for t in self.instruments.split(' ') if t ]
         order = {'score': 0}
         for i, t in enumerate(tags):
             order.setdefault(t.strip('-0123456789'), i*2+1)
-            
+
         return order
 
     def sorter(self):
@@ -79,7 +78,6 @@ class ProjectItem(models.Model):
     approved_by = models.ForeignKey('auth.User', on_delete=models.CASCADE)
     order = models.SmallIntegerField(default=0)
     section = models.CharField(max_length=100, blank=True)
-    
 
     class Meta:
         ordering = ['order', 'work']
@@ -93,16 +91,18 @@ class Collection(models.Model):
     """
     name = models.CharField(max_length=255,
             help_text="Name of the collection")
-    prefix = models.SlugField(max_length=30, default="default",
+    prefix = models.CharField(max_length=255, default="default",
             help_text="Folder to store works in")
-    administrators = models.ManyToManyField('auth.User', related_name="collections", 
+    administrators = models.ManyToManyField('auth.User', related_name="collections",
             help_text="Administrators for this collection")
-    location = models.CharField(max_length=100, 
+    location = models.CharField(max_length=100,
             help_text="Physical location (institution, town...)", blank=True)
-    storage = models.ForeignKey('byostorage.UserStorage', on_delete=models.CASCADE, null=True, blank=True, 
+    storage = models.ForeignKey('byostorage.UserStorage', on_delete=models.CASCADE, null=True, blank=True,
             help_text="User storage for documents")
-    notes = models.TextField(blank=True, 
+    notes = models.TextField(blank=True,
             help_text="Publicly visible notes about collection and loans policy (markdown format)")
+    settings = models.JSONField(default=dict, blank=True,
+            help_text="Storage specific settings")
     nonce = models.SmallIntegerField(default=1,
             help_text="Increment this to reset the authentication links")
 
@@ -116,8 +116,8 @@ class Collection(models.Model):
 
     @property
     def genres(self):
-        return self.meta('genre')    
+        return self.meta('genre')
-    
+
     def has_administrator(self, user):
         if not user.is_authenticated:
             return False
@@ -130,7 +130,7 @@ class Collection(models.Model):
 
     def auth(self):
         return sign_data(f'{self.pk}-{self.nonce}', 12)
-    
+
     def __str__(self):
         return self.name
 
@@ -174,7 +174,7 @@ class Work(models.Model):
         (LICENCE_PERUSAL, 'Perusal Licence'),
         (LICENCE_NONE, 'Internal use only'),
     )
-    
+
     name = models.CharField(max_length=255, help_text="Original name of the work")
     edition = models.CharField(max_length=255, blank=True,
             help_text="Edition details to distinguish multiple versions")
@@ -191,7 +191,7 @@ class Work(models.Model):
     code = models.CharField(max_length=100, blank=True, help_text="Collection specific code or number. Will be auto-generated if not supplied")
     licence = models.PositiveSmallIntegerField(choices=LICENCE_TYPES, default=6, help_text="Copyright status")
     max_projects = models.IntegerField(default=1, help_text="How many active projects can this work be attached to")
-    
+
     # Extra info
     running_time = models.DurationField(null=True, blank=True, help_text="Running time in mm:ss format")
     notes = models.TextField(blank=True)
@@ -219,7 +219,7 @@ class Work(models.Model):
         #return [ s[1] for s in sections ]
         sections = list(dict(sections).items()) # primitive unique()
         return sections
-    
+
     def pdfs(self):
         return self.docs.filter(doctype=Document.DOCTYPE_PDF)
 
@@ -304,7 +304,7 @@ class WorkMeta(models.Model):
         ('style', 'Style'),
         ('orchestration', 'Orchestration'),
     )
-    
+
     work = models.ForeignKey(Work, on_delete=models.CASCADE, related_name='meta_info')
     name = models.SlugField(max_length=20, choices=META_CHOICES)
     value = models.CharField(max_length=255)
@@ -348,7 +348,7 @@ class Document(models.Model):
     def delete(self, *args, **kwargs):
         self.upload.delete(save=False)
         return super().delete(*args, **kwargs)
-    
+
     def filename(self):
         return os.path.basename(self.upload.name)
 
@@ -359,7 +359,7 @@ class Section(models.Model):
     """
     Section is a tagged portion of a Document
     """
-    
+
     PAGE_AUTO = 0
     PAGE_LEFT = 1
     PAGE_RIGHT = 2
@@ -396,7 +396,7 @@ class Section(models.Model):
     def filename(self):
         return slugify(f'{self.doc.work.name} - {self.name}').title() + '.pdf'
 
-    @property 
+    @property
     def pagerange(self):
         if self.start:
             if self.end:
@@ -405,4 +405,5 @@ class Section(models.Model):
         return "all"
 
     def __str__(self):
-        return self.name
+        return self.name
+

app/library/templates/library/collection_list.html

@@ -7,7 +7,7 @@
 <form method="GET" action="{% url 'work_list' %}">
     <div class="field has-addons">
         <div class="control is-expanded">
-            <input class="input" name="filter" type="text" placeholder="Find a work" value="{{ request.GET.filter }}"/>
+            <input class="input" name="q" type="text" placeholder="Find a work" value="{{ request.GET.filter }}"/>
         </div>
         <div class="control">
             <a class="button" href="?"><i class="fas fa-times"></i></a>
@@ -32,10 +32,10 @@
                 </p>
                 <p>
                     {% for tag in collection.tags %}
-                    <a href="{% url 'collection_work_list' collection.pk %}?filter=tag:{{ tag }}" class="tag is-success">{{ tag }}</a>
+                    <a href="{% url 'collection_work_list' collection.pk %}?q=tag:{{ tag }}" class="tag is-success">{{ tag }}</a>
                     {% endfor %}
                     {% for genre in collection.genres %}
-                    <a href="{% url 'collection_work_list' collection.pk %}?filter=genre:{{ genre }}" class="tag is-warning">{{ genre }}</a>
+                    <a href="{% url 'collection_work_list' collection.pk %}?q=genre:{{ genre }}" class="tag is-warning">{{ genre }}</a>
                     {% endfor %}
                 </p>
             </div>
@@ -47,4 +47,4 @@
 <div>
     <small>{{ ensemble.ensemble_code }}</small>
 </div>
-{% endblock %}
+{% endblock %}

app/library/templates/library/document_annotate.html

@@ -139,7 +139,7 @@
   crossorigin="anonymous"></script>
 {{ json_data|json_script:"data" }}
 <script type="text/javascript">
-  let url = "{{ document.upload.url|safe }}";
+  let url = "{% url 'document_download' collection.pk document.pk %}";
 
   // Loaded via <script> tag, create shortcut to access PDF.js exports.
   let pdfjsLib = window['pdfjs-dist/build/pdf'];
@@ -447,4 +447,4 @@
   window.addEventListener('beforeunload', checkSaved);
 
 </script>
-{% endblock %}
+{% endblock %}

app/library/templates/library/storage_browser.html

@@ -0,0 +1,12 @@
+{% load path_filters %}
+<div>
+  <h3>Listing for {{ collection }}</h3>
+  <ul>
+    {% for item in folders %}
+      <li><a href="{% url 'storage_browser_folder' collection=collection.pk folder=item %}">{{ item|basename }}</a></li>
+    {% endfor %}
+    {% for item in files %}
+      <li>{{ item|basename }}</li>
+    {% endfor %}
+  </ul>
+</div>

app/library/templates/library/work_detail.html

@@ -25,46 +25,46 @@
     {% endfor %}
 </h3>
 <p class="subtitle">{% firstof work.composer "Unattributed" %}{% if work.edition %} - {{ work.edition }}{% endif %}</p>
-<section class="block">
-    <p class="block">{{ work.notes }}</p>
-
-    <p class="block">
-    <table class="table">
-        <tr>
-            <th>Location:</th><td><a href="{% url 'collection_work_list' work.collection.pk %}">{{ work.collection }}</a> [{{ work.identifier }}]</td>
-            <th>Orchestration:</th><td>{{ work.orchestration }}</td>
-        </tr><tr>
-            <th>Running time:</th><td>{% firstof work.duration 'Unknown' %}</td>
-            <th>Licence:</th><td>{{ work.get_licence_display }}</td>
-        </tr><tr>
-            <td colspan="4">
-    {% for meta in work.meta %}
-        <a href="{% url 'collection_work_list' work.collection.pk %}?filter={{ meta.name}}:{{ meta.value }}" class="tag" >
-            {{ meta.get_name_display }}:
-            {{ meta.value }}
-        </a>
-    {% endfor %}
-            </td>
-        </tr>
-    </table>
-
-    {% if work.parent %}
-    <p>From <a href="{% url 'work_detail' work.parent.collection.pk work.parent.pk %}">{{ work.parent.name }} - {{ work.parent.composer }}</a>
-    </p>
-    {% endif %}
-
-    {% if work.related_works.count %}
-    <h3>Related</h3>
-    <ul>
-        {% for related in work.related_works.all %}
-        <li><a href="{% url 'work_detail' related.collection.pk related.pk %}">{{ related.name }} - {{ related.composer }}</a></li>
-        {% endfor %}
-    </ul>
-    {% endif %}
-</section>
 <section class="block">
 
     <div class="columns">
+        <div class="column is-half">
+        <p class="block">{{ work.notes }}</p>
+
+        <p class="block">
+        <table class="table">
+            <tr>
+                <th>Location:</th><td><a href="{% url 'collection_work_list' work.collection.pk %}">{{ work.collection }}</a> [{{ work.identifier }}]</td>
+                <th>Orchestration:</th><td>{{ work.orchestration }}</td>
+            </tr><tr>
+                <th>Running time:</th><td>{% firstof work.duration 'Unknown' %}</td>
+                <th>Licence:</th><td>{{ work.get_licence_display }}</td>
+            </tr><tr>
+                <td colspan="4">
+        {% for meta in work.meta %}
+            <a href="{% url 'collection_work_list' work.collection.pk %}?filter={{ meta.name}}:{{ meta.value }}" class="tag" >
+                {{ meta.get_name_display }}:
+                {{ meta.value }}
+            </a>
+        {% endfor %}
+                </td>
+            </tr>
+        </table>
+
+        {% if work.parent %}
+        <p>From <a href="{% url 'work_detail' work.parent.collection.pk work.parent.pk %}">{{ work.parent.name }} - {{ work.parent.composer }}</a>
+        </p>
+        {% endif %}
+
+        {% if work.related_works.count %}
+        <h3>Related</h3>
+        <ul>
+            {% for related in work.related_works.all %}
+            <li><a href="{% url 'work_detail' related.collection.pk related.pk %}">{{ related.name }} - {{ related.composer }}</a></li>
+            {% endfor %}
+        </ul>
+        {% endif %}
+        </div>
         <div class="column is-half">
             <div class="box">
                 <h4 class="subtitle is-size-4">
@@ -79,8 +79,6 @@
                     {% endfor %}
                 </div>
             </div>
-        </div>
-        <div class="column is-half">
             <div class="box">
                 <h4 class="subtitle is-size-4">
                     <span class="icon"><i class="fas fa-print"></i></span>
@@ -206,4 +204,4 @@
 </script>
 {% endblock %}
 
-{% endblock %}
+{% endblock %}

app/library/templates/library/work_list.html

@@ -15,7 +15,7 @@
 <form method="GET">
     <div class="field has-addons">
         <div class="control is-expanded">
-            <input class="input" name="filter" type="text" placeholder="Filter" value="{{ request.GET.filter }}"/>
+            <input class="input" name="q" type="text" placeholder="Filter" value="{{ request.GET.q }}"/>
         </div>
         <div class="control">
             <a class="button" href="?"><i class="fas fa-times"></i></a>
@@ -23,6 +23,15 @@
     </div>
 </form>
 
+{% if letters %}
+<div class="has-text-centered">
+<a href="?">_</a>
+{% for letter in letters %}
+<a href="?start={{ letter }}">{{ letter }}</a>&nbsp;
+{% endfor %}
+</div>
+{% endif %}
+
 <table class="table is-striped is-fullwidth">
     <thead>
         <tr>
@@ -35,7 +44,9 @@
     <tbody>
         {% for work in object_list %}
             <tr>
-                <td><a href="{% url 'work_detail' collection=work.collection.pk pk=work.pk %}">{{ work.name }}</a></td>
+                <td>
+                  <a href="{% url 'work_detail' collection=work.collection_id pk=work.pk %}">{{ work.name }}</a>
+                </td>
                 <td title="{{ work.composer }}">{{ work.composer|truncatewords:3 }}</td>
                 <td class="is-hidden-mobile" title="{{ work.edition }}">{{ work.edition|truncatewords:2 }}</td>
                 {% if not collection %}<td class="is-hidden-touch">{{ work.collection.name }}</td>{% endif %}
@@ -46,20 +57,41 @@
     </tbody>
 </table>
 
+<footer>
 <nav class="pagination is-centered" role="navigation" aria-label="pagination">
-    {% if page_obj.has_previous %}
+    <a class="pagination-previous" 
-    <a class="pagination-previous" href="{% url_update page=page_obj.prev_page_number %}">Previous</a>
+      {% if page_obj.has_previous %}
-    {% endif %}
+        href="{% url_update page=page_obj.previous_page_number %}"
-    {% if page_obj.has_next %}
+      {% else %}
-    <a class="pagination-next" href="{% url_update page=page_obj.next_page_number %}">Next page</a>
+        disabled
-    {% endif %}
+    {% endif %}>
+      Previous</a>
+    <a class="pagination-next"
+      {% if page_obj.has_next %}
+        href="{% url_update page=page_obj.next_page_number %}"
+      {% else %}
+        disabled
+      {% endif %}>
+      Next
+    </a>
     <ul class="pagination-list">
-      {% for page in page_obj.paginator.page_range %}
+      {% for page in page_range %}
         <li>
-            <a class="pagination-link {% if forloop.counter == page_obj.number %}is-current{% endif %}" href="{% url_update page=forloop.counter %}" aria-label="Goto page {{ forloop.counter }}">{{ forloop.counter }}</a>
+            {% if page == '...' %}
+              {{ page }}
+            {% else %}
+              <a class="pagination-link {% if page == page_obj.number %}is-current{% endif %}" href="{% url_update page=page %}" aria-label="Goto page {{ page }}">{{ page }}</a>
+            {% endif %}
         </li>
       {% endfor %}
     </ul>
 </nav>
 
-{% endblock %}
+{% if meta %}
+<div class="has-text-right has-text-grey-light is-size-7">
+Query="{{ meta.query }}"
+</div>
+{% endif %}
+</footer>
+
+{% endblock %}

app/library/templates/library/work_parts_fragment.html

@@ -1,10 +1,11 @@
 <h3 class="subtitle">{{ work.name }}</h3>
-<div class="block">
+<div class="block tags">
 {% for tag, name in work.digital_parts %}
-<a class="tag is-info" href="{% url 'work_download' work.collection_id work.pk %}?tag={{ tag }}">{{ name }}</a>
+<a class="tag is-info" href="{% url 'work_download' work.collection_id work.pk %}?tag={{ tag }}" target="polyphonic_parts">{{ name }}</a>
 {% endfor %}
 </div>
 
+<!--
 <h3 class="subtitle">Files</h3>
 <table class="table is-narrow is-fullwidth">
 <tbody>
@@ -19,6 +20,7 @@
     {% endfor %}
 </tbody>
 </table>
+-->
 
 <p>
 <a href="{% url 'work_detail' work.collection_id work.pk %}">More details...</a>

app/library/urls.py

@@ -35,10 +35,13 @@ urlpatterns = [
     path('collections/<int:collection>/docs/<int:pk>/annotate', views.DocumentAnnotateView.as_view(), name="document_annotate"),
 
     path('collections/<int:collection>/download/<int:section>/<str:filename>', views.PartDownloadView.as_view(), name="part_download"),
+    path('collections/<int:collection>/browse', views.StorageBrowserView.as_view(), name="storage_browser"),
+    path('collections/<int:collection>/browse/<path:folder>', views.StorageBrowserView.as_view(), name="storage_browser_folder"),
 
     #path('api/', include(router.urls))
     path('api/collections/<int:pk>', api.CollectionExportView.as_view(), name="collection_export"),
     path('api/collections/<int:collection>/works/<int:pk>', api.WorkExportView.as_view(), name="work_export"),
     path('api/collections/<int:collection>/import', api.WorkImportView.as_view(), name="work_import"),
     path('api/collections/<int:collection>/bulk_import', api.CollectionImportView.as_view(), name="collection_import"),
+
 ]

app/library/views/__init__.py

@@ -1,4 +1,5 @@
 from django.shortcuts import get_object_or_404, redirect, resolve_url
+from django.views.generic import TemplateView
 from django.views.generic.detail import DetailView, SingleObjectMixin, View
 from django.views.generic.list import ListView, MultipleObjectMixin
 from django.views.generic.edit import CreateView, FormView, UpdateView, DeleteView
@@ -10,70 +11,86 @@ from django.utils.timezone import now
 from django.urls import reverse
 from django.template.loader import render_to_string
 from django.core.exceptions import SuspiciousOperation
+from django.core.paginator import Paginator
 from django.http import Http404, HttpResponseRedirect
 
 import json
 import os.path
+import string
+import math
 import re
 
 from interface.views import EnsembleMixin, ProjectMixin, AuthorizedResourceMixin
 from interface.models import Project
+from interface.utils import signed_url
 from library.models import Collection, Work, Document, Section
 from library.music_tags import MUSIC_TAGS, MusicTag, auto_tag
 from library import forms, models
 from library.pdf_utils import extract_pages, extract_and_concat
+from library.indexer import indexer, model_search
 
 import logging
+
 logger = logging.getLogger(__name__)
 
+
 class ProjectItemListView(ProjectMixin, ListView):
     template_name = "library/item_list.html"
     model = models.ProjectItem
 
     def post(self, request, **kwargs):
-
         project_works = self.project.works.all()
 
-        instruments = request.POST.getlist('instruments')
+        instruments = request.POST.getlist("instruments")
-        works = request.POST.getlist('works')
+        works = request.POST.getlist("works")
 
-        self.request.session['part'] = request.POST.get('part', '')
+        self.request.session["part"] = request.POST.get("part", "")
-        self.request.session['instrument'] = request.POST.get('instrument')
+        self.request.session["instrument"] = request.POST.get("instrument")
 
-        valid_pks = [ x.pk for x in project_works ]
+        valid_pks = [x.pk for x in project_works]
 
         sections = []
 
         for i, pk in enumerate(works):
-
             if int(pk) not in valid_pks:
                 raise Exception(f"Not a valid work pk: {pk}")
 
             tag = instruments[i]
-            if tag == '-':
+            if tag == "-":
                 continue
 
-            part = Section.objects.filter(tag=tag, doc__work=pk).select_related('doc').get()
+            part = (
-            sections.append((part.doc.upload.path, part.doc.work.name, part.start, part.end, 1))
+                Section.objects.filter(tag=tag, doc__work=pk)
+                .select_related("doc")
+                .get()
+            )
+            sections.append(
+                (part.doc.upload.path, part.doc.work.name, part.start, part.end, 1)
+            )
 
         result = extract_and_concat(sections)
 
-        download_name = f'{self.project.name}.pdf'
+        download_name = f"{self.project.name}.pdf"
 
         response = FileResponse(result, content_type="application/pdf")
-        response['Content-Disposition'] = f'inline; filename="{download_name}"'
+        response["Content-Disposition"] = f'inline; filename="{download_name}"'
         return response
 
-
     def get_queryset(self):
-        return super(ProjectItemListView, self).get_queryset().select_related('project', 'work')
+        return (
+            super(ProjectItemListView, self)
+            .get_queryset()
+            .select_related("project", "work")
+        )
 
     def get_context_data(self, **kwargs):
         data = super(ProjectItemListView, self).get_context_data(**kwargs)
-        data['instruments'] = MUSIC_TAGS
+        data["instruments"] = MUSIC_TAGS
-        data['instrument'] = self.request.session.get('instrument', 'Score')
+        data["instrument"] = self.request.session.get("instrument", "Score")
-        data['part'] = self.request.session.get('part', '0')
+        data["part"] = self.request.session.get("part", "0")
-        data['running_time'] = self.get_queryset().aggregate(Sum('work__running_time'))['work__running_time__sum']
+        data["running_time"] = self.get_queryset().aggregate(Sum("work__running_time"))[
+            "work__running_time__sum"
+        ]
         return data
 
 
@@ -98,36 +115,42 @@ class ProjectItemManageView(ProjectMixin, ListView):
         return HttpResponse(status=204)
 
     def get_queryset(self):
-        return super(ProjectItemManageView, self).get_queryset().select_related('project', 'work')
+        return (
+            super(ProjectItemManageView, self)
+            .get_queryset()
+            .select_related("project", "work")
+        )
+
 
 class ProjectItemAddView(ProjectMixin, UpdateView):
     form_class = forms.PlaylistAddForm
     template_name = "interface/default_form.html"
 
     def get_success_url(self):
-        return resolve_url('item_list_manage', project=self.kwargs['project'])
+        return resolve_url("item_list_manage", project=self.kwargs["project"])
 
     def get_object(self):
         return self.get_project()
 
+
 """ COLLECTION VIEWS """
 
-class CollectionMixin(AuthorizedResourceMixin):
 
-    collection = None
+class CollectionMixin(AuthorizedResourceMixin):
+    collection: Collection | None = None
 
     def is_authorized(self):
-        collection_id = self.kwargs['collection']
+        collection_id = self.kwargs["collection"]
         self.collection = get_object_or_404(models.Collection, pk=collection_id)
 
         if super().is_authorized():
             return True
-        
+
         if self.collection.has_administrator(self.request.user):
             self.request.is_admin = True
             return True
 
-        if self.is_authorized_key('collection', collection_id, self.collection.nonce):
+        if self.is_authorized_key("collection", collection_id, self.collection.nonce):
             return True
 
         return False
@@ -135,85 +158,129 @@ class CollectionMixin(AuthorizedResourceMixin):
     def get_context_data(self, **kwargs):
         data = super().get_context_data(**kwargs)
         if self.collection:
-            data['collection'] = self.collection
+            data["collection"] = self.collection
         return data
 
     def get_queryset(self):
         return super().get_queryset().filter(collection=self.collection)
 
-    
+
 class CollectionListView(ListView):
     paginate_by = 20
 
     def get_queryset(self):
-        collections = models.Collection.objects.order_by('name')
+        collections = models.Collection.objects.order_by("name")
 
         if self.request.user.is_anonymous:
             return models.Collection.objects.none()
 
         if self.request.user.is_staff:
             return collections
-        
-        return collections.filter(Q(administrators=self.request.user) | Q(allowed_ensembles__ensemble__admins=self.request.user))
 
-class WorkListView(CollectionMixin, ListView):
+        return collections.filter(
+            Q(administrators=self.request.user)
+            | Q(allowed_ensembles__ensemble__admins=self.request.user)
+        )
+
+
+class WorkListView(CollectionMixin, TemplateView):
     paginate_by = 20
-        
+    template_name = "library/work_list.html"
+
     def get_context_data(self, *args, **kwargs):
         data = super(WorkListView, self).get_context_data(*args, **kwargs)
-        #data['title'] = f'Music available to {self.ensemble.name}' 
+        data["title"] = "My Library"
-        data['title'] = "My Library"
+        data["collection"] = self.collection
+
+        data["query"] = self.request.GET.get("q", "")
+        page = int(self.request.GET.get("page", 1))  # type: ignore
+        if data["query"]:
+            qs = self.get_results(data["query"], page)
+            data["meta"] = qs.meta
+            # data["page_range"] = data["page_obj"]["paginator"]
+        else:
+            qs = self.get_queryset()
+
+            start = self.request.GET.get("start")
+            if start:
+                start = start.upper()
+                qs = qs.filter(name__gte=start, name__lt=start + "~")
+                data["start"] = start
+
+            data["letters"] = string.ascii_uppercase
+
+        paginator = Paginator(qs, self.paginate_by)
+        data["page_obj"] = paginator.get_page(page)
+        data["object_list"] = data["page_obj"]
+        data["page_range"] = paginator.get_elided_page_range(page)
+
         return data
 
+    def get_works(self):
+        raise NotImplementedError
+
+    def get_collections(self):
+        raise NotImplementedError
+
     def get_queryset(self):
         works = self.get_works()
+        return works.order_by("name", "composer", "edition", "pk").distinct()
+
+    def get_results(self, query, page):
+        try:
+            collections = self.get_collections()
+        except IndexError:
+            return []
+
+        return model_search(query, collections, page, self.paginate_by)
 
-        q = self.request.GET.get('filter')
-        if q:
-            if ":" in q:
-                name, _, value = q.partition(":")
-                works = works.filter(meta_info__name=name, meta_info__value__contains=value)
-            else:
-                works = works.filter(Q(name__contains=q) | Q(composer__contains=q) | Q(meta_info__value__contains=q))
-        
-        return works.order_by('name', 'composer', 'edition', 'pk').distinct()
 
 class LibraryWorkListView(WorkListView):
-    
     def is_authorized(self):
         return True
 
-    def get_works(self):
+    def get_collections(self):
         collections = models.Collection.objects.all()
-        
+        if self.request.user.is_superuser:
-        if not self.request.user.is_superuser:
+            return []
-            collections = collections.filter(administrators=self.request.user)
+
+        collections = models.Collection.objects.filter(administrators=self.request.user)
+        return collections.values_list("pk", flat=True)
+
+    def get_works(self):
+        if self.request.user.is_superuser:
+            return Work.objects.all()
+
+        return Work.objects.filter(collection_id__in=self.get_collections())
 
-        return  Work.objects.filter(collection__in=collections).select_related('collection')
 
 class CollectionWorkListView(WorkListView):
-
     def request_denied(self):
-        if 'auth' in self.request.GET:
+        if "auth" in self.request.GET:
-            if self.request.GET['auth'] != self.collection.auth():
+            if self.request.GET["auth"] != self.collection.auth():
                 raise SuspiciousOperation("Bad collection link")
-            self.add_authorized_key('collection', self.collection.pk, self.collection.nonce)
+            self.add_authorized_key(
+                "collection", self.collection.pk, self.collection.nonce
+            )
             return HttpResponseRedirect(self.request.path)
         return super().request_denied()
 
-    def get_works(self):
+    def get_collections(self):
-        works = Work.objects.filter(collection=self.kwargs['collection'])
+        return [int(self.kwargs["collection"])]
 
-        #if self.request.is_admin:
+    def get_works(self):
+        works = Work.objects.filter(collection=self.kwargs["collection"])
+        # if self.request.is_admin:
         #    loan_count = Count('project_items', Q(project_items__checkout__lte=now(), project_items__returned=None))
         #    works = works.annotate(loan_count=loan_count)
         return works
 
     def get_context_data(self, *args, **kwargs):
         data = super(CollectionWorkListView, self).get_context_data(*args, **kwargs)
-        data['title'] = self.collection.name 
+        data["title"] = self.collection.name
         return data
 
+
 class WorkAddView(CollectionMixin, FormView):
     template_name = "interface/default_form.html"
     form_class = forms.WorkCreateForm
@@ -222,38 +289,52 @@ class WorkAddView(CollectionMixin, FormView):
 
     def get_form(self, form_class=None):
         form = super().get_form(form_class)
-        qs = models.Orchestration.objects.filter(Q(collection=None) | Q(collection=self.collection))
+        qs = models.Orchestration.objects.filter(
-        form.fields['orchestration'].queryset = qs.order_by('-collection_id', 'pk')
+            Q(collection=None) | Q(collection=self.collection)
+        )
+        form.fields["orchestration"].queryset = qs.order_by("-collection_id", "pk")
         return form
 
     def form_valid(self, form):
         work = form.save(commit=False)
-        #work.ensemble_id = self.request.ensemble_id
+        # work.ensemble_id = self.request.ensemble_id
         work.collection_id = self.collection.pk
         work.save()
-        
+
         # handle the files
-        uploads = self.request.FILES.getlist('uploads')
+        uploads = self.request.FILES.getlist("uploads")
         docs = []
         for f in uploads:
             docs.append(work.docs.create(upload=f).pk)
 
+        ix = indexer.get_index()
+        indexer.index_works(ix, [work])
+
         if len(docs) == 1:
-            return redirect('document_annotate', docs[0])
+            return redirect("document_annotate", docs[0])
         else:
-            return redirect('work_detail', collection=self.collection.pk, pk=work.pk)
+            return redirect("work_detail", collection=self.collection.pk, pk=work.pk)
+
 
 class WorkDetailView(CollectionMixin, DetailView):
-    model = models.Work 
+    model = models.Work
+
 
 class WorkUpdateView(CollectionMixin, UpdateView):
     model = models.Work
     form_class = forms.WorkCreateForm
-    template_name = 'interface/default_form.html'
+    template_name = "interface/default_form.html"
+
+    def form_valid(self, form):
+        response = super().form_valid(form)
+
+        ix = indexer.get_index()
+        indexer.index_works(ix, [self.object])
+
+        return response
 
     def get_success_url(self):
-        return resolve_url('work_detail', self.collection.pk, self.kwargs['pk'])
+        return resolve_url("work_detail", self.collection.pk, self.kwargs["pk"])
-
 
 
 class WorkAddToProject(CollectionMixin, FormView):
@@ -261,13 +342,13 @@ class WorkAddToProject(CollectionMixin, FormView):
     form_class = forms.ProjectSelectForm
     template_name = "interface/default_form.html"
     title = "Select project to add work to"
-    
+
     def get_object(self):
-        return Work.objects.get(pk=self.kwargs['pk'])
+        return Work.objects.get(pk=self.kwargs["pk"])
 
     def get_form(self):
         f = super(WorkAddToProject, self).get_form()
-        qs = f.fields['project'].queryset.select_related('ensemble') 
+        qs = f.fields["project"].queryset.select_related("ensemble")
 
         # Limit to projects for ensembles where we are an admin and they haven't occured yet
         qs = qs.for_user(self.request.user).current()
@@ -276,63 +357,73 @@ class WorkAddToProject(CollectionMixin, FormView):
         work = self.get_object()
         qs = qs.exclude(pk__in=work.projects.all())
 
-        f.fields['project'].queryset = qs.order_by('ensemble__name', 'name')
+        f.fields["project"].queryset = qs.order_by("ensemble__name", "name")
         return f
 
     def form_valid(self, form):
         work = self.get_object()
-        project = form.cleaned_data['project']
+        project = form.cleaned_data["project"]
-        work.project_items.create(project=project, approved_by=self.request.user, checkout=now())
+        work.project_items.create(
-        return redirect('item_list', project=project.pk)
+            project=project, approved_by=self.request.user, checkout=now()
+        )
+        return redirect("item_list", project=project.pk)
+
 
 class WorkPartsView(CollectionMixin, DetailView):
     model = models.Work
-    template_name = "library/work_parts_fragment.html" 
+    template_name = "library/work_parts_fragment.html"
+
 
 class WorkPartSetView(CollectionMixin, DetailView):
     template_name = "library/work_partset.html"
 
     def post(self, request, *args, **kwargs):
-
         work = self.get_object()
 
-        parts = request.POST.getlist('parts')
+        parts = request.POST.getlist("parts")
-        copies = request.POST.getlist('copies')
+        copies = request.POST.getlist("copies")
 
         sections = []
 
         for i, tag in enumerate(parts):
             c = int(copies[i])
             if c > 0:
-                for part in models.Section.objects.select_related('doc').filter(tag=tag, doc__work=work):
+                for part in models.Section.objects.select_related("doc").filter(
-                    sections.append((part.doc.upload.path, part.name, part.start, part.end, c))
+                    tag=tag, doc__work=work
+                ):
+                    sections.append(
+                        (part.doc.upload.path, part.name, part.start, part.end, c)
+                    )
 
         result = extract_and_concat(sections)
 
-        download_name = f'{work.name}.pdf'
+        download_name = f"{work.name}.pdf"
 
         response = FileResponse(result, content_type="application/pdf")
-        response['Content-Disposition'] = f'inline; filename="{download_name}"'
+        response["Content-Disposition"] = f'inline; filename="{download_name}"'
         return response
 
     def get_queryset(self):
         works = Work.objects.all()
 
         if not self.request.is_admin:
-            works = works.filter(collection__allowed_ensembles__ensemble=self.request.ensemble_id)
+            works = works.filter(
+                collection__allowed_ensembles__ensemble=self.request.ensemble_id
+            )
         return works
 
+
 class WorkDownloadView(CollectionMixin, SingleObjectMixin, View):
     model = models.Work
 
     def get(self, request, *args, **kwargs):
         self.object = self.get_object()
 
-        tags = request.GET.getlist('tag')
+        tags = request.GET.getlist("tag")
 
         if not tags:
             raise Http404("No tags given")
-        
+
         sections = list(self.object.tagged_sections(*tags))
         print(sections)
 
@@ -344,84 +435,104 @@ class WorkDownloadView(CollectionMixin, SingleObjectMixin, View):
             logger.debug("Redirecting to url")
             return redirect(sections[0].upload.url)
 
-        result = extract_and_concat([ (s.upload.path, s.upload.name, s.start, s.end, 1) for s in sections ])
+        result = extract_and_concat(
+            [(s.upload.path, s.upload.name, s.start, s.end, 1) for s in sections]
+        )
 
-        tag_names = " - ".join([ str(MusicTag.from_tag(tag)) for tag in tags ])
+        tag_names = " - ".join([str(MusicTag.from_tag(tag)) for tag in tags])
 
-        download_name = f'{self.object.name} - {tag_names}.pdf'
+        download_name = f"{self.object.name} - {tag_names}.pdf"
 
         response = FileResponse(result, content_type="application/pdf")
-        response['Content-Disposition'] = f'inline; filename="{download_name}"'
+        response["Content-Disposition"] = f'inline; filename="{download_name}"'
         return response
 
+
 class WorkAddDocumentView(CollectionMixin, CreateView):
     template_name = "interface/default_form.html"
     model = Document
-    fields = ['upload']
+    fields = ["upload"]
-    
+
     def title(self):
-        work = Work.objects.get(pk=self.kwargs['pk'])
+        work = Work.objects.get(pk=self.kwargs["pk"])
         return f"Add a document to {work.name}"
 
     def form_invalid(self, form):
-        if self.request.headers['Accept'] == 'application/json':
+        if self.request.headers["Accept"] == "application/json":
             return HttpResponse(status=400)
         return super().form_invalid(form)
 
     def form_valid(self, form):
-        orig_name, ext = os.path.splitext(form.cleaned_data['upload'].name)
+        orig_name, ext = os.path.splitext(form.cleaned_data["upload"].name)
         logger.info("Uploaded: %s", orig_name)
 
         doc = form.save(commit=False)
-        doc.doctype = models.Document.DOCTYPE_MAP.get(ext.lower(), models.Document.DOCTYPE_MISC)
+        doc.doctype = models.Document.DOCTYPE_MAP.get(
-        doc.work_id = self.kwargs['pk']
+            ext.lower(), models.Document.DOCTYPE_MISC
+        )
+        doc.work_id = self.kwargs["pk"]
         doc.save()
 
         # auto tag the document
 
-        #name, ext = os.path.splitext(os.path.basename(doc.upload.name))
+        # name, ext = os.path.splitext(os.path.basename(doc.upload.name))
         if doc.doctype == models.Document.DOCTYPE_PDF:
             inst = auto_tag(orig_name)
             if inst:
                 doc.sections.create(tag=inst.abbreviate())
 
-        if self.request.headers['Accept'] == 'application/json':
+        if self.request.headers["Accept"] == "application/json":
             filename = os.path.basename(doc.upload.name)
-            return JsonResponse({
+            return JsonResponse(
-                "message": "created",
+                {
-                "id": doc.pk,
+                    "message": "created",
-                "entry": render_to_string('library/document_entry.html',
+                    "id": doc.pk,
-                    {'collection': self.collection, 'doc': doc, 'request': self.request}
+                    "entry": render_to_string(
-                )
+                        "library/document_entry.html",
-            }, status=201)
+                        {
+                            "collection": self.collection,
+                            "doc": doc,
+                            "request": self.request,
+                        },
+                    ),
+                },
+                status=201,
+            )
+
+        return redirect("document_annotate", self.collection.pk, doc.pk)
 
-        return redirect('document_annotate', self.collection.pk, doc.pk)
 
 class DocumentMixin(CollectionMixin):
     model = models.Document
 
     def get_queryset(self):
-        qs = models.Document.objects.select_related('work')
+        qs = models.Document.objects.select_related("work")
         if self.request.is_admin:
             return qs
         return qs.filter(work__collection=self.collection)
 
+
 class DocumentDetailView(DocumentMixin, DetailView):
     pass
 
-class DocumentDownloadView(DocumentMixin, SingleObjectMixin, View):
 
+class DocumentDownloadView(DocumentMixin, SingleObjectMixin, View):
     def get(self, request, **args):
         self.request = request
         self.args = args
         self.object = self.get_object()
 
-        #response = FileResponse(self.object.upload, content_type="application/pdf")
+        if request.GET.get("method") == "direct":
-        #return response
+            return redirect(self.object.upload.url)
-        return redirect(self.object.upload.url)
+
+        response = FileResponse(
+            self.object.upload.open("rb"), content_type="application/pdf"
+        )
+        return response
+
 
 class DocumentAnnotateView(DocumentMixin, DetailView):
-    template_name = 'library/document_annotate.html'
+    template_name = "library/document_annotate.html"
 
     def post(self, request, **args):
         self.request = request
@@ -433,8 +544,8 @@ class DocumentAnnotateView(DocumentMixin, DetailView):
         with transaction.atomic():
             self.object.sections.all().delete()
             for tag, start, end in data:
-                #pages.sort()
+                # pages.sort()
-                #end = pages[-1] if len(pages) > 1 else None
+                # end = pages[-1] if len(pages) > 1 else None
                 o = self.object.sections.create(tag=tag, start=start, end=end)
 
         return HttpResponse(status=204)
@@ -443,23 +554,29 @@ class DocumentAnnotateView(DocumentMixin, DetailView):
         data = super(DocumentAnnotateView, self).get_context_data(**kwargs)
 
         pages = []
-        for part in data['document'].sections.all():
+        for part in data["document"].sections.all():
             pages.append((part.tag, part.start, part.end))
 
-        data['json_data'] = {'pageTags': pages, 'instruments': dict(MUSIC_TAGS)}
+        data["url"] = signed_url(
+            "document_download",
+            collection=data["collection"].pk,
+            pk=data["document"].pk,
+        )
+
+        data["json_data"] = {"pageTags": pages, "instruments": dict(MUSIC_TAGS)}
         return data
 
-class DocumentDeleteView(DocumentMixin, DeleteView):
 
-    #def get_template_names(self):
+class DocumentDeleteView(DocumentMixin, DeleteView):
+    # def get_template_names(self):
     #   return ["interface/default_form.html"]
 
     def get_success_url(self):
-        return resolve_url('work_detail', self.collection.pk, self.object.work_id)
+        return resolve_url("work_detail", self.collection.pk, self.object.work_id)
+
 
 class PartDownloadView(CollectionMixin, SingleObjectMixin, View):
-
+    pk_url_kwarg = "section"
-    pk_url_kwarg = 'section'
 
     def get(self, request, **args):
         self.request = request
@@ -469,14 +586,34 @@ class PartDownloadView(CollectionMixin, SingleObjectMixin, View):
         if self.object.start is None:
             return redirect(self.object.doc.upload.url)
 
-        result = extract_pages(self.object.doc.upload.path, self.object.doc.work.name, self.object.start, self.object.end)
+        result = extract_pages(
+            self.object.doc.upload.path,
+            self.object.doc.work.name,
+            self.object.start,
+            self.object.end,
+        )
 
-        #download_name = f'{self.object.doc.work.name}_{self.object.instrument}.pdf'
+        # download_name = f'{self.object.doc.work.name}_{self.object.instrument}.pdf'
 
         response = FileResponse(result, content_type="application/pdf")
-        response['Content-Disposition'] = f'inline; filename="{self.args["filename"]}"'
+        response["Content-Disposition"] = f'inline; filename="{self.args["filename"]}"'
         return response
 
     def get_object(self):
+        return (
+            Section.objects.filter(doc__work__collection=self.collection)
+            .select_related("doc", "doc__work")
+            .get(pk=self.kwargs["section"])
+        )
 
-        return Section.objects.filter(doc__work__collection=self.collection).select_related('doc', 'doc__work').get(pk=self.kwargs['section'])
+
+class StorageBrowserView(CollectionMixin, TemplateView):
+    template_name = "library/storage_browser.html"
+
+    def get_context_data(self, **kwargs):
+        data = super().get_context_data(**kwargs)
+        folder = self.kwargs.get("folder") or data["collection"].prefix
+        data["folders"], data["files"] = (
+            data["collection"].storage.instance().listdir(folder)
+        )
+        return data

app/polyphonic/default_settings.py

@@ -16,7 +16,7 @@ from os import environ
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
 
-SECRET_KEY=environ.get('SECRET_KEY')
+SECRET_KEY = environ.get("SECRET_KEY")
 
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/
@@ -25,90 +25,89 @@ SECRET_KEY=environ.get('SECRET_KEY')
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = False
 
-ALLOWED_HOSTS = ['localhost']
+ALLOWED_HOSTS = ["localhost"]
+INTERNAL_IPS = ["127.0.0.1"]
 
 # Application definition
 
-POLYPHONIC_MODULES = [
+POLYPHONIC_MODULES = ["library"]
-    'library'
-]
 
 INSTALLED_APPS = [
-    'django.contrib.admin',
+    "django.contrib.admin",
-    'django.contrib.auth',
+    "django.contrib.auth",
-    'django.contrib.contenttypes',
+    "django.contrib.contenttypes",
-    'django.contrib.sessions',
+    "django.contrib.sessions",
-    'django.contrib.messages',
+    "django.contrib.messages",
-    'django.contrib.staticfiles',
+    "django.contrib.staticfiles",
-    'django_markdown2',
+    "django_markdown2",
-    'rest_framework',
+    "rest_framework",
-    'crispy_forms',
+    "crispy_forms",
-    'crispy_bulma',
+    "crispy_bulma",
-    'byostorage',
+    "byostorage",
-    'interface',
+    "interface",
 ]
 
 CRISPY_ALLOWED_TEMPLATE_PACKS = ("bulma",)
 CRISPY_TEMPLATE_PACK = "bulma"
 
 MIDDLEWARE = [
-    'django.middleware.security.SecurityMiddleware',
+    "django.middleware.security.SecurityMiddleware",
-    'django.contrib.sessions.middleware.SessionMiddleware',
+    "django.contrib.sessions.middleware.SessionMiddleware",
-    'django.middleware.common.CommonMiddleware',
+    "django.middleware.common.CommonMiddleware",
-    'django.middleware.csrf.CsrfViewMiddleware',
+    "django.middleware.csrf.CsrfViewMiddleware",
-    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    "django.contrib.auth.middleware.AuthenticationMiddleware",
-    'django.contrib.messages.middleware.MessageMiddleware',
+    "django.contrib.messages.middleware.MessageMiddleware",
-    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    "django.middleware.clickjacking.XFrameOptionsMiddleware",
 ]
 
-ROOT_URLCONF = 'polyphonic.urls'
+ROOT_URLCONF = "polyphonic.urls"
 
 TEMPLATES = [
     {
-        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        "BACKEND": "django.template.backends.django.DjangoTemplates",
-        'DIRS': [],
+        "DIRS": [],
-        'APP_DIRS': True,
+        "APP_DIRS": True,
-        'OPTIONS': {
+        "OPTIONS": {
-            'context_processors': [
+            "context_processors": [
-                'django.template.context_processors.debug',
+                "django.template.context_processors.debug",
-                'django.template.context_processors.request',
+                "django.template.context_processors.request",
-                'django.contrib.auth.context_processors.auth',
+                "django.contrib.auth.context_processors.auth",
-                'django.contrib.messages.context_processors.messages',
+                "django.contrib.messages.context_processors.messages",
             ],
         },
     },
 ]
 
-WSGI_APPLICATION = 'polyphonic.wsgi.application'
+WSGI_APPLICATION = "polyphonic.wsgi.application"
 
 
 # Database
 # https://docs.djangoproject.com/en/3.1/ref/settings/#databases
 
 DATABASES = {
-    'default': {
+    "default": {
-        'ENGINE': 'django.db.backends.sqlite3',
+        "ENGINE": "django.db.backends.sqlite3",
-        'NAME': BASE_DIR / 'db.sqlite3',
+        "NAME": BASE_DIR / "db.sqlite3",
     }
 }
 
-DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
+DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
 
 # Password validation
 # https://docs.djangoproject.com/en/3.1/ref/settings/#auth-password-validators
 
 AUTH_PASSWORD_VALIDATORS = [
     {
-        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
     },
     {
-        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
     },
     {
-        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+        "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
     },
     {
-        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+        "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
     },
 ]
 LOGIN_REDIRECT_URL = "/"
@@ -117,8 +116,8 @@ LOGIN_REDIRECT_URL = "/"
 # https://docs.djangoproject.com/en/3.1/topics/i18n/
 
 # Localisation (localization?)
-LANGUAGE_CODE = 'en-us'
+LANGUAGE_CODE = "en-us"
-TIME_ZONE = 'Australia/Melbourne'
+TIME_ZONE = "Australia/Melbourne"
 
 USE_I18N = True
 
@@ -130,9 +129,8 @@ USE_TZ = True
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/3.1/howto/static-files/
 
-STATIC_URL = '/static/'
+STATIC_URL = "/static/"
-STATIC_ROOT = 'static'
+STATIC_ROOT = "static"
 
 # Need to set this
-AWS_BUCKET = ''
+AWS_BUCKET = ""
-

app/polyphonic/settings.py

@@ -3,5 +3,4 @@ try:
 except ImportError:
     from .default_settings import *
 
-
+INSTALLED_APPS += POLYPHONIC_MODULES
-INSTALLED_APPS += POLYPHONIC_MODULES

app/polyphonic/urls.py

@@ -13,18 +13,20 @@ Including another URLconf
     1. Import the include() function: from django.urls import include, path
     2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
+
 from django.contrib import admin
 from django.urls import path, re_path, include
 
 urlpatterns = [
-    path('admin/', admin.site.urls),
+    path("admin/", admin.site.urls),
-    path('', include('interface.urls')),
+    path("", include("interface.urls")),
-    #path('', include('submissions.urls')),
+    # path('', include('submissions.urls')),
-    path('', include('library.urls')),
+    path("", include("library.urls")),
 ]
 
 try:
     import debug_toolbar
-    urlpatterns.append(path('__debug__', include(debug_toolbar.urls)))
+
+    urlpatterns.append(path("__debug__", include(debug_toolbar.urls)))
 except ImportError:
     pass

app/requirements.txt

@@ -1,12 +1,11 @@
-asgiref==3.4.1
+asgiref==3.8.1
 boto3==1.18.34
 botocore==1.21.34
 certifi==2022.12.7
 charset-normalizer==2.1.1
 crispy-bulma==0.8.0
-Django==3.2.7
+Django==5.1
-#-e git+ssh://git@gitea.tfconsulting.com.au/tris/django-byostorage.git@227ce5fccb5864657f080c25528263b655c2b6b7#egg=django_byostorage
+django-byostorage @ git+https://gitea.tfconsulting.com.au/tris/django-byostorage.git@9903bb00888f20dfd2d39754e5ee22eeb5f36298
-git+https://gitea.tfconsulting.com.au/tris/django-byostorage.git
 django-crispy-forms==1.14.0
 django-markdown2==0.3.1
 django-rest-framework==0.1.0
@@ -14,11 +13,14 @@ django-storages==1.13.1
 django_debug_toolbar==3.8.1
 djangorestframework==3.14.0
 djantic==0.7.0
+greenlet==3.0.3
 idna==3.4
 jmespath==0.10.0
 markdown2==2.4.1
+msgpack==1.0.8
 pydantic==1.10.2
 pydantic-django==0.1.1
+pynvim==0.5.0
 python-dateutil==2.8.2
 pytz==2021.1
 requests==2.28.1

pyproject.toml

@@ -0,0 +1,35 @@
+[project]
+name = "polyphonic"
+version = "0.8.3"
+description = "Polyphonic Ensemble Manager"
+authors = [
+    {name = "Tris Forster",email = "tris@tfconsulting.com.au"}
+]
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "django (==6.0.4)",
+    "django-crispy-forms",
+    "django-markdown2",
+    "django-rest-framework",
+    "crispy-bulma (>=0.12.0,<0.13.0)",
+    "django-byostorage @ git+https://gitea.tfconsulting.com.au/tris/django-byostorage.git@9903bb00888f20dfd2d39754e5ee22eeb5f36298",
+    "requests (>=2.32.5,<3.0.0)",
+    "django-storages (>=1.14.6,<2.0.0)",
+    "boto3 (>=1.40.20,<2.0.0)",
+    "whoosh (>=2.7.4,<3.0.0)"
+]
+
+[tool.poetry]
+packages = [{include = "*", from="app"}]
+
+[tool.poetry.group.dev.dependencies]
+django-debug-toolbar = "5.2"
+
+[tool.poetry.scripts]
+manage = "manage:main"
+
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"