Improved auto tagging

.gitignore

@@ -1,13 +1,15 @@
 __pycache__
 *.pyc
-db.sqlite3
+*.sqlite3
+credentials.json
 credentials
 local_settings.py
-env
+.coverage
-old
+/env
-test.*
+/data
-static
+/old
-teststore
+/static
-cache
+/teststore
-local_storage
+/cache
-media
+/local_storage
+/media

Makefile

@@ -1,6 +1,11 @@
 PYTHON=env/bin/python
 DROPZONE=5.7.0
 
+test:
+	coverage run --include "app/*" --omit "*/migrations/*" app/manage.py test app
+	coverage html
+	coverage report
+
 dev-setup:
 	env/bin/pip install -r requirements.txt
 	env/bin/pip install -r dev-requirements.txt

app/interface/forms.py

@@ -25,14 +25,16 @@ class BaseForm(forms.Form):
 
 class ProjectForm(forms.ModelForm, BaseForm):
 
-
     class Meta:
         model = models.Project
-        fields = ['name', 'description', 'event_date']
+        fields = ['name', 'description', 'modules', 'event_date']
         #widgets = {
         #    'event_date': forms.DateTimeInput(attrs={'type': 'date'})
         #}
 
+    modules = forms.MultipleChoiceField(choices=[ (x, x.title()) for x in models.settings.POLYPHONIC_MODULES ],
+                                        widget=forms.CheckboxSelectMultiple, required=False)
+
 class ResourceForm(forms.ModelForm, BaseForm):
 
     class Meta:

app/interface/models.py

@@ -41,6 +41,19 @@ def rough_date(d):
 def generate_code(length=9):
     return "".join([ random.choice('0123456789') for _ in range(length) ])
 
+class EnsembleQuerySet(models.QuerySet):
+    
+    def for_user(self, user, ensemble_keys=[], project_keys=[]):
+        if user.is_superuser:
+            return self 
+
+        f = models.Q(slug__in=ensemble_keys) | models.Q(projects__in=project_keys)
+        
+        if user.is_authenticated:
+            f |= models.Q(admins=user.pk)
+
+        return self.filter(f).distinct()
+
 class Ensemble(models.Model):
     ''' A group that plays together
     
@@ -57,11 +70,13 @@ class Ensemble(models.Model):
     nonce = models.SmallIntegerField(default=1,
             help_text="Increment this to reset the authentication links")
 
+    objects = EnsembleQuerySet.as_manager()
+
     class Meta:
         ordering = ('slug', )
 
     def active_projects(self):
-        return self.projects.filter(active=True, event_date__gte=timezone.now())
+        return self.projects.active().current()
 
     def has_admin(self, user):
         if not user.is_authenticated:
@@ -84,6 +99,23 @@ class Ensemble(models.Model):
     def __str__(self):
         return self.name
 
+class ProjectQuerySet(models.QuerySet):
+    def current(self):
+        return self.filter(models.Q(event_date__gte=(timezone.now()-timezone.timedelta(7))) | models.Q(event_date=None))
+    
+    def active(self):
+        return self.filter(active=True)
+
+    def for_user(self, user, project_keys=[], ensemble_keys=[]):
+        if user.is_superuser:
+            return self 
+
+        f = models.Q(pk__in=project_keys) | models.Q(ensemble__slug__in=ensemble_keys)
+        
+        if user.is_authenticated:
+            f |= models.Q(ensemble__admins=user.pk)
+
+        return self.filter(f)
 
 class Project(models.Model):
     ''' A Project linked to an ensemble
@@ -98,6 +130,8 @@ class Project(models.Model):
     nonce = models.SmallIntegerField(default=1,
             help_text="Increment this to reset the authentication links")
 
+    objects = ProjectQuerySet.as_manager()
+
     class Meta:
         ordering = ['active', 'event_date']
 
@@ -113,6 +147,8 @@ class Project(models.Model):
 
     @property
     def rough_date(self):
+        if not self.event_date:
+            return "No timescale"
         in_past, s = rough_date(self.event_date)
         if in_past:
             return f"{s} ago"
@@ -134,19 +170,6 @@ class Project(models.Model):
     def auth(self):
         return sign_data(f'{self.pk}-{self.nonce}', 12)
 
-    @classmethod
-    def for_user(cls, user, project_keys=[], ensemble_keys=[]):
-        projects = cls.objects.select_related('ensemble')
-        if user.is_superuser:
-            return projects 
-
-        f = models.Q(pk__in=project_keys) | models.Q(ensemble__slug__in=ensemble_keys)
-        
-        if user.is_authenticated:
-            f |= models.Q(ensemble__admins=user.pk)
-
-        return projects.filter(f)
-
     def __str__(self):
         return self.name
 
@@ -156,6 +179,9 @@ class Module(models.Model):
     name = models.SlugField(max_length=20, choices=[ (x, x.title()) for x in settings.POLYPHONIC_MODULES ])
     project = models.ForeignKey(Project, related_name="modules", on_delete=models.CASCADE)
 
+    def __str__(self):
+        return self.name
+
 def resource_key(resource, filename):
     return f'{resource.project.folder}/resources/{filename}'
 

app/interface/static/interface/js/interface.js

@@ -0,0 +1,29 @@
+document.addEventListener('DOMContentLoaded', () => {
+
+  // Get all "navbar-burger" elements
+  const navbarBurgers = Array.prototype.slice.call(document.querySelectorAll('.navbar-burger'), 0);
+
+  // Add a click event on each of them
+  navbarBurgers.forEach( el => {
+    el.addEventListener('click', () => {
+
+      // Get the target from the "data-target" attribute
+      const target = document.getElementById(el.dataset.target);
+
+      // Toggle the "is-active" class on both the "navbar-burger" and the "navbar-menu"
+      el.classList.toggle('is-active');
+      target.classList.toggle('is-hidden-touch');
+
+    });
+  });
+
+
+  // set all active links
+  const here = location.toString();
+  for (const el of document.getElementsByTagName('a')) {
+      if (el.href == here) {
+          el.classList.add('is-active');
+      }
+  }
+
+});

app/interface/templates/base.html

@@ -9,17 +9,22 @@
     <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.3/css/bulma.min.css">
     <link rel="stylesheet" href="{% static 'interface/css/polyphonic.css' %}"></link>
     <script src="{% static 'interface/js/interface.js' %}"></script>
+    <script src="//unpkg.com/alpinejs" defer></script>
+    <script src="//cdnjs.cloudflare.com/ajax/libs/jquery/3.5.1/jquery.min.js" defer></script>
+    <script src="//kit.fontawesome.com/c837098e5b.js" crossorigin="anonymous" defer></script>
     <title>{% block title %}Polyphonic{% endblock %}</title>
     {% block media %}{% endblock %}
+    <style>{% block style %}{% endblock %}</style>
   </head>
+  
   <body>
 
   {% block navigation %}
   <nav class="navbar" role="navigation">
     <div class="navbar-brand has-text-primary">
       <a class="navbar-item" href="/">
-        <span class="icon fancy is-size-2 mx-4"><i class="fas fa-random"></i></span>
+        <span class="icon fancy is-size-2 is-size-4-mobile mx-4"><i class="fas fa-random"></i></span>
-        <span class="fancy is-size-2">Polyphonic</span>
+        <span class="fancy is-size-2 is-size-4-mobile">Polyphonic</span>
       </a>
       <span class="navbar-item is-hidden-mobile fancy is-size-5">Musical Ensemble Manager</span>
 
@@ -34,7 +39,7 @@
     <div id="navbarMain" class="navbar-menu">
 
       <div class="navbar-end">
-        <span class="navbar-item is-size-4">{% firstof ensemble project.ensemble %}</span>
+        <span class="navbar-item is-size-5-touch is-size-4-tablet">{% firstof ensemble project.ensemble %}</span>
       </div>
 
     </div>
@@ -47,8 +52,6 @@
 {% endblock %}
 
   <!-- late load scripts -->
-  <script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
-  <script src="https://kit.fontawesome.com/c837098e5b.js" crossorigin="anonymous"></script>
   {% block scripts %}
   {% endblock %} 
   </body>

app/interface/templates/interface/ensemble_detail.html

@@ -21,14 +21,16 @@
 
 {% block page %}
 <h3 class="title">Projects for {{ensemble.name }}</h3>
-<p>{{ ensemble.details|markdown }}</p>
+<div class="content">
+{{ ensemble.details|markdown }}
+</div>
 
-<h4>Administrators</h4>
+<div class="block">
-<ul>
+Contacts: 
     {% for admin in ensemble.admins.all %}
-    <li><a href="mailto:{{ admin.email }}">{% firstof admin.get_full_name admin.get_username %}</a></li>
+    <a href="mailto:{{ admin.email }}" class="tag">{% firstof admin.get_full_name admin.get_username %}</a>
     {% endfor %}
-</ul>
+</div>
 
 {% include 'interface/project_items.html' %}
 
@@ -39,11 +41,10 @@
             <p class="card-header-title">Admin Details</p>
         </header>
         <div class="card-content">
-
+            <ul>
-            <p>
+                <li><a href="{{ ensemble_link }}">Ensemble Sharing Link</a></li>
-                Joining instructions for participants<br/><br/>
+                <li><a href="{% url 'project_create' ensemble.pk %}">Add a new project</a></li>
-                URL: <a href="{{ ensemble_link }}">{{ ensemble_link }}</a><br/>
+            </ul>
-            </p>
         </div>
     </div>
 </div>

app/interface/templates/interface/ensemble_list.html

@@ -19,22 +19,18 @@
         <div class="card">
             <div class="card-content">
                 <div class="media">
+                    
                     <div class="media-left">
                         <figure class="image is-48x48">
-                        <img src="https://bulma.io/images/placeholders/96x96.png" alt="Placeholder image">
+                        <img src="https://www.gravatar.com/avatar/{{ ensemble.email }}?d=mp" alt="Placeholder image">
                         </figure>
                     </div>
-                    <div class="media-content">
+                    <div class="media-content" style="min-height: 60px">
                         <a href="{% url 'ensemble_detail' ensemble.slug %}">
                             <p class="title is-4">{{ ensemble.name }}</p>
                         </a>
                     </div>
                 </div>
-                {% if ensemble.details %}
-                    <div class="content">
-                        {{ ensemble.details | markdown }}
-                    </div>
-                {% endif %}
             </div>
             <div class="card-footer">
                 {% with projects=ensemble.active_projects.count %}

app/interface/templates/interface/project_base.html

@@ -2,9 +2,7 @@
 
 {% block content %}
 
-
+<div class="columns mx-2">
-
-<div class="columns">
     <div class="column is-narrow is-hidden-touch" id="projectMenu">
         <div style="margin: auto 2em;">
             <aside class="menu">
@@ -14,7 +12,7 @@
                     <li><a href="{% url 'ensemble_list' %}">Ensembles</a></li>
                     <li><a href="{% url 'project_list' %}">Projects</a></li>
                     {% if request.user.is_authenticated %}
-                    <li><a href="{% url 'work_list' %}">Library</a></li>
+                    <!--li><a href="{% url 'work_list' %}">Library</a></li-->
                     <li><a href="{% url 'collection_list' %}">Collections</a></li>
                     {% endif %}
                 </ul>
@@ -27,13 +25,20 @@
                             <li><a class="nav-link" href="{% url 'item_list' project=project.pk %}">My Music</a></li>
                         {% endif %}
                         {% for page in project.wiki_pages.all %}
-                            <li><a class="nav-link 
+                            <li><a class="nav-link" 
                                 href="{% url 'wiki' project=project.id pk=page.id %}">{{ page.title }}</a></li>
                         {% endfor %}
                         <li><a role="tab" href="{% url 'resource_list' project=project.pk %}">Resources</a></li>
                     </ul>                
                 {% endif %}
 
+                {% if collection %}
+                <p class="menu-label">Collection</p>
+                <ul class="menu-list">
+                    <li><a role="tab" href="{% url 'collection_work_list' collection.pk %}">Work List</a></li>
+                </ul>
+                {% endif %}
+
                 <p class="menu-label">Admin</p>
                 <ul class="menu-list">
                     {% if request.user.is_staff %}
@@ -76,9 +81,9 @@
         </div>
 
         {% endif %}
-        <section class="section">
+        <div class="block">
             {% if request.is_admin %}
-            <div class="admin-tools is-pulled-right">
+            <div class="admin-tools is-pulled-right is-hidden-mobile">
             {% block admin %}
             {% endblock %}
             </div>
@@ -87,7 +92,7 @@
             {% block page %}
             No content
             {% endblock %}
-        </section>
+        </div>
     </div>
 </div>
 

app/interface/templates/interface/project_detail.html

@@ -14,7 +14,10 @@
 {% endblock %}
 
 {% block page %}
-<div class="box">
+<div class="columns is-multiline">
+    <div class="column ">
+        <div class="box">
+        
         {% if project.event_date %}
             <h3 class="subtitle is-centered">
                 <b>{{ project.event_date|date:"l jS F Y, g:i A" }}</b>
@@ -34,20 +37,6 @@
             {% endif %}
         </div>
     
-
-
-    {% if 'library' in modules %}
-        <div class="block">
-        {% include 'library/project_detail.html' %}
-        </div>
-    {% endif %}
-
-    {% if 'submission' in modules %}
-        <div class="block">
-        {% include 'submissions/project_detail.html' %}
-        </div>
-    {% endif %}
-    
         {% if project.owner %}
             <div class="block">
             {% if project.owner.email %}
@@ -57,17 +46,39 @@
             {% endif %}
             </div>
         {% endif %}
+        </div>
+    </div>
 
-    {% if request.is_admin %}
+    {% if 'library' in modules %}
-        <div class="block">
+    <div class="column is-one-third">
-            <a href="{{ project_link }}">Project Link</a>
+        {% include 'library/project_detail.html' %}
+    </div>
+    {% endif %}
+
+    {% if 'submission' in modules %}
+    <div class="column">
+        {% include 'submissions/project_detail.html' %}
     </div>
     {% endif %}
 
 </div>
 
+{% if request.is_admin %}
+<div class="box">
+    <h3 class="subtitle">Admin Actions</h3>
+    <ul>
+        <li><a href="{{ project_link }}">Project Link</a></li>
+        {% if 'library' in modules %}
+        <li><a href="{% url 'item_list_manage' project=project.pk %}">Manage items</a></li>
+        {% endif %}
+    </ul>
+</div>
+{% endif %}
+
+<!--
 <div>
 <a href="{% url 'forget_resource' 'project' project.pk %}">Forget this project</a>
 </div>
+-->
 
 {% endblock %}

app/interface/templates/interface/project_items.html

@@ -12,16 +12,19 @@
                     <p class="card-header-icon" style="color: black;">{{ project.rough_date }}</p>
             </header>
             </a>
-            <div class="card-content">
+            <div class="card-content" style="height: 100px; overflow: hidden">
-                <div class="content" style="height: 100px; overflow: hidden">
+                <div class="content">
                     {{ project.description | markdown }}
+                </div>
+            </div>
+
             {% if not ensemble %}
-                    <div class="has-text-centered"><i><small>With {{ project.ensemble }}</small></i></div>
+            <div class="card-footer">
+                <a class="card-footer-item" href="{% url 'ensemble_detail' project.ensemble.slug %}">{{ project.ensemble }}</a>
+            </div>
             {% endif %}
         </div>
     </div>
-        </div>
-    </div>
 {% empty %}
     <div class="hero">
         <div class="hero-body">

app/interface/templates/interface/resource_list.html

@@ -9,6 +9,7 @@
 {% endblock %}
 
 {% block page %}
+<h3 class="subtitle">Resources</h3>
 <div class="columns is-multiline">
 {% for resource in object_list %}
     {% with download=resource.file.url %}
@@ -51,7 +52,7 @@
     {% endwith %}
 {% empty %}
     <div class="column">
-    <p>There are resources for this project</p>
+    <p>There are no resources for this project</p>
     </div>
 {% endfor %}
 </div>

app/interface/tests/__init__.py

@@ -12,6 +12,8 @@ class AccessTestCase(TestCase):
 
     PROJECTS = ()
 
+    PROTECTED_URLS = ()
+
     @classmethod
     def setUpTestData(cls):
 
@@ -39,6 +41,14 @@ class AccessTestCase(TestCase):
         
         return
 
+    def test_protected_views(self):
+
+        self.assertAccess({ x: False for x in self.PROTECTED_URLS })
+
+        if 'admin' in self.users:
+            self.client.force_login(self.users['admin'])
+            self.assertAccess({ x: True for x in self.PROTECTED_URLS })
+
     def login(self, user, passwd):
         response = self.client.post('/login', {'username': user, 'password': passwd})
         self.assertEqual(response.status_code, 302, f"Failed to login as {user}")

app/interface/tests/test_access.py

@@ -26,6 +26,20 @@ class InterfaceAccessTestCase(AccessTestCase):
         {'name': 'Current Repertoire', 'ensemble': 'be-sharps'},
     )
 
+    PROTECTED_URLS = (
+        '/ensembles/be-sharps',
+        '/ensembles/be-sharps/new-project',
+
+        '/projects/3',
+        '/projects/3/resources',
+        '/projects/3/resources/add',
+
+        '/admin/interface/ensemble/',
+        '/admin/interface/project/',
+        '/admin/interface/resource/',
+        '/admin/interface/wikipage/',
+    )
+
     def test_bad_login(self):
         with self.assertRaisesMessage(self.failureException, 'Failed to login as admin'):
             self.login('admin', 'admin')

app/interface/views.py

@@ -35,7 +35,7 @@ class AuthorizedResourceMixin(object):
     def is_authorized(self):
         "By default check if superuser or a signed request"
         if self.request.is_admin:
-            logger.debug("is_authorized: superuser")
+            #logger.debug("is_authorized: superuser")
             return True
         
         if 'sig' in self.request.GET:
@@ -51,12 +51,12 @@ class AuthorizedResourceMixin(object):
     def is_authorized_key(self, resource, key, auth):
         current = self.get_authorized_keys(resource).get(str(key), None)
         if current is None:
-            logger.debug("is_authorized_key: %s %s not in session", resource, key)
+            #logger.debug("is_authorized_key: %s %s not in session", resource, key)
             return False
         if auth == current:
             return True
 
-        logger.info("Authorisation revoked")
+        #logger.info("Authorisation revoked")
         self.del_authorized_key(resource, key)
         return False
 
@@ -196,19 +196,22 @@ class EnsembleListView(AuthorizedResourceMixin, ListView):
         return True
     
     def get_queryset(self):
-        ensembles = models.Ensemble.objects.all()
+        return models.Ensemble.objects.for_user(self.request.user,
+                                                self.get_authorized_keys('ensemble').keys(),
+                                                self.get_authorized_keys('project').keys())
+        #ensembles = models.Ensemble.objects.all()
 
-        if self.request.is_admin:
+        #if self.request.is_admin:
-            return ensembles
+        #    return ensembles
 
         # limit to registered ensembles
-        f = Q(slug__in=self.get_authorized_keys('ensemble').keys()) | Q(projects__in=self.get_authorized_keys('project').keys())
+        #f = Q(slug__in=self.get_authorized_keys('ensemble').keys()) | Q(projects__in=self.get_authorized_keys('project').keys())
         
         # or ensembles where the user is admin
-        if self.request.user.is_authenticated:
+        #if self.request.user.is_authenticated:
-            f |= Q(admins=self.request.user.pk)
+        #    f |= Q(admins=self.request.user.pk)
 
-        return ensembles.filter(f).distinct()
+        #return ensembles.filter(f).distinct()
 
 class EnsembleDetailView(EnsembleMixin, DetailView):
 
@@ -232,12 +235,11 @@ class EnsembleDetailView(EnsembleMixin, DetailView):
         if inactive:
             projects = projects.order_by('-pk')
         else:
-            projects = projects.filter(active=True, event_date__gte=timezone.now()-timezone.timedelta(7))
+            projects = projects.active().current()
 
         data['inactive'] = inactive
         data['object_list'] = projects
         if self.request.is_admin:
-            #data['ensemble_link'] = signed_url(f'{self.request.path}?nonce={self.ensemble.nonce}')
             data['ensemble_link'] = self.request.path + "?auth=" + self.ensemble.auth()
         return data
 
@@ -254,14 +256,12 @@ class ProjectListView(ProjectMixin, ListView):
         return True
     
     def get_project_queryset(self):
-        return models.Project.for_user(self.request.user,
+        return models.Project.objects.for_user(self.request.user,
                                        self.get_authorized_keys('project'),
                                        self.get_authorized_keys('ensemble'))
 
     def get_queryset(self):
-        qs = self.get_project_queryset()
+        return self.get_project_queryset().current().active()
-        f = Q(event_date__gte=(timezone.now()-timezone.timedelta(7))) | Q(event_date=None)
-        return qs.filter(f, active=True)
 
 class ProjectCreateView(EnsembleMixin, CreateView):
     admin_required = True
@@ -270,11 +270,16 @@ class ProjectCreateView(EnsembleMixin, CreateView):
     title = "Add a new project"
     form_class = forms.ProjectForm
 
+
     def form_valid(self, form):
         self.object = form.save(commit=False)
         self.object.ensemble = self.ensemble
         self.object.owner = self.request.user
         self.object.save()
+
+        for module in form.cleaned_data['modules']:
+            self.object.modules.create(name=module)
+
         return redirect('project_detail', project=self.object.pk)
 
 class ProjectDetailView(ProjectMixin, DetailView):
@@ -306,12 +311,25 @@ class ProjectUpdateView(ProjectMixin, UpdateView):
     def get_object(self):
         return self.project
 
+    def get_initial(self):
+        data = super().get_initial()
+        data['modules'] = self.object.active_modules
+        print(data)
+        return data
+    
+    def form_valid(self, form):
+        self.object = form.save()
+        current = set(self.object.active_modules)
+        desired = set(form.cleaned_data['modules'])
+        self.object.modules.exclude(name__in=desired).delete()
+        for module in desired-current:
+            self.object.modules.create(name=module)
+        return redirect('project_detail', self.kwargs['project'])
+
     @property
     def cancel_url(self):
-        return self.get_success_url()
+        return resolve_url('project_detail', self.kwargs['project'])
 
-    def get_success_url(self):
-        return resolve_url('project_detail', project=self.kwargs['project'])
 
 # Old Makefile from submission module
 #class ProjectMakefileView(EnsembleMixin, DetailView):

app/library/admin.py

@@ -34,7 +34,7 @@ admin.site.register(models.Work, WorkAdmin)
 
 class SectionInline(admin.TabularInline):
     model = models.Section
-    fields = ['type', 'tag', 'ordinal', 'start', 'end', 'page']
+    fields = ['tag', 'start', 'end', 'page']
 
 class DocumentAdmin(admin.ModelAdmin):
     list_display = ['work', '__str__']

app/library/forms.py

@@ -25,5 +25,9 @@ class PlaylistAddForm(forms.Form):
     def save(self):
         self.instance.works.add(self.cleaned_data['work'])
 
-class ProjectSelectForm(forms.Form):
+class ProjectEnsembleChoiceField(forms.ModelChoiceField):
-    project = forms.ModelChoiceField(queryset=Project.objects.all())
+    def label_from_instance(self, obj):
+        return f"{obj.ensemble.name} - {obj.name}"
+
+class ProjectSelectForm(BaseForm):
+    project = ProjectEnsembleChoiceField(queryset=Project.objects.all())

app/library/models.py

@@ -9,6 +9,7 @@ from django.core.files.storage import get_storage_class
 from django.db.models import Q, Count, Min, Max
 
 import re
+import os.path
 
 from byostorage.user import BYOStorage
 from byostorage.cached import CachedStorage
@@ -36,8 +37,6 @@ class Orchestration(models.Model):
     Stores a list of instrument codes as a single entry (space delimited).
     Can be global or ensemble specific
     """
-    # TODO: change ensemble to collection
-    #ensemble = models.ForeignKey('interface.Ensemble', on_delete=models.CASCADE, related_name="orchestrations", null=True, blank=True)
     collection = models.ForeignKey('Collection', on_delete=models.CASCADE, related_name="custom_orchestrations", null=True, blank=True)
     name = models.CharField(max_length=100)
     instruments = models.TextField()
@@ -204,19 +203,25 @@ class Work(models.Model):
     def folder(self):
         return f"{slugify(self.composer)}/{slugify(self.name)}-{self.pk:04d}"
 
-    def extract(self, *tags):
+    def tagged_sections(self, *tags):
-
         qs = self.docs.filter(sections__tag__in=tags)
         qs = qs.annotate(Count('sections'), end=Min('sections__end'), start=Max('sections__start')) \
             .filter(sections__count=len(tags))
+        return qs
 
-        return list(qs.values_list('upload', 'start', 'end'))
+    def list_sections(self, *tags):
+        return list(self.tagged_sections(*tags).values_list('upload', 'start', 'end'))
 
     @property
     def digital_parts(self):
-        sections = [ (s.tag, s) for s in Section.objects.filter(doc__work=self.pk) ]
+        sections = [ (s.tag, s.name) for s in Section.objects.filter(doc__work=self.pk) ]
         sections.sort(key=self.orchestration.sorter())
-        return [ s[1] for s in sections ]
+        #return [ s[1] for s in sections ]
+        sections = list(dict(sections).items()) # primitive unique()
+        return sections
+    
+    def pdfs(self):
+        return self.docs.filter(doctype=Document.DOCTYPE_PDF)
 
     @property
     def physical_parts(self):
@@ -234,10 +239,6 @@ class Work(models.Model):
     def meta(self):
         return self.meta_info.exclude(name='tag')
 
-    @property
-    def active_projects(self):
-        return self.projects.filter(active=True)
-
     @property
     def current_loans(self):
         return self.project_items.filter(checkout__lte=now(), returned=None).select_related('project')
@@ -276,12 +277,20 @@ class Work(models.Model):
         return f"{composer[:4]}-{work}-{self.pk:05d}".upper()
 
     def assigned_instruments(self):
-        return Section.objects.filter(doc__work_id=self.pk, type=Section.TYPE_INSTRUMENT).values_list('tag', flat=True)
+        return Section.objects.filter(doc__work_id=self.pk).values_list('tag', flat=True)
 
     def unassigned_instruments(self):
         assigned = set(self.assigned_instruments())
         return [ x for x in self.orchestration.as_list() if not x[0] in assigned ]
 
+    def music_tags(self):
+        tags = dict(self.orchestration.as_list())
+        for section in Section.objects.filter(doc__work_id=self.pk):
+            tags.setdefault(section.tag, section.name)
+
+        return tags.items()
+
+
     def __str__(self):
         return f"{self.name} ({self.composer})"
 
@@ -315,15 +324,21 @@ class Document(models.Model):
     DOCTYPE_PDF = 1
     DOCTYPE_AUDIO = 2
     DOCTYPE_VIDEO = 3
-    DOCTYPE_SOURCE = 4
+    DOCTYPE_MISC= 4
 
     DOCTYPES = (
         (DOCTYPE_PDF, 'PDF'),
         (DOCTYPE_AUDIO, 'Audio'),
         (DOCTYPE_VIDEO, 'Video'),
-        (DOCTYPE_SOURCE, 'Source'),
+        (DOCTYPE_MISC, 'Misc'),
     )
 
+    DOCTYPE_MAP = {
+        '.pdf': DOCTYPE_PDF,
+        '.mp3': DOCTYPE_AUDIO,
+        '.mp4': DOCTYPE_VIDEO,
+    }
+
     work = models.ForeignKey('Work', on_delete=models.CASCADE, related_name="docs")
     doctype = models.PositiveSmallIntegerField(choices=DOCTYPES, default=DOCTYPE_PDF)
     upload = models.FileField(upload_to=doc_upload_filename, storage=library_storage)
@@ -334,6 +349,9 @@ class Document(models.Model):
         self.upload.delete(save=False)
         return super().delete(*args, **kwargs)
     
+    def filename(self):
+        return os.path.basename(self.upload.name)
+
     def __str__(self):
         return self.upload.name
 

app/library/music_tags.py

@@ -6,16 +6,19 @@ mvmt    Movement
 ex      Excerpt
 sec     Section
 pce     Piece
+no      Number
 """
 
 # taken from https://imslp.org/wiki/IMSLP:Abbreviations_for_MusicTags
-# Abbreviations at the top will take precidence in reverse lookups
+# Include any aliases at the top
 
 INSTRUMENTS = """
 score   Score
 
-cb      Double bass
+cb      Double Bass
-mall    Mallet percussion
+mall    Mallet Percussion
+vln     Violin
+vla     Viola
 
 acc 	Accordion 
 afl 	Alto flute 
@@ -228,6 +231,35 @@ class MusicTag(namedtuple('MusicTag', ('name', 'variant'), defaults=[None])):
             return f"{self.name} {self.variant}"
         return self.name
 
+import re
+PATTERNS = [re.compile('([A-Za-z]+)[_\- ]*(\d+)'), re.compile('([A-Za-z]+)()')]
+
+def auto_tag(filename):
+    '''
+    
+    >>> auto_tag('Ode to Joy - Violin 1.pdf')
+    MusicTag(name='Violin', variant=1)
+    >>> auto_tag('Ode to Joy_Cello.pdf')
+    MusicTag(name='Cello', variant=None)
+    >>> auto_tag('Ode to Joy violin - 1.pdf')
+    MusicTag(name='Violin', variant=1)
+    >>> auto_tag('Ode to Joy - vla.pdf')
+    MusicTag(name='Viola', variant=None)
+    >>> auto_tag('Ode to Joy - fl-2 (piccolo).pdf')
+    MusicTag(name='Flute', variant=2)
+    '''
+
+    for pattern in PATTERNS:
+        for inst, ordinal in pattern.findall(filename):
+            inst = inst.lower()
+            ordinal = int(ordinal) if ordinal else None
+            if inst in MUSIC_TAG_BY_NAME:
+                return MusicTag(inst.title(), ordinal)
+            if inst in MUSIC_NAME_BY_TAG:
+                return MusicTag(MUSIC_NAME_BY_TAG[inst], ordinal)
+    
+    
+
 if __name__ == "__main__":
     import doctest
     print(doctest.testmod())

app/library/pdf_utils.py

@@ -24,12 +24,12 @@ def extract_and_concat(items):
         if count == 0:
             continue
 
-        if start is None:
+        if not start:
             sections.append(source)
 
         else:
 
-            if end is None:
+            if not end:
                 end = start
 
             dest = os.path.join(d.name, f'section_{i}.pdf')

app/library/templates/library/collection_list.html

@@ -3,6 +3,19 @@
 {% block page %}
 <h3 class="title">Library collections for {% firstof request.user.first_name request.user.username %}</h3>
 
+<div class="block">
+<form method="GET" action="{% url 'work_list' %}">
+    <div class="field has-addons">
+        <div class="control is-expanded">
+            <input class="input" name="filter" type="text" placeholder="Find a work" value="{{ request.GET.filter }}"/>
+        </div>
+        <div class="control">
+            <a class="button" href="?"><i class="fas fa-times"></i></a>
+        </div>
+    </div>
+</form>
+</div>
+
 <div class="columns is-multiline">
 {% for collection in object_list %}
     <div class="column is-half">

app/library/templates/library/document_annotate.html

@@ -75,11 +75,11 @@
     </div>
   </div>
   <ul id="unassigned-area">
-  {% for tag, inst in document.work.unassigned_instruments %}
+  {% for tag, inst in document.work.music_tags %}
     <li class="is-clickable" onclick="assignInstrument('{{tag}}', this)")>{{ inst }}</li>
   {% endfor %}
-    <li><a onclick="document.getElementById('add-modal').classList.add('is-active')">Add instrument</a></li>
   </ul>
+  <a onclick="document.getElementById('add-modal').classList.add('is-active')">Add Tag</a>
   </div>
   <div class="column is-narrow">
     <div class="has-text-centered">
@@ -122,7 +122,7 @@
           <input type="number" class="input" max="4" min="1" size="3" id="add-instrument-variant"/>
         </span>
         <span class="control">
-          <button class="button is-primary" onclick="addInstrument()">Add</button>
+          <button class="button is-primary" onclick="addInstrument();">Add</button>
         </span>
       </div>
     </section>
@@ -301,12 +301,19 @@
     variant.value = '';
     name.value = '';
 
+    let li = document.createElement('li');
+    li.classList.add("is-clickable");
+    li.addEventListener('click', () => assignInstrument(tag, li));
+    li.innerHTML = get_instrument(tag);
+    document.getElementById('unassigned-area').appendChild(li);
+    
     addTag(tag, pageNum, pageNum);
+    closeAddModal();
   }
 
   function assignInstrument(tag, el) {
     addTag(tag, pageNum, pageNum);
-    el.remove();
+    //el.remove();
   }
 
   function addTag(tag, start, end) {
@@ -334,11 +341,6 @@
     del.className = "icon is-action";
     del.innerHTML = '<i class="fas fa-trash-alt" title="Remove this tag"></i>';
     del.addEventListener('click', () => {
-      let li = document.createElement('li');
-      li.classList.add("is-clickable");
-      li.addEventListener('click', () => assignInstrument(tag, li));
-      li.innerHTML = get_instrument(el.dataset.tag);
-      document.getElementById('unassigned-area').appendChild(li);
       el.remove();
       dirty=true;
     });

app/library/templates/library/document_entry.html

@@ -9,8 +9,10 @@
     </td>
     <td class="has-text-right" style="white-space: nowrap;">
     {% if request.is_admin %}
+        {% if doc.doctype == 1 %}
         <a href="{% url 'document_annotate' collection.pk doc.pk %}"><i class="fas fa-tags"
                 title="Manage Tags"></i></a>
+        {% endif %}
         <a href="{% url 'document_delete' collection.pk doc.pk %}"><i class="fas fa-trash-alt" title="Delete Document"></i></a>
     {% endif %}
     </td>

app/library/templates/library/item_list.html

@@ -1,21 +1,14 @@
 {% extends "interface/project_base.html" %}
 
-{% block admin %}
-<a class="button is-link" href="{% url 'item_list_manage' project.pk %}">
-    <span class="icon"><i class="fas fa-list"></i></span>
-    <span>Change items</span>
-</a>
-{% endblock %}
-
 {% block page %}
 <form action="" method="post" target="_blank">
 {% csrf_token %}
 <div>
-    <div class="field is-grouped is-grouped-centered">
+    <div class="field is-grouped is-grouped-centered is-grouped-multiline">
-        <div class="field has-addons control">
+        <div class="field has-addons control has-addons-centered is-expanded">
             
-            <span class="control">
+            <span class="control is-hidden-mobile">
-                <a class="button is-static">Your Instrument</a>
+                <a class="button is-static">Instrument</a>
             </span>
 
             <span class="control">
@@ -47,12 +40,12 @@
     </div>
 </div>
 
-<table class="table is-striped is-fullwidth">
+<table class="table is-striped mx-auto">
     <thead>
         <tr>
             <th/>
             <th>Piece</th>
-            <th>Running time</th>
+            <th class="is-hidden-mobile">Running time</th>
             <th>Part</th>
             <th/>
         </tr>
@@ -63,25 +56,27 @@
                 <td>{{ forloop.counter }}.</td>
                 <td>
                     {% if request.is_admin %}
-                    <a href="{% url 'work_detail' item.work.pk %}">{{ item.work.name }}</a>
+                    <a href="{% url 'work_detail' item.work.collection.pk item.work.pk %}">{{ item.work.name }}</a>
                     {% else %}
                     {{ item.work.name }}
                     {% endif %}
                 </td>
-                <td>{% firstof item.work.running_time "------" %}</td>
+                <td class="is-hidden-mobile">{% firstof item.work.running_time "------" %}</td>
                 <td class="select-cell">
                     <input type="hidden" name="works" value="{{ item.work.pk }}"/>
-                    <span class="select is-small">
+                    <span class="select is-small" style="width: 100%">
-                    <select name="instruments">
+                    <select name="instruments" id="part-{{ item.work.pk }}" style="width: 100%">
                         <option value='-'>None</option>
-                        {% for part in item.work.digital_parts %}
+                        {% for tag, name in item.work.digital_parts %}
-                        <option value='{{ part.tag }}'>{{ part.instrument }}</option>
+                        <option value='{{ tag }}'>{{ name }}</option>
                         {% endfor %}
                     </select>
                     </span>
                 </td>
                 <td>
-                    <a href=""><i class="fas fa-download"></i></a>
+                    <span class="is-action" onclick="downloadPart({{ item.work.collection_id }}, {{ item.work.pk }})">
+                        <i class="fas fa-download" title="Download Part"></i>
+                    </span>
                 </td>
             </tr>
         {% endfor %}
@@ -90,11 +85,13 @@
         <tr>
             <td/>
             <td/>
-            <td>{{ running_time }}</td>
+            <td>{% firstof running_time "------" %}</td>
             <td/>
             <td>
+            <!--
                 <button class="button is-link is-small" type="submit"><span class="icon"><i class="fas fa-copy"></i></span><span>Single combined PDF</span></button>
                 <a class="button is-link is-small"><span class="icon"><i class="fas fa-archive"></i></span><span>Individual files (zipped)</span></a>
+            -->
             </td>
         </tr>
     </tfoot>
@@ -154,6 +151,17 @@ function selectParts(inst, part) {
 
 }
 
+function downloadPart(collection, work) {
+    let part = document.getElementById("part-" + work).value;
+    if (part == '-') {
+        alert("No part selected!");
+        return;
+    }
+    console.log(part);
+    let url = "/collections/" + collection + "/works/" + work + "/download?tag=" + part;
+    window.location = url;
+}
+
 document.getElementById("instrument-name").value = localStorage.getItem('instrument-name', 'All');
 document.getElementById("part-preference").value = localStorage.getItem('part-preference', 'Any');
 updateParts();

app/library/templates/library/project_detail.html

@@ -1,12 +1,50 @@
-{% with project.items.count as items %}
+<div class="panel">
-{% if items %}
+    <p class="panel-heading">
-<p>
+        Items
-There {{ items|pluralize:"is,are" }} currently {{ items}} item{{ items|pluralize }} for this project.<br/>
+    </p>
-Click the '<a href="{% url 'item_list' project=project.pk %}">My Music</a>' button below to access your parts...
+  
-</p>
+    {% for item in project.items.all %}
-{% else %}
+        {% with work=item.work %}
-<p>
+        <a class="panel-block" onclick="showDownloadOptions('{% url 'work_parts' work.collection_id  work.pk %}')">
-There are no items listed yet for this project - please check back later... 
+            {{ item.work.name }}
-</p>
+        </a>
-{% endif %}
+        {% endwith %}
-{% endwith %}
+    {% empty %}
+        <span class="panel-block">
+            There are no items listed yet for this project - please check back later...
+        </span>
+    {% endfor %}
+</div>
+
+<div class="modal" id="download-modal">
+  <div class="modal-background" onclick="closeDownloadModal()"></div>
+  <div class="modal-card">
+    <header class="modal-card-head">
+      <p class="modal-card-title">Parts</p>
+      <button class="delete" aria-label="close" onclick="closeDownloadModal()"></button>
+    </header>
+    <section class="modal-card-body" id="download-target">
+    </section>
+  </div>
+</div>
+
+<script>
+
+function showDownloadOptions(url) {
+    let target = document.getElementById("download-target");
+    target.innerHTML = "Loading available parts...";
+
+    document.getElementById("download-modal").classList.add('is-active');
+    fetch(url).then((data) => {
+        console.log(data);
+        data.text().then((text) => {
+            target.innerHTML = text;
+        });
+    });
+}
+
+function closeDownloadModal() {
+    document.getElementById("download-modal").classList.remove('is-active');
+}
+
+</script>

app/library/templates/library/project_menu.html

@@ -2,6 +2,9 @@
 <ul class="menu-list">
 {% if project %}
 <li><a href="{% url 'item_list' project=project.pk %}">My Music</a></li>
+{% endif %}
+{% if collection %}
+
 {% endif %}
 {% if request.user.is_authenticated %}
 <li><a href="{% url 'work_list' %}">Library</a></li>

app/library/templates/library/work_detail.html

@@ -49,7 +49,7 @@
     </table>
 
     {% if work.parent %}
-    <p>From <a href="{% url 'work_detail' collection.pk work.parent.pk %}">{{ work.parent.name }} - {{ work.parent.composer }}</a>
+    <p>From <a href="{% url 'work_detail' work.parent.collection.pk work.parent.pk %}">{{ work.parent.name }} - {{ work.parent.composer }}</a>
     </p>
     {% endif %}
 
@@ -57,7 +57,7 @@
     <h3>Related</h3>
     <ul>
         {% for related in work.related_works.all %}
-        <li><a href="{% url 'work_detail' related.pk %}">{{ related.name }} - {{ related.composer }}</a></li>
+        <li><a href="{% url 'work_detail' related.collection.pk related.pk %}">{{ related.name }} - {{ related.composer }}</a></li>
         {% endfor %}
     </ul>
     {% endif %}
@@ -87,15 +87,17 @@
                     Digital Parts
                 </h4>
                 <div class="tags">
-                    {% if work.digital_parts %}
+                    {% with sections=work.digital_parts %}
+                        {% if sections %}
                         <a class="tag is-danger" href="{% url 'work_partset' collection.pk work.pk %}">Full Set</a>
                         {% endif %}
-                    {% for section in work.digital_parts %}
+                        {% for tag, name in sections %}
-                    <a class="tag is-info" href="{% url 'part_download' collection.pk section.pk section.filename %}"
+                        <a class="tag is-info" href="{% url 'work_download' collection.pk work.pk %}?tag={{ tag }}"
-                        target="section_{{ section.pk }}" rel="">{{ section.name }}</a>
+                            target="section_{{ tag }}" rel="">{{ name }}</a>
                         {% empty %}
                         <p class="is-italic">No digital parts available</p>
                         {% endfor %}
+                    {% endwith %}
                 </div>
             </div>
         </div>
@@ -129,7 +131,7 @@
             {% if request.is_admin %}
             <div class="column is-one-quarter">
                 <h4 class="is-size-5">Upload files</h4>
-                <form action="{% url 'document_add' collection.pk object.pk %}" class="dropzone" id="doc-upload">
+                <form action="{% url 'document_add' collection.pk object.pk %}" class="dropzone" id="doc-upload" style="-moz-user-select: none">
                     {% csrf_token %}
                 </form>
             </div>

app/library/templates/library/work_list.html

@@ -29,10 +29,7 @@
             <th>Work</th>
             <th>Composer</th>
             <th class="is-hidden-mobile">Edition</th>
-            <th class="is-hidden-touch">Collection</th>
+            {% if not collection %}<th class="is-hidden-touch">Collection</th>{% endif %}
-            {% if request.is_admin %}
-            <th class="is-hidden-mobile">Copies</th>
-            {% endif %}
         </tr>
     </thead>
     <tbody>
@@ -41,10 +38,7 @@
                 <td><a href="{% url 'work_detail' collection=work.collection.pk pk=work.pk %}">{{ work.name }}</a></td>
                 <td title="{{ work.composer }}">{{ work.composer|truncatewords:3 }}</td>
                 <td class="is-hidden-mobile" title="{{ work.edition }}">{{ work.edition|truncatewords:2 }}</td>
-                <td class="is-hidden-touch">{{ work.collection.name }}</td>
+                {% if not collection %}<td class="is-hidden-touch">{{ work.collection.name }}</td>{% endif %}
-                {% if request.is_admin %}
-                <td class="is-hidden-mobile {{ work.is_available|yesno:'has-text-success,has-text-danger' }}">{{ work.available }}</td>
-                {% endif %}
             </tr>
         {% empty %}
             <tr><td colspan="4">No works found</td></tr>

app/library/templates/library/work_parts_fragment.html

@@ -0,0 +1,25 @@
+<h3 class="subtitle">{{ work.name }}</h3>
+<div class="block">
+{% for tag, name in work.digital_parts %}
+<a class="tag is-info" href="{% url 'work_download' work.collection_id work.pk %}?tag={{ tag }}">{{ name }}</a>
+{% endfor %}
+</div>
+
+<h3 class="subtitle">Files</h3>
+<table class="table is-narrow is-fullwidth">
+<tbody>
+    {% for doc in work.pdfs %}
+    <tr>
+        <td><a href="{{ doc.upload.url }}">{{ doc.filename }}</a></td>
+    </tr>
+    {% empty %}
+    <tr>
+        <td>There are no files available for this work</td>
+    </tr>
+    {% endfor %}
+</tbody>
+</table>
+
+<p>
+<a href="{% url 'work_detail' work.collection_id work.pk %}">More details...</a>
+</p>

app/library/templates/library/work_partset.html

@@ -3,35 +3,51 @@
 
 
 {% block page %}
+<h3 class="subtitle"><a href="{% url 'work_detail' collection.pk work.pk %}">{{ work.name }}</a></h3>
+
+<p class="block">
+    You can generate a custom partset for printing - select the number of copies of each you want...
+</p>
+
+<div class="columns is-centered">
+    <div class="column is-narrow">
 
 <form action="" method="post" target="_blank">
     {% csrf_token %}
     
-    <div class="admin-tools is-pulled-right">
+    <table class="table">
-    <button type="submit" class="button is-link">
+        <thead>
+
+        </thead>
+        <tbody>
+        {% for tag, name in work.digital_parts %}
+            <tr>
+                <td>{{ name }}</td>
+                <td>
+                    <input name="parts" type="hidden" value="{{ tag }}">
+                    <input class="input is-small" name="copies" type="number" value="{% if tag == 'score' %}0{% else %}1{% endif %}">
+                </td>
+            </tr>
+        {% endfor %}
+        </tbody>
+    </table>
+
+    <div class="field is-grouped">
+        <div class="control">
+            <button class="button is-link">
                 <span class="icon"><i class="fas fa-print"></i></span>
                 <span>Print Set</span>
             </button>
+        </div>
+        <div class="control">
             <a class="button is-link is-light" href="{% url 'work_detail' collection.pk object.pk %}">
                 <span>Cancel</span>
             </a>
         </div>
-
-
-    <h3 class="subtitle"><a href="{% url 'work_detail' collection.pk work.pk %}">{{ work.name }}</a></h3>
-
-    <p class="block">
-        You can generate a custom partset for printing - select the number of copies of each you want...
-    </p>
-
-    <div class="columns is-multiline is-mobile">
-        {% for part in work.digital_parts %}
-        <div class="column is-3 has-text-right">
-            <span style="white-space: nowrap">{{ part.name }}</span>
-            <input name="parts" type="hidden" value="{{ part.tag }}">
-            <input class="input" name="copies" type="number" value="{% if part.tag == 'score' %}0{% else %}1{% endif %}" size="1">
-        </div>
-        {% endfor %}
     </div>
+    
 </form>
+
+</div>
+</div>
 {% endblock %}

app/library/tests.py

@@ -1,8 +1,11 @@
 from interface.tests import AccessTestCase 
 
-from django.contrib.auth.models import User
+from byostorage.user import UserStorage
-from interface.models import Ensemble, Project
 from . import models
+from .views.api import WorkSerializer
+
+import tempfile
+import json
 
 class LibraryTestCase(AccessTestCase):
 
@@ -30,7 +33,8 @@ class LibraryTestCase(AccessTestCase):
     )
 
     WORKS = (
-        {'name': 'Baby on Board', 'collection': 'ned'},
+        {'name': 'Baby on Board', 'collection': 'ned', 'docs': [{'upload': 'local:baby_on_board.pdf'}]},
+        {'name': 'Star Spangled Banner', 'collection': 'sel'},
     )
 
     PROTECTED_URLS = (
@@ -41,47 +45,61 @@ class LibraryTestCase(AccessTestCase):
         '/collections/2/works/1/partset',
         '/collections/2/works/1/add_to_project',
         '/collections/2/works/1/upload',
+        '/collections/2/docs/1/annotate',
+
+        # Need to add storage before we can test these
+        '/api/collections/2',
+        '/api/collections/2/works/1',
+
+        '/admin/library/collection/',
+        '/admin/library/document/',
+        '/admin/library/ensembleaccess/',
+        '/admin/library/orchestration/',
+        '/admin/library/projectitem/',
+        '/admin/library/work/',
     )
 
     @classmethod
     def setUpTestData(cls):
         super().setUpTestData()
         
+        cls.temp_dir = tempfile.TemporaryDirectory()
+        cls.storage = UserStorage.objects.create(name='local', storage='django.core.files.storage.FileSystemStorage',
+                                            settings_data=json.dumps({'location': cls.temp_dir.name, 'base_url': 'file://' + cls.temp_dir.name}))
         cls.collections = {}
         for details in cls.COLLECTIONS:
             admins = details.pop('admins', [])
-            obj = models.Collection.objects.create(**details)
+            obj = models.Collection.objects.create(storage=cls.storage, **details)
             for admin in admins:
                 obj.administrators.add(cls.users[admin])
             cls.collections[details['prefix']] = obj
 
         cls.works = {}
         for details in cls.WORKS:
-            collection = details.pop('collection')
+            collection = cls.collections[details.pop('collection')]
-            cls.works[details['name']] = models.Work.objects.create(collection=cls.collections[collection], **details)
+            #details.setdefault('docs', [])
+            #details.setdefault('meta_info', [])
+            #s = WorkSerializer(data=details)
+            #assert s.is_valid(), s.errors
+            #s.save(collection_id=collection.pk)
+            docs = details.pop('docs', [])
+            obj = models.Work.objects.create(collection=collection, **details)
+            for doc in docs:
+                obj.docs.create(**doc)
+            cls.works[details['name']] = obj
 
-    def oldSetUp(self):
-        self.homer = User.objects.create(username='homer')
-        self.ned = User.objects.create(username="ned")
-        self.lisa = User.objects.create(username="lisa")
-        self.dewey = User.objects.create(username="dewey")
 
-        self.be_sharps = self.homer.ensembles.create(name='Be Sharps', code="barbershop")
+    def setUp(self):
-        self.sesd = self.dewey.ensembles.create(name="Springfield Elementary School Band", code="sax")
+        pass
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.temp_dir.cleanup()
         
-        self.sel = self.lisa.collections.create(name="Springfield Elementary Library")
-        self.flanders = self.ned.collections.create(name="Neds Shed")
 
     def test_integration(self):
         pass
 
-    def test_get_views(self):
-
-        self.assertAccess({ x: False for x in self.PROTECTED_URLS })
-
-        self.login('admin', 'secret')
-        self.assertAccess({ x: True for x in self.PROTECTED_URLS })
-
     def test_superuser_access(self):
         self.login('admin', 'secret')
         self.assertAccess({
@@ -123,6 +141,11 @@ class LibraryTestCase(AccessTestCase):
             '/collections/2/works/1': False,
         })
 
+    def test_export_and_import(self):
+        self.login('admin', 'secret')
+        data = self.client.get('/api/collections/1/works/2', HTTP_ACCEPT="application/json").json()
+        response = self.client.post('/api/collections/2/import', data, "application/json")
+        self.assertEqual(response.status_code, 201) 
 
     def test_movement_from_large_work(self):
         '''
@@ -139,15 +162,15 @@ class LibraryTestCase(AccessTestCase):
             doc.sections.create(tag=g)
 
         # no tags - get nothing (should it be everything?)
-        self.assertEqual(work.extract(), [])
+        self.assertEqual(work.list_sections(), [])
 
         # single tag - should get just that range
-        self.assertEqual(work.extract('vl-1'), [('sel/beethoven/some_quartet/some_quartet_vl-1.pdf', None, None)])
+        self.assertEqual(work.list_sections('vl-1'), [('sel/beethoven/some_quartet/some_quartet_vl-1.pdf', None, None)])
 
         # single tag - returns all documents with that range
-        result = work.extract('mvmt-2')
+        result = work.list_sections('mvmt-2')
         self.assertEqual(len(result), 4)
 
         # multiple tags - returns the overlapping portion of all documents that have all tags
-        self.assertEqual(work.extract('vl-1', 'mvmt-2'), [('sel/beethoven/some_quartet/some_quartet_vl-1.pdf', 4, 8)])
+        self.assertEqual(work.list_sections('vl-1', 'mvmt-2'), [('sel/beethoven/some_quartet/some_quartet_vl-1.pdf', 4, 8)])
-        self.assertEqual(work.extract('vl-1', 'vl-2'), [])
+        self.assertEqual(work.list_sections('vl-1', 'vl-2'), [])

app/library/urls.py

@@ -16,7 +16,7 @@ urlpatterns = [
     path('projects/<int:project>/items/manage', views.ProjectItemManageView.as_view(), name="item_list_manage"),
     path('projects/<int:project>/items/append', views.ProjectItemAddView.as_view(), name="item_list_append"),
     
-    path('library', views.WorkListView.as_view(), name="work_list"),
+    path('library', views.LibraryWorkListView.as_view(), name="work_list"),
 
     path('collections', views.CollectionListView.as_view(), name="collection_list"),
     path('collections/<int:collection>', views.CollectionWorkListView.as_view(), name="collection_work_list"),
@@ -25,8 +25,10 @@ urlpatterns = [
     path('collections/<int:collection>/works/<int:pk>', views.WorkDetailView.as_view(), name="work_detail"),
     path('collections/<int:collection>/works/<int:pk>/edit', views.WorkUpdateView.as_view(), name="work_edit"),
     path('collections/<int:collection>/works/<int:pk>/partset', views.WorkPartSetView.as_view(), name="work_partset"),
+    path('collections/<int:collection>/works/<int:pk>/parts', views.WorkPartsView.as_view(), name="work_parts"),
     path('collections/<int:collection>/works/<int:pk>/add_to_project', views.WorkAddToProject.as_view(), name="work_add_to_project"),
     path('collections/<int:collection>/works/<int:pk>/upload', views.WorkAddDocumentView.as_view(), name="document_add"),
+    path('collections/<int:collection>/works/<int:pk>/download', views.WorkDownloadView.as_view(), name="work_download"),
 
     path('collections/<int:collection>/docs/<int:pk>/delete', views.DocumentDeleteView.as_view(), name="document_delete"),
     path('collections/<int:collection>/docs/<int:pk>/download', views.DocumentDownloadView.as_view(), name="document_download"),
@@ -35,8 +37,8 @@ urlpatterns = [
     path('collections/<int:collection>/download/<int:section>/<str:filename>', views.PartDownloadView.as_view(), name="part_download"),
 
     #path('api/', include(router.urls))
-    path('api/library/collections/<int:pk>/export', api.CollectionExportView.as_view(), name="collection_export"),
+    path('api/collections/<int:pk>', api.CollectionExportView.as_view(), name="collection_export"),
-    path('api/library/works/<int:pk>/export', api.WorkExportView.as_view(), name="work_export"),
+    path('api/collections/<int:collection>/works/<int:pk>', api.WorkExportView.as_view(), name="work_export"),
-    path('api/library/collections/<int:pk>/import', api.WorkImportView.as_view(), name="work_import"),
+    path('api/collections/<int:collection>/import', api.WorkImportView.as_view(), name="work_import"),
-    path('api/library/collections/<int:pk>/bulk_import', api.CollectionImportView.as_view(), name="collection_import"),
+    path('api/collections/<int:collection>/bulk_import', api.CollectionImportView.as_view(), name="collection_import"),
 ]

app/library/views/__init__.py

@@ -19,10 +19,12 @@ import re
 from interface.views import EnsembleMixin, ProjectMixin, AuthorizedResourceMixin
 from interface.models import Project
 from library.models import Collection, Work, Document, Section
-from library.music_tags import MUSIC_TAGS
+from library.music_tags import MUSIC_TAGS, MusicTag, auto_tag
 from library import forms, models
 from library.pdf_utils import extract_pages, extract_and_concat
 
+import logging
+logger = logging.getLogger(__name__)
 
 class ProjectItemListView(ProjectMixin, ListView):
     template_name = "library/item_list.html"
@@ -30,9 +32,7 @@ class ProjectItemListView(ProjectMixin, ListView):
 
     def post(self, request, **kwargs):
 
-        project = self.get_project()
+        project_works = self.project.works.all()
-
-        project_works = project.works.all()
 
         instruments = request.POST.getlist('instruments')
         works = request.POST.getlist('works')
@@ -58,7 +58,7 @@ class ProjectItemListView(ProjectMixin, ListView):
 
         result = extract_and_concat(sections)
 
-        download_name = f'{project.name}.pdf'
+        download_name = f'{self.project.name}.pdf'
 
         response = FileResponse(result, content_type="application/pdf")
         response['Content-Disposition'] = f'inline; filename="{download_name}"'
@@ -74,10 +74,6 @@ class ProjectItemListView(ProjectMixin, ListView):
         data['instrument'] = self.request.session.get('instrument', 'Score')
         data['part'] = self.request.session.get('part', '0')
         data['running_time'] = self.get_queryset().aggregate(Sum('work__running_time'))['work__running_time__sum']
-        #if running_time:
-        #    data['running_time'] = "{0:d}:{1:02d}".format(int(running_time / 60), running_time % 60)
-        #else:
-        #    data['running_time'] = "-:--"
         return data
 
 
@@ -117,6 +113,9 @@ class ProjectItemAddView(ProjectMixin, UpdateView):
 """ COLLECTION VIEWS """
 
 class CollectionMixin(AuthorizedResourceMixin):
+
+    collection = None
+
     def is_authorized(self):
         collection_id = self.kwargs['collection']
         self.collection = get_object_or_404(models.Collection, pk=collection_id)
@@ -147,7 +146,7 @@ class CollectionListView(ListView):
     paginate_by = 20
 
     def get_queryset(self):
-        collections = models.Collection.objects.all()
+        collections = models.Collection.objects.order_by('name')
 
         if self.request.user.is_anonymous:
             return models.Collection.objects.none()
@@ -160,18 +159,6 @@ class CollectionListView(ListView):
 class WorkListView(CollectionMixin, ListView):
     paginate_by = 20
         
-    def request_denied(self):
-        if 'auth' in self.request.GET:
-            if self.request.GET['auth'] != self.collection.auth():
-                raise SuspiciousOperation("Bad collection link")
-            self.add_authorized_key('collection', self.collection.pk, self.collection.nonce)
-            return HttpResponseRedirect(self.request.path)
-        return super().request_denied()
-
-    def get_works(self):
-        collections = CollectionMixin.get_queryset(self)
-        return  Work.objects.filter(collection__in=collections).select_related('collection')
-        
     def get_context_data(self, *args, **kwargs):
         data = super(WorkListView, self).get_context_data(*args, **kwargs)
         #data['title'] = f'Music available to {self.ensemble.name}' 
@@ -189,16 +176,37 @@ class WorkListView(CollectionMixin, ListView):
             else:
                 works = works.filter(Q(name__contains=q) | Q(composer__contains=q) | Q(meta_info__value__contains=q))
         
-        return works.order_by('name', 'composer', 'edition', 'pk')
+        return works.order_by('name', 'composer', 'edition', 'pk').distinct()
+
+class LibraryWorkListView(WorkListView):
+    
+    def is_authorized(self):
+        return True
+
+    def get_works(self):
+        collections = models.Collection.objects.all()
+        
+        if not self.request.user.is_superuser:
+            collections = collections.filter(administrators=self.request.user)
+
+        return  Work.objects.filter(collection__in=collections).select_related('collection')
 
 class CollectionWorkListView(WorkListView):
 
+    def request_denied(self):
+        if 'auth' in self.request.GET:
+            if self.request.GET['auth'] != self.collection.auth():
+                raise SuspiciousOperation("Bad collection link")
+            self.add_authorized_key('collection', self.collection.pk, self.collection.nonce)
+            return HttpResponseRedirect(self.request.path)
+        return super().request_denied()
+
     def get_works(self):
         works = Work.objects.filter(collection=self.kwargs['collection'])
 
-        if self.request.is_admin:
+        #if self.request.is_admin:
-            loan_count = Count('project_items', Q(project_items__checkout__lte=now(), project_items__returned=None))
+        #    loan_count = Count('project_items', Q(project_items__checkout__lte=now(), project_items__returned=None))
-            works = works.annotate(loan_count=loan_count)
+        #    works = works.annotate(loan_count=loan_count)
         return works
 
     def get_context_data(self, *args, **kwargs):
@@ -244,7 +252,7 @@ class WorkUpdateView(CollectionMixin, UpdateView):
     template_name = 'interface/default_form.html'
 
     def get_success_url(self):
-        return resolve_url('work_detail', self.kwargs['pk'])
+        return resolve_url('work_detail', self.collection.pk, self.kwargs['pk'])
 
 
 
@@ -259,16 +267,16 @@ class WorkAddToProject(CollectionMixin, FormView):
 
     def get_form(self):
         f = super(WorkAddToProject, self).get_form()
-        qs = f.fields['project'].queryset 
+        qs = f.fields['project'].queryset.select_related('ensemble') 
 
         # Limit to projects for ensembles where we are an admin and they haven't occured yet
-        qs = qs.filter(ensemble__admins=self.request.user, event_date__gt=now())
+        qs = qs.for_user(self.request.user).current()
 
         # dont show projects already added to
         work = self.get_object()
         qs = qs.exclude(pk__in=work.projects.all())
 
-        f.fields['project'].queryset = qs
+        f.fields['project'].queryset = qs.order_by('ensemble__name', 'name')
         return f
 
     def form_valid(self, form):
@@ -277,7 +285,9 @@ class WorkAddToProject(CollectionMixin, FormView):
         work.project_items.create(project=project, approved_by=self.request.user, checkout=now())
         return redirect('item_list', project=project.pk)
 
-    
+class WorkPartsView(CollectionMixin, DetailView):
+    model = models.Work
+    template_name = "library/work_parts_fragment.html" 
 
 class WorkPartSetView(CollectionMixin, DetailView):
     template_name = "library/work_partset.html"
@@ -294,7 +304,7 @@ class WorkPartSetView(CollectionMixin, DetailView):
         for i, tag in enumerate(parts):
             c = int(copies[i])
             if c > 0:
-                part = models.Section.objects.select_related('doc').get(tag=tag, doc__work=work)
+                for part in models.Section.objects.select_related('doc').filter(tag=tag, doc__work=work):
                     sections.append((part.doc.upload.path, part.name, part.start, part.end, c))
 
         result = extract_and_concat(sections)
@@ -312,6 +322,38 @@ class WorkPartSetView(CollectionMixin, DetailView):
             works = works.filter(collection__allowed_ensembles__ensemble=self.request.ensemble_id)
         return works
 
+class WorkDownloadView(CollectionMixin, SingleObjectMixin, View):
+    model = models.Work
+
+    def get(self, request, *args, **kwargs):
+        self.object = self.get_object()
+
+        tags = request.GET.getlist('tag')
+
+        if not tags:
+            raise Http404("No tags given")
+        
+        sections = list(self.object.tagged_sections(*tags))
+        print(sections)
+
+        if len(sections) == 0:
+            raise Http404("No matching sections")
+
+        if len(sections) == 1 and sections[0].start == 0:
+            # bypass extraction and redirect to the url
+            logger.debug("Redirecting to url")
+            return redirect(sections[0].upload.url)
+
+        result = extract_and_concat([ (s.upload.path, s.upload.name, s.start, s.end, 1) for s in sections ])
+
+        tag_names = " - ".join([ str(MusicTag.from_tag(tag)) for tag in tags ])
+
+        download_name = f'{self.object.name} - {tag_names}.pdf'
+
+        response = FileResponse(result, content_type="application/pdf")
+        response['Content-Disposition'] = f'inline; filename="{download_name}"'
+        return response
+
 class WorkAddDocumentView(CollectionMixin, CreateView):
     template_name = "interface/default_form.html"
     model = Document
@@ -327,23 +369,21 @@ class WorkAddDocumentView(CollectionMixin, CreateView):
         return super().form_invalid(form)
 
     def form_valid(self, form):
+        orig_name, ext = os.path.splitext(form.cleaned_data['upload'].name)
+        logger.info("Uploaded: %s", orig_name)
+
         doc = form.save(commit=False)
+        doc.doctype = models.Document.DOCTYPE_MAP.get(ext.lower(), models.Document.DOCTYPE_MISC)
         doc.work_id = self.kwargs['pk']
         doc.save()
 
         # auto tag the document
 
-        name, _ = os.path.splitext(os.path.basename(doc.upload.name))
+        #name, ext = os.path.splitext(os.path.basename(doc.upload.name))
-        parts = re.split(r'[^A-Za-z]+', name)
+        if doc.doctype == models.Document.DOCTYPE_PDF:
-        parts.reverse()
+            inst = auto_tag(orig_name)
-        for word in parts:
+            if inst:
-            try:
+                doc.sections.create(tag=inst.abbreviate())
-                tag = INSTRUMENT_TAGS[word.lower()] 
-                doc.sections.create(tag=tag)
-                break
-            except KeyError:
-                pass
-
 
         if self.request.headers['Accept'] == 'application/json':
             filename = os.path.basename(doc.upload.name)
@@ -395,7 +435,7 @@ class DocumentAnnotateView(DocumentMixin, DetailView):
             for tag, start, end in data:
                 #pages.sort()
                 #end = pages[-1] if len(pages) > 1 else None
-                o = self.object.sections.create(tag=tag, type=models.Section.TYPE_INSTRUMENT, start=start, end=end)
+                o = self.object.sections.create(tag=tag, start=start, end=end)
 
         return HttpResponse(status=204)
 
@@ -426,6 +466,9 @@ class PartDownloadView(CollectionMixin, SingleObjectMixin, View):
         self.args = args
         self.object = self.get_object()
 
+        if self.object.start is None:
+            return redirect(self.object.doc.upload.url)
+
         result = extract_pages(self.object.doc.upload.path, self.object.doc.work.name, self.object.start, self.object.end)
 
         #download_name = f'{self.object.doc.work.name}_{self.object.instrument}.pdf'
@@ -434,10 +477,6 @@ class PartDownloadView(CollectionMixin, SingleObjectMixin, View):
         response['Content-Disposition'] = f'inline; filename="{self.args["filename"]}"'
         return response
 
-    def get_queryset(self):
+    def get_object(self):
 
-        if self.request.is_admin:
+        return Section.objects.filter(doc__work__collection=self.collection).select_related('doc', 'doc__work').get(pk=self.kwargs['section'])
-            parts = Section.objects.all()
-        else:
-            parts = Section.objects.filter(doc__work__collection__allowed_ensembles__ensemble=self.request.ensemble_id)
-        return parts.select_related('doc', 'doc__work')

app/library/views/api.py

@@ -35,14 +35,16 @@ class WorkExportView(EnsembleMixin, WorkMixin, View):
 from interface.views import AuthorizedResourceMixin
 
 from rest_framework import routers, serializers, viewsets
+from rest_framework.exceptions import APIException
 
 from library.models import Collection, Work, Document, Section, WorkMeta
 
 
 import requests
 from io import BytesIO
-import tempfile
+import urllib
 import shutil
+import os.path
 
 from django.db import transaction
 from django.core.files.uploadedfile import TemporaryUploadedFile
@@ -67,25 +69,22 @@ class SectionSerializer(serializers.ModelSerializer):
     def to_representation(self, instance):
         start = instance.start or 0
         end = instance.end or 0
-        return f"{instance.tag}:{instance.type}:{start}:{end}"
+        return f"{instance.tag}:{start}:{end}"
 
     def to_internal_value(self, data):
-        tag, section_type, start, end = data.split(":")
+        tag, start, end = data.split(":")
-        try:
         start = int(start)
-        except:
-            start = 0
-        try:
         end = int(end)
-        except:
+        if start < 1:
-            end = 0
+            start = None
-        return super().to_internal_value({'tag': tag, 'type': int(section_type), 'start': start, 'end': end})
+        if end < 1:
+            end = None
+        return super().to_internal_value({'tag': tag, 'start': start, 'end': end})
 
 class DocumentSerializer(serializers.ModelSerializer):
 
     upload = serializers.URLField()
     sections = SectionSerializer(many=True)
-    #doctype = serializers.CharField(source='get_doctype_display')
 
     #def to_internal_value(self, data):
     #    r = requests.get(data['upload'], stream=True)
@@ -97,8 +96,7 @@ class DocumentSerializer(serializers.ModelSerializer):
 
     def to_representation(self, instance):
         data = super().to_representation(instance)
-        if data['upload'][0] == '/':
+        data['upload'] = instance.upload.url
-            data['upload'] = 'http://localhost:8000' + (data['upload'])
         return data
 
     def create(self, validated_data):
@@ -136,8 +134,13 @@ class WorkSerializer(serializers.ModelSerializer):
             for d in docs:
                 sections = d.pop('sections', [])
 
+                url = urllib.parse.urlparse(d['upload'])
+                filename = os.path.basename(url.path)
+
                 r = requests.get(d['upload'], stream=True)
-                f = TemporaryUploadedFile(d['upload'], r.headers['content-type'], r.headers['content-length'], r.encoding)
+                if r.status_code != 200:
+                    raise APIException("Failed to download file")
+                f = TemporaryUploadedFile(filename, r.headers['content-type'], r.headers.get('content-length'), r.encoding)
                 shutil.copyfileobj(r.raw, f.file)
                 r.close()
                 d['upload'] = f
@@ -171,19 +174,24 @@ class CollectionExportView(AuthorizedResourceMixin, generics.RetrieveAPIView):
     serializer_class = CollectionSerializer
 
     def get_queryset(self):
+        if self.request.user.is_superuser:
+            return Collection.objects.all()
         return Collection.objects.filter(administrators=self.request.user)
 
 class WorkExportView(AuthorizedResourceMixin, generics.RetrieveAPIView):
     serializer_class = WorkSerializer
 
     def get_queryset(self):
-        return Work.objects.filter(collection__administrators=self.request.user)
+        works = Work.objects.filter(collection=self.kwargs['collection'])
+        if self.request.user.is_superuser:
+            return works 
+        return works.filter(collection__administrators=self.request.user)
 
 class WorkImportView(AuthorizedResourceMixin, generics.CreateAPIView):
     serializer_class = WorkSerializer
 
     def perform_create(self, serializer):
-        serializer.save(collection_id=self.kwargs['pk'])
+        serializer.save(collection_id=self.kwargs['collection'])
 
 class CollectionImportView(AuthorizedResourceMixin, generics.CreateAPIView):
     serializer_class = CollectionSerializer