Tweeking section handling

Dockerfile

@@ -6,15 +6,16 @@ WORKDIR /root
 RUN python3 -m ensurepip
 RUN pip3 install -U pip --no-cache-dir
 
+COPY app/requirements.txt .
+RUN pip3 install -r requirements.txt --no-cache-dir
+
 COPY app /opt/polyphonic
 WORKDIR /opt/polyphonic
 
 COPY docker_settings.py polyphonic/local_settings.py
 
-RUN pip3 install -r requirements.txt --no-cache-dir
-
 RUN mkdir /var/polyphonic
 RUN SECRET_KEY=_ python3 manage.py collectstatic --noinput
 
 ENTRYPOINT ["python3", "manage.py"]
-CMD ["runserver", "0.0.0.0:8000", "--insecure"]
+CMD ["runserver", "0.0.0.0:8000", "--insecure"]

app/interface/templates/interface/project_base.html

@@ -43,7 +43,7 @@
 
                 <p class="menu-label">Admin</p>
                 <ul class="menu-list">
-                    {% if request.is_admin %}
+                    {% if request.is_admin or request.user.is_superuser %}
                     <li><a class="admin-link" href="{% url 'collection_list' %}">Collections</a></li>
                     {% endif %}
                     {% if request.user.is_superuser %}

app/library/admin.py

@@ -33,7 +33,7 @@ admin.site.register(models.Work, WorkAdmin)
 
 class SectionInline(admin.TabularInline):
     model = models.Section
-    fields = ['tag', 'start', 'end']
+    fields = ['type', 'tag', 'ordinal', 'start', 'end']
 
 class DocumentAdmin(admin.ModelAdmin):
     list_display = ['work', '__str__']

app/library/imslp.py

@@ -2,9 +2,12 @@
 from collections import namedtuple
 
 # taken from https://imslp.org/wiki/IMSLP:Abbreviations_for_Instruments
+# Place any extra abbreviations at the top
 
 ABBREVIATIONS = """
 score   Score
+cb      Double bass
+
 acc 	Accordion 
 afl 	Alto flute 
 alt 	Alto (voice) (contralto) 

app/library/migrations/0004_auto_20230101_1535.py

@@ -0,0 +1,40 @@
+# Generated by Django 3.2.7 on 2023-01-01 04:35
+
+import byostorage.cached
+import byostorage.user
+from django.db import migrations, models
+import library.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('library', '0003_auto_20221201_1540'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='section',
+            options={'ordering': ['doc', 'type', 'start', 'pk']},
+        ),
+        migrations.AddField(
+            model_name='section',
+            name='type',
+            field=models.SmallIntegerField(choices=[(1, 'Instrument'), (2, 'Movement')], default=1),
+            preserve_default=False,
+        ),
+        migrations.AlterField(
+            model_name='document',
+            name='upload',
+            field=models.FileField(storage=byostorage.cached.CachedStorage(byostorage.user.BYOStorage()), upload_to=library.models.doc_upload_filename),
+        ),
+        migrations.AlterField(
+            model_name='work',
+            name='original_parts',
+            field=models.JSONField(default=dict, help_text='Original printed parts (IMSLP format)'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='work',
+            unique_together=set(),
+        ),
+    ]

app/library/migrations/0005_auto_20230101_1547.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.2.7 on 2023-01-01 04:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('library', '0004_auto_20230101_1535'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='section',
+            options={'ordering': ['type', 'ordinal', 'doc', 'start', 'pk']},
+        ),
+        migrations.AddField(
+            model_name='section',
+            name='ordinal',
+            field=models.IntegerField(default=0),
+        ),
+    ]

app/library/models.py

@@ -7,9 +7,8 @@ from django.utils.functional import cached_property
 from django.core.files.storage import get_storage_class
 from django.db.models import Q, Count, Min, Max
 
-import os.path
-
 from byostorage.user import BYOStorage
+from byostorage.cached import CachedStorage
 from .imslp import Instrument
 
 import logging
@@ -24,26 +23,10 @@ logger = logging.getLogger(__name__)
 #    library_storage = get_storage_class()()
 #logger.info("Library storage: %s", library_storage.__class__.__name__)
 
+# FIXME: move back to settings
+library_storage = CachedStorage(BYOStorage())
 
 
-DOCTYPES = [
-    (1, 'PDF'),
-    (2, 'Audio'),
-    (3, 'Video'),
-    (4, 'Source'),
-]
-
-LICENCE_TYPES = [
-    (2, 'Public Domain'),
-    (4, 'Copyright Expired'),
-    (6, 'Copyrighted'),
-    (10, 'Internal use only'),
-]
-
-ACCESS_TYPES = [
-    (1, 'Unlimited'),
-    (2, 'Approval required'),
-]
 
 
 '''
@@ -105,7 +88,20 @@ class Collection(models.Model):
             help_text="User storage for documents")
     notes = models.TextField(blank=True, 
             help_text="Publicly visible notes about collection and loans policy (markdown format)")
+
+    def meta(self, name):
+        items = WorkMeta.objects.filter(work__collection=self.pk, name=name).values_list('value', flat=True).distinct()
+        return items
+
+    @property
+    def tags(self):
+        return self.meta('tag')
+
+    @property
+    def genres(self):
+        return self.meta('genre')    
     
+
     def __str__(self):
         return self.name
 
@@ -113,6 +109,15 @@ class EnsembleAccess(models.Model):
     """
     Can have different access levels to a collection
     """
+
+    ACCESS_UNLIMITED = 1
+    ACCESS_APPROVED = 2
+
+    ACCESS_TYPES = (
+        (ACCESS_UNLIMITED, 'Unlimited'),
+        (ACCESS_APPROVED, 'Approval required'),
+    )
+
     ensemble = models.ForeignKey('interface.Ensemble', on_delete=models.CASCADE, related_name="allowed_collections")
     collection = models.ForeignKey(Collection, on_delete=models.CASCADE, related_name="allowed_ensembles")
     access_type = models.PositiveSmallIntegerField(choices=ACCESS_TYPES, default=2)
@@ -120,19 +125,27 @@ class EnsembleAccess(models.Model):
     class Meta:
         verbose_name_plural = "Ensemble access"
 
-META_TAGS = (
-    ('tag', 'Tag'),
-    ('arr', 'Arranger'),
-    ('lyrics', 'Lyracist'),
-    ('genre', 'Genre'),
-    ('style', 'Style'),
-    ('orchestration', 'Orchestration'),
-)
 
 class Work(models.Model):
     """
     A musical work 'owned' by a collection from a licencing perspective.
     """
+    LICENCE_PUBLIC = 2
+    LICENCE_EXPIRED = 4
+    LICENCE_RECORDING = 5
+    LICENCE_PERFORMANCE = 6
+    LICENCE_PERUSAL = 8
+    LICENCE_NONE = 10
+
+    LICENCE_TYPES = (
+        (LICENCE_PUBLIC, 'Public Domain'),
+        (LICENCE_EXPIRED, 'Copyright Expired'),
+        (LICENCE_RECORDING, 'Recording Licence'),
+        (LICENCE_PERFORMANCE, 'Performance Licence'),
+        (LICENCE_PERUSAL, 'Perusal Licence'),
+        (LICENCE_NONE, 'Internal use only'),
+    )
+    
     name = models.CharField(max_length=255, help_text="Original name of the work")
     edition = models.CharField(max_length=255, blank=True,
             help_text="Edition details to distinguish multiple versions")
@@ -141,7 +154,7 @@ class Work(models.Model):
     composer = models.CharField(max_length=255, default='Anon',
             help_text="Surname, Initials")
 
-    original_parts = models.JSONField(default=dict, blank=True, help_text="Original printed parts (IMSLP format)")
+    original_parts = models.JSONField(default=dict, help_text="Original printed parts (IMSLP format)")
 
     # Collection details
     collection = models.ForeignKey(Collection, on_delete=models.CASCADE, related_name="works")
@@ -235,8 +248,18 @@ class Work(models.Model):
         return f"{self.name} ({self.composer})"
 
 class WorkMeta(models.Model):
+
+    META_CHOICES = (
+        ('tag', 'Tag'),
+        ('arr', 'Arranger'),
+        ('lyrics', 'Lyracist'),
+        ('genre', 'Genre'),
+        ('style', 'Style'),
+        ('orchestration', 'Orchestration'),
+    )
+    
     work = models.ForeignKey(Work, on_delete=models.CASCADE, related_name='meta_info')
-    name = models.SlugField(max_length=20, choices=META_TAGS)
+    name = models.SlugField(max_length=20, choices=META_CHOICES)
     value = models.CharField(max_length=255)
 
 def doc_upload_filename(doc, filename):
@@ -250,9 +273,22 @@ class Document(models.Model):
     """
     Document represents a single file stored in the storage backend.
     """
+
+    DOCTYPE_PDF = 1
+    DOCTYPE_AUDIO = 2
+    DOCTYPE_VIDEO = 3
+    DOCTYPE_SOURCE = 4
+
+    DOCTYPES = (
+        (DOCTYPE_PDF, 'PDF'),
+        (DOCTYPE_AUDIO, 'Audio'),
+        (DOCTYPE_VIDEO, 'Video'),
+        (DOCTYPE_SOURCE, 'Source'),
+    )
+
     work = models.ForeignKey('Work', on_delete=models.CASCADE, related_name="docs")
     doctype = models.PositiveSmallIntegerField(choices=DOCTYPES, default=1)
-    upload = models.FileField(upload_to=doc_upload_filename, storage=BYOStorage())
+    upload = models.FileField(upload_to=doc_upload_filename, storage=library_storage)
     created = models.DateTimeField(auto_now_add=True)
     version = models.CharField(max_length=30, blank=True)
 
@@ -267,21 +303,54 @@ class Section(models.Model):
     """
     Section is a tagged portion of a Document
     """
+
+    TYPE_INSTRUMENT = 1
+    TYPE_MOVEMENT = 2
+    TYPE_EXCERPT = 3
+
+    SECTION_TYPES = (
+        (TYPE_INSTRUMENT, "Instrument"),
+        (TYPE_MOVEMENT, "Movement"),
+        (TYPE_EXCERPT, "Excerpt"),
+    )
+
+    SECTION_CLASSES = {
+        TYPE_INSTRUMENT: 'info',
+        TYPE_MOVEMENT: 'success',
+        TYPE_EXCERPT: 'warning',
+    }
+
+
+    type = models.SmallIntegerField(choices=SECTION_TYPES)
     doc = models.ForeignKey(Document, on_delete=models.CASCADE, related_name="sections")
-    tag = models.CharField(max_length=50)
+    tag = models.CharField(max_length=50, blank=True)
+    ordinal = models.IntegerField(default=0)
     start = models.SmallIntegerField(null=True, blank=True)
     end = models.SmallIntegerField(null=True, blank=True)
 
     class Meta:
-        ordering = ['doc', 'start', 'pk']
+        ordering = ['type', 'ordinal', 'doc', 'start', 'pk']
 
     @property
-    def instrument(self):
+    def name(self):
-        return Instrument.from_tag(self.tag)
+        if self.type == self.TYPE_INSTRUMENT:
+            instr = Instrument.from_tag(self.tag)
+            if self.ordinal:
+                return f'{instr} {self.ordinal}'
+            return str(instr)
+        return f"{self.ordinal} - {self.tag}"
+
+    #@property
+    #def instrument(self):
+    #    return Instrument.from_tag(self.tag)
+
+    @property
+    def bulma_class(self):
+        return self.SECTION_CLASSES[self.type]
 
     @property
     def filename(self):
-        return slugify(f'{self.doc.work.name}_{self.instrument}') + '.pdf'
+        return slugify(f'{self.doc.work.name} - {self.name}') + '.pdf'
 
     @property 
     def pagerange(self):
@@ -292,4 +361,4 @@ class Section(models.Model):
         return "all"
 
     def __str__(self):
-        return f'{self.doc.upload} [{self.pagerange}]'
+        return self.name

app/library/templates/library/collection_list.html

@@ -1,7 +1,7 @@
 {% extends "interface/project_base.html" %}
 
 {% block page %}
-<h3 class="title">Library collections for {{ request.user }}</h3>
+<h3 class="title">Library collections for {% firstof request.user.first_name request.user.username %}</h3>
 
 <div class="columns is-multiline">
 {% for collection in object_list %}
@@ -17,6 +17,14 @@
                     {% if collection.location %}{{ collection.location }},{% endif %}
                     {{ collection.works.count }} items.
                 </p>
+                <p>
+                    {% for tag in collection.tags %}
+                    <a href="{% url 'collection_work_list' collection.pk %}?filter=tag:{{ tag }}" class="tag is-success">{{ tag }}</a>
+                    {% endfor %}
+                    {% for genre in collection.genres %}
+                    <a href="{% url 'collection_work_list' collection.pk %}?filter=genre:{{ genre }}" class="tag is-warning">{{ genre }}</a>
+                    {% endfor %}
+                </p>
             </div>
         </div>
     </div>

app/library/templates/library/document_entry.html

@@ -1,13 +1,13 @@
 {% load path_filters %}
 <tr>
-    <td><a href="{% url 'document_download' pk=doc.pk %}" target="_blank">
+    <td><a href="{{ doc.upload.url }}" target="_blank">
             {{ doc.upload.name|basename }}</a></td>
     <td>
-        {% for part in doc.sections.all %}
+        {% for section in doc.sections.all %}
-            <a class="tag is-info" target="_blank" href="{% url 'part_download' pk=part.pk filename=part.filename %}">{{ part.instrument }}</a>
+            <a class="tag is-{{ section.bulma_class }}" target="_blank" href="{% url 'part_download' pk=section.pk filename=section.filename %}">{{ section.name }}</a>
         {% endfor %}
     </td>
-    <td class="has-text-right">
+    <td class="has-text-right" style="white-space: nowrap;">
     {% if request.is_admin %}
         <a href="{% url 'document_annotate' pk=doc.pk %}"><i class="fas fa-tags"
                 title="Manage Tags"></i></a>

app/library/templates/library/work_detail.html

@@ -21,7 +21,7 @@
 <h3 class="title">
     {{ work.name }}
     {% for tag in work.tags %}
-        <span class="tag is-success">{{ tag }}</span>
+        <a href="{% url 'collection_work_list' work.collection.pk %}?filter=tag:{{ tag }}" class="tag is-success">{{ tag }}</a>
     {% endfor %}
 </h3>
 <p class="subtitle">{% firstof work.composer "Unattributed" %}{% if work.edition %} - {{ work.edition }}{% endif %}</p>
@@ -30,10 +30,10 @@
 
     <p class="block">
     Location: <a href="{% url 'collection_work_list' work.collection.pk %}">{{ work.collection }}</a> [{{ work.identifier }}]<br/>
-    Running time: {{ work.duration }}<br/>
+    Running time: {% firstof work.duration 'Unknown' %}<br/>
     Licence: {{ work.get_licence_display }}<br/>
     {% for meta in work.meta %}
-    {{ meta.get_name_display }}: {{ meta.value }}<br/>
+    {{ meta.get_name_display }}: <a href="{% url 'collection_work_list' work.collection.pk %}?filter={{ meta.name}}:{{ meta.value }}">{{ meta.value }}</a><br/>
     {% endfor %}
     </p>
 
@@ -79,9 +79,9 @@
                     {% if work.digital_parts %}
                     <a class="tag is-danger" href="{% url 'work_partset' pk=work.pk %}">Full Set</a>
                     {% endif %}
-                    {% for part in work.digital_parts %}
+                    {% for section in work.digital_parts %}
-                    <a class="tag is-info" href="{% url 'part_download' pk=part.pk filename=part.filename %}"
+                    <a class="tag is-info" href="{% url 'part_download' pk=section.pk filename=section.filename %}"
-                        target="part_{{ part.pk }}" rel="">{{ part.instrument }}</a>
+                        target="section_{{ section.pk }}" rel="">{{ section.name }}</a>
                     {% empty %}
                     <p class="is-italic">No digital parts available</p>
                     {% endfor %}

app/library/urls.py

@@ -1,8 +1,15 @@
-from django.urls import path
+from django.urls import path, include
 from django.contrib.auth import views as auth_views
+from rest_framework import routers
 
 from . import views
 
+from library.views import api
+
+#router = routers.DefaultRouter()
+#router.register(r'collection', external.CollectionViewSet, basename="collection")
+#router.register(r'work', external.WorkViewSet, basename="work")
+
 urlpatterns = [
     
     path('projects/<int:project>/items', views.ProjectItemListView.as_view(), name="item_list"),
@@ -11,7 +18,7 @@ urlpatterns = [
 
     path('library/collections', views.CollectionListView.as_view(), name="collection_list"),
     path('library/collections/<int:pk>', views.CollectionWorkListView.as_view(), name="collection_work_list"),
-    path('library/collection/<int:pk>/create', views.WorkAddView.as_view(), name="work_add"),
+    path('library/collections/<int:pk>/create', views.WorkAddView.as_view(), name="work_add"),
     
     path('library/works', views.WorkListView.as_view(), name="work_list"),
     path('library/works/<int:pk>', views.WorkDetailView.as_view(), name="work_detail"),
@@ -24,4 +31,9 @@ urlpatterns = [
     path('library/documents/<int:pk>/download', views.DocumentDownloadView.as_view(), name="document_download"),
     path('library/documents/<int:pk>/annotate', views.DocumentAnnotateView.as_view(), name="document_annotate"),
     path('library/parts/<int:pk>/<str:filename>', views.PartDownloadView.as_view(), name="part_download"),
+
+    #path('api/', include(router.urls))
+    path('api/library/collections/<int:pk>/export', api.CollectionExportView.as_view(), name="collection_export"),
+    path('api/library/works/<int:pk>/export', api.WorkExportView.as_view(), name="work_export"),
+    path('api/library/collections/<int:pk>/import', api.WorkImportView.as_view(), name="work_import"),
 ]

app/library/views/__init__.py

@@ -15,10 +15,10 @@ import re
 
 from interface.views import EnsembleMixin, ProjectMixin
 from interface.models import Project
-from .models import Collection, Work, Document, Section
+from library.models import Collection, Work, Document, Section
-from .imslp import INSTRUMENT_TAGS, INSTRUMENTS
+from library.imslp import INSTRUMENT_TAGS, INSTRUMENTS
-from . import forms, models
+from library import forms, models
-from .pdf_utils import extract_pages, extract_and_concat
+from library.pdf_utils import extract_pages, extract_and_concat
 
 class ProjectItemListView(ProjectMixin, ListView):
     template_name = "library/item_list.html"
@@ -364,10 +364,10 @@ class PartDownloadView(EnsembleMixin, SingleObjectMixin, View):
 
         result = extract_pages(self.object.doc.upload.path, self.object.doc.work.name, self.object.start, self.object.end)
 
-        download_name = f'{self.object.doc.work.name}_{self.object.instrument}.pdf'
+        #download_name = f'{self.object.doc.work.name}_{self.object.instrument}.pdf'
 
         response = FileResponse(result, content_type="application/pdf")
-        response['Content-Disposition'] = f'inline; filename="foo.pdf"'
+        response['Content-Disposition'] = f'inline; filename="{self.args["filename"]}"'
         return response
 
     def get_queryset(self):

app/library/views/api.py

@@ -0,0 +1,161 @@
+"""
+Views relating to importing and exporting collection items
+"""
+"""
+from interface.views import EnsembleMixin
+from library.views import WorkMixin
+from django.views.generic import View
+from django.http import JsonResponse
+
+from djantic import ModelSchema
+from library.models import Work, Document, Section
+
+class DocumentSchema(ModelSchema):
+    class Config:
+        model = Document
+
+
+class WorkSchema(ModelSchema):
+
+    docs: DocumentSchema
+
+    class Config:
+        model = Work
+        exclude = ['licence']
+
+class WorkExportView(EnsembleMixin, WorkMixin, View):
+    
+    def get(self, request, *args, **kwargs):
+        obj = self.get_queryset().get(pk=kwargs['pk'])
+        schema = WorkSchema.from_orm(obj)
+        return JsonResponse(schema.dict())
+
+"""
+
+from library.views import WorkMixin
+from interface.views import EnsembleMixin
+
+from rest_framework import routers, serializers, viewsets
+
+from library.models import Collection, Work, Document, Section
+
+
+import requests
+from io import BytesIO
+import tempfile
+import shutil
+
+from django.db import transaction
+from django.core.files.uploadedfile import TemporaryUploadedFile
+
+
+class SectionSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = Section
+        exclude = ['id', 'doc']
+
+    def to_representation(self, instance):
+        start = instance.start or 0
+        end = instance.end or 0
+        return f"{instance.tag}:{instance.type}:{start}:{end}"
+
+    def to_internal_value(self, data):
+        tag, section_type, start, end = data.split(":")
+        try:
+            start = int(start)
+        except:
+            start = 0
+        try:
+            end = int(end)
+        except:
+            end = 0
+        return super().to_internal_value({'tag': tag, 'type': int(section_type), 'start': start, 'end': end})
+
+class DocumentSerializer(serializers.ModelSerializer):
+
+    upload = serializers.URLField()
+    sections = SectionSerializer(many=True)
+    #doctype = serializers.CharField(source='get_doctype_display')
+
+    #def to_internal_value(self, data):
+    #    r = requests.get(data['upload'], stream=True)
+    #    with tempfile.NamedTemporaryFile('wb') as f:
+    #        shutil.copyfileobj(r.raw, f)
+    #        data['upload'] = f.name
+    #        print(repr(data))
+    #        return super().to_internal_value(data)
+
+    def to_representation(self, instance):
+        data = super().to_representation(instance)
+        if data['upload'][0] == '/':
+            data['upload'] = 'http://localhost:8000' + (data['upload'])
+        return data
+
+    def create(self, validated_data):
+        print("CREATE", validated_data)
+        return super().create(validated_data)
+
+    def validate(self, data):
+        print("VALIDATE", data)
+        return super().validate(data)
+
+    def validate_upload(self, value):
+        print("VALIDATE", value)
+        return value
+
+    class Meta:
+        model = Document
+        exclude = ["id", "work", "version", "created"]
+
+# Serializers define the API representation.
+class WorkSerializer(serializers.ModelSerializer):
+
+    docs = DocumentSerializer(many=True)
+    
+    class Meta:
+        model = Work
+        exclude = ['id', 'collection', 'projects', 'parent']
+
+    def create(self, validated):
+        with transaction.atomic():
+            docs = validated.pop('docs', [])
+            work = Work.objects.create(**validated)
+
+            for d in docs:
+                sections = d.pop('sections', [])
+
+                r = requests.get(d['upload'], stream=True)
+                f = TemporaryUploadedFile(d['upload'], r.headers['content-type'], r.headers['content-length'], r.encoding)
+                shutil.copyfileobj(r.raw, f.file)
+                r.close()
+                d['upload'] = f
+                doc = Document.objects.create(work_id=work.pk, **d)
+
+                for s in sections:
+                    Section.objects.create(doc_id=doc.pk, **s)
+
+            return work
+
+class CollectionSerializer(serializers.Serializer):
+    works = WorkSerializer(many=True)
+
+from rest_framework import generics
+
+class CollectionExportView(generics.RetrieveAPIView):
+    serializer_class = CollectionSerializer
+
+    def get_queryset(self):
+        return Collection.objects.filter(administrators=self.request.user)
+
+class WorkExportView(generics.RetrieveAPIView):
+    serializer_class = WorkSerializer
+
+    def get_queryset(self):
+        return Work.objects.filter(collection__administrators=self.request.user)
+
+class WorkImportView(generics.CreateAPIView):
+    serializer_class = WorkSerializer
+
+    def perform_create(self, serializer):
+        serializer.save(collection_id=self.kwargs['pk'])
+

app/polyphonic/default_settings.py

@@ -41,6 +41,7 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'django_markdown2',
+    'rest_framework',
     'crispy_forms',
     'crispy_bulma',
     'byostorage',