Author SHA1 Message Date
tris 0bf7841345 Adding icons to sync buttons 2026-06-19 22:35:42 +10:00
tris 965573c03b Merged from master 2026-06-19 22:29:44 +10:00
tris b5d8d2d775 Basic web sync working 2026-06-19 22:27:03 +10:00
tris 0e69cdeca4 Tweeked deployment (again!) 2026-06-19 14:03:50 +10:00
tris e797876c16 Merge pull request 'Material Icons' (#17) from material into master
Reviewed-on: #17
2026-06-19 13:58:45 +10:00
tris 1301d19c08 Switched to material icons 2026-06-16 09:18:21 +10:00
tris ca3effcad1 Fixed my music page 2026-05-29 22:47:33 +10:00
tris 5468f6d3e7 Added handling of gdrive resource_keys - fixes #16 2026-05-27 22:46:08 +10:00
tris e46d8145a7 Install from git 2026-05-27 09:35:15 +10:00
tris 3444cdbc59 Merge pull request 'gdrive' (#15) from gdrive into master
Reviewed-on: #15
2026-05-24 16:45:13 +10:00
tris 6e77474d15 Couple of fixes 2026-05-24 16:42:51 +10:00
tris 72bad08a61 Merge branch 'master' into gdrive 2026-05-24 14:24:34 +10:00
tris b217fbea5e Sorted out linting 2026-05-24 14:23:59 +10:00
tris cec64ecd2f Working with manual sync 2026-05-24 13:51:23 +10:00
tris d30005d5b6 Indexing bug fix 2026-05-24 11:48:42 +10:00
tris ef981e06e8 Shared gdrive access 2026-05-24 11:35:18 +10:00
tris 4cef5800bc Merge pull request 'Repackaged as a single project' (#13) from repackage into master
Reviewed-on: #13
2026-05-24 11:11:27 +10:00
tris 78441dc142 Version bump 2026-05-24 11:07:49 +10:00
tris c96a2ad80b Refactored docker 2026-05-24 11:00:05 +10:00
tris 7e0c77f260 Refactored to polyphonic package 2026-05-24 10:51:43 +10:00
tris ff17114514 Minor test change 2026-05-24 10:07:05 +10:00
tris 533d6f09e8 Fixed compose launch 2026-05-23 22:10:37 +10:00
tris eeb35ce4f6 Modify PYTHONPATH 2026-05-23 21:46:51 +10:00
tris 947626c2af Added indexer tests 2026-05-23 11:38:40 +10:00
tris c1f0e48f80 Another go at sorting the settings mess 2026-05-23 10:00:09 +10:00
tris ab7d32d46e Renamed to poly-tool 2026-05-23 00:05:29 +10:00
tris 5116247ae8 Simplified settings 2026-05-22 23:51:25 +10:00
tris 93c4926dfd Updated docker build 2026-05-22 14:11:03 +10:00
tris 27d1b03c3c Added fuzzy matching 2026-05-14 11:02:55 +10:00
tris 4e102c07ac Skip migrations 2026-05-13 11:08:03 +10:00
121 changed files with 875 additions and 492 deletions
+3
View File
@@ -5,7 +5,10 @@ __pycache__
credentials.json
credentials
local_settings.py
local.mk
.coverage
.lint
.deploy
Session.vim
poetry.lock
/env
+16 -12
View File
@@ -1,21 +1,25 @@
FROM alpine:3.14
FROM alpine:latest
RUN apk add --no-cache python3 git ghostscript sqlite
ENV TARGET=/opt/polyphonic
ENV RELEASE=polyphonic-0.8.4-py3-none-any.whl
#ENV RELEASE=git+https://gitea.tfconsulting.com.au/projects/polyphonic.git
RUN apk add --no-cache python3 py3-pip git ghostscript sqlite
WORKDIR /root
RUN python3 -m ensurepip
RUN pip3 install -U pip --no-cache-dir
COPY app/requirements.txt .
RUN pip3 install -r requirements.txt --no-cache-dir
RUN python3 -m venv ${TARGET}
ENV PATH="${TARGET}/bin:$PATH"
COPY app /opt/polyphonic
WORKDIR /opt/polyphonic
COPY dist/${RELEASE} .
RUN pip3 install ${RELEASE} --no-cache-dir
RUN pip3 install gunicorn whitenoise
COPY docker_settings.py polyphonic/local_settings.py
RUN SECRET_KEY=_ python3 manage.py collectstatic --noinput
WORKDIR ${TARGET}
RUN SECRET_KEY=_ poly-tool collectstatic --noinput
VOLUME ["/var/polyphonic"]
EXPOSE 8000/tcp
ENTRYPOINT ["python3", "manage.py"]
CMD ["runserver", "0.0.0.0:8000", "--insecure"]
CMD ["gunicorn", "-b", "0.0.0.0", "polyphonic.config.wsgi"]
+22 -5
View File
@@ -1,15 +1,32 @@
PYTHON=env/bin/python
DROPZONE=5.7.0
test: check
poetry run coverage run --include "app/*" --omit "*/migrations/*" app/manage.py test app
VERSION=0.8.4
export DJANGO_SETTINGS_MODULE=polyphonic.config.settings.dev
-include local.mk
test: .coverage
check: .lint
pre-commit: check test
.coverage: polyphonic
poetry run coverage run --include "polyphonic/*" --omit "*/migrations/*" polyphonic/manage.py test polyphonic
poetry run coverage html
poetry run coverage report
check:
poetry run ruff check app
.lint: polyphonic
poetry run ruff check polyphonic
poetry run ruff format --check polyphonic
touch $@
build:
build: dist/polyphonic-${VERSION}-py3-none-any.whl
dist/polyphonic-${VERSION}-py3-none-any.whl: polyphonic
poetry build
dev-setup:
-18
View File
@@ -1,18 +0,0 @@
from django import template
from django.utils import timesince
register = template.Library()
def roughtimesince(value):
return timesince.timesince(value, depth=1)
register.filter("roughtimesince", roughtimesince)
def roughtimeuntil(value):
return timesince.timeuntil(value, depth=1)
register.filter("roughtimeuntil", roughtimeuntil)
-75
View File
@@ -1,75 +0,0 @@
from library.models import Collection, Work, WorkMeta, Document
import logging
logger = logging.getLogger(__name__)
def sync_work(work: Work):
logger.info("Syncing '%s'", work.name)
folder_id = work.meta_info.get(name="folderid").value
storage = work.collection.storage.instance()
prefix = work.collection.storage.name
_, files = storage.listdir(folder_id)
existing = set(
[
storage.parse_id(x.partition(":")[2])
for x in work.docs.values_list("upload", flat=True)
]
)
logger.debug("%d existing documents", len(existing))
for file in files:
if file.id in existing:
logger.debug("%30s: Skipping existing (%s)", file.name, file.id)
existing.discard(file.id)
continue
if not file.name.lower().endswith(".pdf"):
logger.debug("%40s: Not a PDF", file.name)
continue
logger.info("%40s: Adding", file.name)
doc = work.docs.create(upload=f"{prefix}:{file}", doctype=Document.DOCTYPE_PDF)
doc.auto_tag()
for uri in existing:
logger.warning("Local entry not in folder: %s", uri)
def sync_collection(collection: Collection, sync_existing: bool = False):
logger.info("Syncing '%s'", collection)
if not collection.storage.storage.endswith("GDriveLinkStorage"):
raise RuntimeError("Not a gdrive storage")
if not collection.prefix:
raise KeyError("Prefix must store folder id")
existing = dict(
WorkMeta.objects.filter(
work__collection=collection, name="folderid"
).values_list("value", "work_id")
)
storage = collection.storage.instance()
folders, _ = storage.listdir(collection.prefix)
for folder in folders:
if folder.id in existing:
if sync_existing:
logger.info("%40s: Syncing (%s)", folder.name, folder.id[:12])
sync_work(Work.objects.get(pk=existing[folder.id]))
del existing[folder.id]
continue
logger.info("%40s: Adding", folder.name)
work = Work(name=folder.name, collection=collection)
work.save()
work.meta_info.create(name="folderid", value=folder.id)
sync_work(work)
for folderid, work in existing:
logger.warning("Folder for work %d no longer in drive (%s)", work, folderid)
-120
View File
@@ -1,120 +0,0 @@
from django.core.files.storage import Storage
from collections import namedtuple
import requests
import re
from gzip import GzipFile
import logging
logger = logging.getLogger(__name__)
SHARED_FOLDER = re.compile(r"https://drive.google.com/drive/folders/(\w+)")
SHARED_FILE = re.compile(r"https://drive.google.com/file/d/([\w\-]+)")
FILES_API = "https://www.googleapis.com/drive/v3/files"
class DriveObject(namedtuple("DriveObject", ("id", "name"))):
@classmethod
def from_string(cls, s):
return cls(*s.split("/", 1))
def __str__(self):
return f"{self.id}/{self.name}"
class GDriveLinkStorage(Storage):
is_writable = False
def __init__(self, api_key):
self.api_key = api_key
super().__init__()
def parse_id(self, name):
parts = name.split("/")
return parts[0]
def extract_id(self, url, *patterns):
logger.debug("EXTRACT_ID: %r", url)
for pattern in patterns:
match = pattern.match(url)
if match:
return match.groups()[0]
raise FileNotFoundError(f"Not a valid url: {url}")
def get_json(self, url):
logger.debug("GET_JSON: %s", url)
response = requests.get(url)
data = response.json()
logger.debug("Data: %r", data)
return data
def listdir(self, path) -> tuple[list[str], list[str]]:
# used to test for valid connection parameters - should do something to validate API key here
if path == "":
return [], []
logger.debug("LISTDIR: %s", path)
folder_id = self.parse_id(path)
url = f"{FILES_API}?q='{folder_id}'+in+parents&key={self.api_key}"
data = self.get_json(url)
files = []
folders = []
for x in data["files"]:
if x["mimeType"] == "application/vnd.google-apps.folder":
# folders.append(f"{x['id']}/{x['name']}")
folders.append(DriveObject(x["id"], x["name"]))
else:
# files.append(f"{x['id']}/{x['name']}")
files.append(DriveObject(x["id"], x["name"]))
return folders, files
def get_meta(self, name):
file_id = self.parse_id(name)
url = f"{FILES_API}/{file_id}?key={self.api_key}"
return self.get_json(url)
def open(self, name, mode="rb"):
file_id = self.parse_id(name)
url = f"{FILES_API}/{file_id}?alt=media&key={self.api_key}"
logger.info("URL: %s", url)
response = requests.get(url, stream=True)
return GzipFile(name, "rb", 9, response.raw)
def size(self, name):
raise NotImplementedError()
def delete(self, name):
pass
def url(self, name):
logger.debug("URL: %r", name)
file_id = self.parse_id(name)
return f"https://drive.usercontent.google.com/download?export=download&id={file_id}&confirm=yes"
def get_folder_id(self, url):
try:
return self.extract_id(url, SHARED_FOLDER)
except FileNotFoundError:
return None
def get_file_id(self, url):
try:
return self.extract_id(url, SHARED_FILE)
except FileNotFoundError:
return None
"""
def import_link(self, url) -> str:
file_id = self.extract_id(url, SHARED_FILE)
meta = self.get_meta(file_id)
return f"{file_id}/{meta['name']}"
def folder_import(self, url) -> list[str]:
folder_id = self.extract_id(url, SHARED_FOLDER)
_, files = self.listdir(folder_id)
return files
"""
-41
View File
@@ -1,41 +0,0 @@
{% extends "interface/project_base.html" %}
{% block admin %}
<a href="{% url 'work_detail' collection=collection.pk pk=object.pk %}" class="button is-link is-light">
<span>Back to work</span>
</a>
{% endblock %}
{% block page %}
<h3 class="subtitle"><a href="{% url 'work_detail' collection.pk object.pk %}">{{ object.name }}</a></h3>
<div class="m-3">
<p>This page lets you link a work to a google drive folder. You can either paste a public link to a folder to enable syncing or a file to add individually</p>
</div>
<div class="m-3">
<p>
{% if meta.folderid %}
This work is currently linked to <b>{{ meta.folderid }}</b>. Pasting a new folder link will overwrite this.
{% else %}
There is currently no shared drive folder linked to this work - paste one here to enable syncing.
{% endif %}
</p>
</div>
<div>
<form method="post">
<div class="field">
<div class="control">
<input name="link" class="input is-expanded" type="text" placeholder="Shared link">
</div>
{% for error in form.errors.link %}
<p class="help is-danger">{{ error }}</p>
{% endfor %}
</div>
<div class="field">
<div class="control">
<button class="button is-info" type="submit">Add</button>
</div>
</div>
{% csrf_token %}
</form>
</div>
{% endblock %}
-6
View File
@@ -1,6 +0,0 @@
try:
from .local_settings import * # noqa
except ImportError:
from .default_settings import * # noqa
INSTALLED_APPS += POLYPHONIC_MODULES # type: ignore #noqa
+14
View File
@@ -0,0 +1,14 @@
services:
polyphonic:
image: "polyphonic:0.8.4"
build: "."
ports:
- "8001:8000"
volumes:
- "./data:/var/polyphonic"
- "./local_settings.py:/opt/polyphonic/local_settings.py"
env_file: "compose.env"
environment:
DJANGO_SETTINGS_MODULE: local_settings
PYTHONPATH: /opt/polyphonic
WORK_DIR: /var/polyphonic
-8
View File
@@ -1,8 +0,0 @@
from .default_settings import *
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': '/var/polyphonic/db.sqlite3',
}
}
@@ -11,6 +11,6 @@ import os
from django.core.asgi import get_asgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'polyphonic.settings')
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "polyphonic.settings")
application = get_asgi_application()
@@ -11,26 +11,29 @@ https://docs.djangoproject.com/en/3.1/ref/settings/
"""
from pathlib import Path
from os import environ
import os
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
BASE_DIR = Path(__file__).resolve().parent.parent.parent.parent
SECRET_KEY = environ.get("SECRET_KEY")
# A place to put things
WORK_DIR = os.environ.get("WORK_DIR") or os.path.join(BASE_DIR, "data")
# Will fail to start if not defined
SECRET_KEY = os.environ.get("SECRET_KEY")
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = False
ALLOWED_HOSTS = ["localhost"]
INTERNAL_IPS = ["127.0.0.1"]
ALLOWED_HOSTS = ["localhost", "127.0.0.1"]
# Application definition
POLYPHONIC_MODULES = ["library"]
POLYPHONIC_MODULES = ["polyphonic.library"]
INSTALLED_APPS = [
"django.contrib.admin",
@@ -44,9 +47,11 @@ INSTALLED_APPS = [
"crispy_forms",
"crispy_bulma",
"byostorage",
"interface",
"polyphonic.interface",
]
INSTALLED_APPS += POLYPHONIC_MODULES
CRISPY_ALLOWED_TEMPLATE_PACKS = ("bulma",)
CRISPY_TEMPLATE_PACK = "bulma"
@@ -60,7 +65,7 @@ MIDDLEWARE = [
"django.middleware.clickjacking.XFrameOptionsMiddleware",
]
ROOT_URLCONF = "polyphonic.urls"
ROOT_URLCONF = "polyphonic.config.urls"
TEMPLATES = [
{
@@ -78,7 +83,7 @@ TEMPLATES = [
},
]
WSGI_APPLICATION = "polyphonic.wsgi.application"
WSGI_APPLICATION = "polyphonic.config.wsgi.application"
# Database
@@ -87,7 +92,7 @@ WSGI_APPLICATION = "polyphonic.wsgi.application"
DATABASES = {
"default": {
"ENGINE": "django.db.backends.sqlite3",
"NAME": BASE_DIR / "db.sqlite3",
"NAME": os.path.join(WORK_DIR, "db.sqlite3"),
}
}
@@ -120,17 +125,18 @@ LANGUAGE_CODE = "en-us"
TIME_ZONE = "Australia/Melbourne"
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.1/howto/static-files/
STATIC_URL = "/static/"
STATIC_ROOT = "static"
# Need to set this
AWS_BUCKET = ""
# Library settings
CACHED_STORAGE_REMOTE = "byostorage.user.BYOStorage"
CACHED_STORAGE_DIR = os.path.join(WORK_DIR, "cache")
WHOOSH_INDEX = os.path.join(WORK_DIR, "index")
STORAGE_CLASSES = ["polyphonic.library.gdrive.storage.GDriveLinkStorage"]
+28
View File
@@ -0,0 +1,28 @@
from .base import * # noqa
from os import environ
DEBUG = True
SECRET_KEY = "DO NOT USE IN PRODUCTION"
# Enable debug toolbar
INSTALLED_APPS.append("debug_toolbar") # noqa
MIDDLEWARE.insert(1, "debug_toolbar.middleware.DebugToolbarMiddleware") # noqa
INTERNAL_IPS = ["127.0.0.1"]
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"handlers": {
"console": {
"class": "logging.StreamHandler",
"level": environ.get("DEBUG_LEVEL", "WARNING"),
},
},
"loggers": {
"polyphonic": {
"handlers": ["console"],
"level": environ.get("DEBUG_LEVEL", "WARNING"),
}
},
}
+4
View File
@@ -0,0 +1,4 @@
from .base import * # noqa
# Enable WhiteNoise for static files
MIDDLEWARE.insert(1, "whitenoise.middleware.WhiteNoiseMiddleware") # noqa
@@ -19,9 +19,9 @@ from django.urls import path, include
urlpatterns = [
path("admin/", admin.site.urls),
path("", include("interface.urls")),
path("", include("polyphonic.interface.urls")),
# path('', include('submissions.urls')),
path("", include("library.urls")),
path("", include("polyphonic.library.urls")),
]
try:
@@ -11,6 +11,6 @@ import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'polyphonic.settings')
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "polyphonic.config.settings.base")
application = get_wsgi_application()
@@ -2,4 +2,4 @@ from django.apps import AppConfig
class InterfaceConfig(AppConfig):
name = "interface"
name = "polyphonic.interface"
@@ -4,7 +4,7 @@ import byostorage.user
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
import interface.models
import polyphonic.interface.models
class Migration(migrations.Migration):
@@ -23,7 +23,7 @@ class Migration(migrations.Migration):
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(help_text='Display name', max_length=100)),
('slug', models.SlugField(editable=False, help_text='Short name for the ensemble - used for folders', max_length=100, unique=True)),
('code', models.CharField(default=interface.models.generate_code, help_text='Ensemble registration code', max_length=9)),
('code', models.CharField(default=polyphonic.interface.models.generate_code, help_text='Ensemble registration code', max_length=9)),
('passphrase', models.CharField(help_text='Used to register ensembles', max_length=100)),
('details', models.TextField(blank=True, help_text='Description of the ensemble (markdown)')),
('admins', models.ManyToManyField(related_name='ensembles', to=settings.AUTH_USER_MODEL)),
@@ -60,7 +60,7 @@ class Migration(migrations.Migration):
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.CharField(max_length=100)),
('description', models.TextField(blank=True)),
('file', models.FileField(storage=byostorage.user.BYOStorage(), upload_to=interface.models.resource_key)),
('file', models.FileField(storage=byostorage.user.BYOStorage(), upload_to=polyphonic.interface.models.resource_key)),
('media_type', models.CharField(choices=[('audio', 'Audio'), ('video', 'Video'), ('general', 'General')], default='*', max_length=10)),
('visible', models.BooleanField(default=True)),
('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='resources', to='interface.project')),

Before

Width:  |  Height:  |  Size: 426 KiB

After

Width:  |  Height:  |  Size: 426 KiB

Before

Width:  |  Height:  |  Size: 258 B

After

Width:  |  Height:  |  Size: 258 B

@@ -26,4 +26,31 @@ document.addEventListener('DOMContentLoaded', () => {
}
}
});
});
async function background_api(el) {
console.log(el.dataset["api"]);
let text = el.innerHTML;
el.disabled = true;
el.innerHTML = "Running..."
const response = await fetch(el.dataset.api);
console.log(response);
el.innerHTML = text;
if(!response.ok) {
el.disabled = false;
throw new Error(`Response: ${response.status}`);
}
if (el.dataset.success) {
text = el.dataset.success;
}
el.innerHTML = text;
const data = await response.json();
console.log(data);
}
@@ -11,7 +11,8 @@
<script src="{% static 'interface/js/interface.js' %}"></script>
<script src="//unpkg.com/alpinejs" defer></script>
<script src="//cdnjs.cloudflare.com/ajax/libs/jquery/3.5.1/jquery.min.js" defer></script>
<script src="//kit.fontawesome.com/c837098e5b.js" crossorigin="anonymous" defer></script>
<!-- script src="//kit.fontawesome.com/c837098e5b.js" crossorigin="anonymous" defer></script -->
<link href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined" rel="stylesheet" />
<title>{% block title %}Polyphonic{% endblock %}</title>
{% block media %}{% endblock %}
<style>{% block style %}{% endblock %}</style>
@@ -23,7 +24,7 @@
<nav class="navbar" role="navigation">
<div class="navbar-brand has-text-primary">
<a class="navbar-item" href="/">
<span class="icon fancy is-size-2 is-size-4-mobile mx-4"><i class="fas fa-random"></i></span>
<span class="icon fancy mx-4"><span class="material-symbols-outlined is-size-1 is-size-3-mobile">groups</span></span>
<span class="fancy is-size-2 is-size-4-mobile">Polyphonic</span>
</a>
<span class="navbar-item is-hidden-mobile fancy is-size-5">Musical Ensemble Manager</span>
@@ -6,7 +6,7 @@
{% crispy_field field 'class' 'file-input'%}
<span class="file-cta">
<span class="file-icon">
<i class="fas fa-upload"></i>
<span class="material-symbols-outlined">file_upload</span>
</span>
<span class="file-label">
Choose a file…
@@ -25,4 +25,4 @@
fileName.textContent = fileInput.files[0].name;
}
}
</script>
</script>
@@ -1,19 +1,20 @@
{% extends "interface/project_base.html" %}
{% load md2 %}
{% load polyphonic %}
{% block admin %}
<a href="{% url 'project_create' object.slug %}" class="button is-link">
<span class="icon"><i class="fas fa-plus-circle"></i></span>
{{ "add_notes"|icon }}
<span>Add project</span>
</a>
{% if inactive %}
<a href="?" class="button is-link">
<span class="icon"><i class="fas fa-archive"></i></span>
{{ "preview_off"|icon }}
<span>Hide old</span>
</a>
{% else %}
<a href="?inactive" class="button is-link">
<span class="icon"><i class="fas fa-archive"></i></span>
{{ "preview"|icon }}
<span>Show all</span>
</a>
{% endif %}
@@ -54,4 +55,4 @@ Contacts:
<a href="{% url 'forget_resource' 'ensemble' ensemble.slug %}">Forget this ensemble</a>
</div>
{% endblock %}
{% endblock %}
@@ -5,7 +5,7 @@
{% comment %}
<div class="admin-tools is-pulled-right">
<a class="button is-link" href="{% url 'register' %}">
<span class="icon"><i class="fas fa-plus-circle"></i></span>
{% icon "add_file" %}
<span>Register another</span>
</a>
</div>
@@ -25,10 +25,13 @@
<img src="https://www.gravatar.com/avatar/{{ ensemble.email }}?d=mp" alt="Placeholder image">
</figure>
</div>
<div class="media-content" style="min-height: 60px">
<div class="media-content" style="min-height: 100px">
<a href="{% url 'ensemble_detail' ensemble.slug %}">
<p class="title is-4">{{ ensemble.name }}</p>
</a>
<div class="mt-3">
{{ ensemble.details|markdown }}
</div>
</div>
</div>
</div>
@@ -45,4 +48,4 @@
</div>
{% endfor %}
</div>
{% endblock %}
{% endblock %}
@@ -4,11 +4,11 @@
{% block admin %}
<a href="{% url 'wiki_create' project=project.pk %}" class="button is-link">
<span class="icon"><i class="fas fa-file"></i></span>
<span>Add Page</span>
{{ "add_notes"|icon }}
<span>Add Page</span>
</a>
<a href="{% url 'project_edit' project=project.pk %}" class="button is-link">
<span class="icon"><i class="fas fa-edit"></i></span>
{{ "edit"|icon }}
<span>Edit</span>
</a>
{% endblock %}
@@ -33,4 +33,4 @@
</div>
</div>
{% endfor %}
</div>
</div>
@@ -1,9 +1,10 @@
{% extends "interface/project_base.html" %}
{% load md2 %}
{% load polyphonic %}
{% block admin %}
<a class="button is-link" href="{% url 'resource_create' project=project.pk %}">
<span class="icon"><i class="fas fa-plus-circle"></i></span>
{% icon "add_notes" %}
<span>Add new</span>
</a>
{% endblock %}
@@ -27,11 +28,11 @@
<div class="card-header-icon">
{% if request.is_admin %}
<a href="{% url 'resource_upload' project=project.pk pk=resource.pk %}" class="icon" title="Upload">
<i class="fas fa-upload"></i>
<a href="{% url 'resource_upload' project=project.pk pk=resource.pk %}" title="Upload">
{% icon "upload_file" %}
</a>
<a href="{% url 'resource_edit' project=project.pk pk=resource.pk %}" class="icon" title="Edit">
<i class="fas fa-edit"></i>
<a href="{% url 'resource_edit' project=project.pk pk=resource.pk %}" title="Edit">
{% icon "edit" %}
</a>
{% endif %}
</div>
@@ -1,9 +1,10 @@
{% extends "interface/project_base.html" %}
{% load polyphonic %}
{% block admin %}
<a href="{% url 'wiki_edit' project=project.pk pk=wikipage.pk %}" class="button is-link">
<span class="icon"><i class="fas fa-edit"></i></span>
<span>Edit</span>
{{ "edit"|icon }}
<span>Edit</span>
</a>
{% endblock %}
@@ -12,4 +13,4 @@
<div class="box content wiki-page">
{{ wiki_html|safe }}
</div>
{% endblock %}
{% endblock %}
@@ -0,0 +1,36 @@
from django import template
from django.utils import timesince
from django.utils.html import format_html
register = template.Library()
@register.filter("icon", is_safe=True)
def material_icon(value):
return f'<span class="icon"><span class="material-symbols-outlined">{value}</span></span>'
@register.simple_tag
def icon(name, element="span", classes=[]):
classes = ["icon"] + classes
return format_html(
'<{} class="{}"><span class="material-symbols-outlined">{}</span></{}>',
element,
" ".join(classes),
name,
element,
)
def roughtimesince(value):
return timesince.timesince(value, depth=1)
register.filter("roughtimesince", roughtimesince)
def roughtimeuntil(value):
return timesince.timeuntil(value, depth=1)
register.filter("roughtimeuntil", roughtimeuntil)
@@ -1,5 +1,5 @@
from django.test import TestCase
from interface import models
from polyphonic.interface import models
from django.contrib.auth.models import User
from django.utils import timezone
from datetime import timedelta
@@ -1,5 +1,4 @@
from interface import models
from polyphonic.interface import models
from . import AccessTestCase
@@ -15,7 +15,7 @@ from django.contrib.auth import logout
from markdown2 import markdown
from . import models, forms
from interface.utils import check_signed_url
from .utils import check_signed_url
import logging
@@ -2,4 +2,4 @@ from django.apps import AppConfig
class LibraryConfig(AppConfig):
name = "library"
name = "polyphonic.library"
@@ -1,7 +1,7 @@
from django import forms
from .models import Work
from interface.models import Project
from interface.forms import BaseForm
from polyphonic.interface.models import Project
from polyphonic.interface.forms import BaseForm
class WorkCreateForm(forms.ModelForm, BaseForm):
+116
View File
@@ -0,0 +1,116 @@
from polyphonic.library.models import Collection, Work, WorkMeta, Document
from byostorage.models import UserStorage
import logging
logger = logging.getLogger(__name__)
def sync_work(work: Work):
try:
folder_id = work.meta_info.get(name="folderid").value
except WorkMeta.DoesNotExist as err:
raise IndexError("Work not currently linked to a gdrive folder") from err
logger.info("Syncing '%s' from %r", work.name, folder_id)
results = {
"folder_id": folder_id,
"added": [],
"skipped": [],
"missing": [],
}
storage = UserStorage.objects.get(name="gdrive").instance()
existing = set(
[
storage.parse_resource(x.partition(":")[2]).id
for x in work.docs.values_list("upload", flat=True)
]
)
logger.debug("%d existing documents", len(existing))
_, files = storage.listdir(folder_id)
logger.debug("Remote files: %r", files)
results["files"] = len(files)
for file in files:
if file.id in existing:
logger.debug("%30s: Skipping existing (%s)", file.name, file.id)
existing.discard(file.id)
continue
if not file.name.lower().endswith(".pdf"):
logger.debug("%40s: Not a PDF", file.name)
results["skipped"].append({"file": file.name, "file_id": file.id})
continue
results["added"].append({"file": file.name, "file_id": file.id})
doc = work.docs.create(upload=f"gdrive:{file}", doctype=Document.DOCTYPE_PDF)
doc.auto_tag()
for uri in existing:
results["missing"].append({"uri": uri})
logger.warning("Local entry not in folder: %s", uri)
return results
def sync_partial_collection(collection: Collection, sync_existing: bool = True):
works = Work.objects.filter(collection=collection, meta_info__name="folderid")
result = []
for work in works:
result.append(sync_work(work))
return result
def sync_collection(collection: Collection, sync_existing: bool = False):
logger.info("Syncing '%s'", collection)
if not collection.storage.storage.endswith("GDriveLinkStorage"):
return sync_partial_collection(collection, sync_existing)
existing = dict(
WorkMeta.objects.filter(
work__collection=collection, name="folderid"
).values_list("value", "work_id")
)
folder = collection.prefix
storage = collection.storage.instance()
folders, _ = storage.listdir(folder)
results = {
"folders": len(folders),
"added": [],
"missing": [],
}
for folder in folders:
if folder[0] == "_":
continue
if folder.id in existing:
if sync_existing:
logger.info("%40s: Syncing (%s)", folder.name, folder.id[:12])
sync_work(Work.objects.get(pk=existing[folder.id]))
del existing[folder.id]
continue
logger.info("%40s: Adding", folder.name)
work = Work(name=folder.name, collection=collection)
work.save()
work.meta_info.create(name="folderid", value=folder.id)
results["added"].append({"work": work.pk, "folder_id": folder.id})
sync_work(work)
for folderid, work in existing.items():
results["missing"].append({"work": work, "folder_id": folderid})
logger.warning("Folder for work %d no longer in drive (%s)", work, folderid)
return results
+147
View File
@@ -0,0 +1,147 @@
from django.core.files.storage import Storage
from collections import namedtuple
import requests
import re
from gzip import GzipFile
import logging
logger = logging.getLogger(__name__)
SHARED_FOLDER = re.compile(
r"https://drive.google.com/drive[u0-9\/]+folders/([\w\-]+)(\?resourcekey=([\w\-]+))?"
)
SHARED_FILE = re.compile(
r"https://drive.google.com/file/d/([\w\-]+)(\?resourcekey=([\w\-]+))?"
)
FILES_API = "https://www.googleapis.com/drive/v3/files"
class DriveObject(namedtuple("DriveObject", ("id", "key", "name"))):
@classmethod
def from_string(cls, s: str):
resource, _, name = s.partition("/")
id, _, key = resource.partition("#")
return cls(id, key, name)
def __str__(self):
resource = f"{self.id}#{self.key}" if self.key else self.id
return f"{resource}/{self.name}"
class GDriveLinkStorage(Storage):
is_writable = False
def __init__(self, api_key):
self.api_key = api_key
super().__init__()
def parse_resource(self, name) -> DriveObject:
return DriveObject.from_string(name)
def extract_resource(self, url, *patterns) -> DriveObject:
logger.debug("EXTRACT_RESOURCE: %r", url)
for pattern in patterns:
match = pattern.match(url)
if match:
groups = match.groups()
logger.debug(groups)
if len(groups) == 3:
return DriveObject(groups[0], groups[2], "")
return DriveObject(groups[0], None, "")
raise FileNotFoundError(f"Not a valid url: {url}")
def get_json(self, url, resource: DriveObject):
headers = {}
if resource.key:
headers["X-Goog-Drive-Resource-Keys"] = f"{resource.id}/{resource.key}"
logger.debug("GET_JSON: %s %r", url, headers)
response = requests.get(url, headers=headers)
data = response.json()
logger.debug("Data: %r", data)
return data
def listdir(self, path) -> tuple[list[str], list[str]]:
# used to test for valid connection parameters - should do something to validate API key here
logger.debug("listdir: %s", path)
if path == "":
return [], []
files = []
folders = []
folder = self.parse_resource(path)
url = f"{FILES_API}?q='{folder.id}'+in+parents&key={self.api_key}"
data = self.get_json(url, folder)
while True:
for x in data["files"]:
if x["mimeType"] == "application/vnd.google-apps.folder":
folders.append(
DriveObject(x["id"], x.get("resourceKey"), x["name"])
)
else:
files.append(DriveObject(x["id"], x.get("resourceKey"), x["name"]))
token = data.get("nextPageToken")
if token is None:
return folders, files
data = self.get_json(f"{url}&pageToken={token}", folder)
def get_meta(self, name):
file_resource = self.parse_resource(name)
url = f"{FILES_API}/{file_resource.id}?key={self.api_key}"
return self.get_json(url, file_resource)
def open(self, name, mode="rb"):
resource = self.parse_resource(name)
url = f"{FILES_API}/{resource.id}?alt=media&key={self.api_key}"
headers = {}
if resource.key:
headers["X-Goog-Drive-Resource-Keys"] = f"{resource.id}/{resource.key}"
logger.info("URL: %s [%r]", url, headers)
response = requests.get(url, headers=headers, stream=True)
return GzipFile(name, "rb", 9, response.raw)
def size(self, name):
raise NotImplementedError()
def delete(self, name):
pass
def url(self, name):
logger.debug("URL: %r", name)
resource = self.parse_resource(name)
uri = f"https://drive.usercontent.google.com/download?export=download&id={resource.id}&confirm=yes"
if resource.key:
uri += f"&resourcekey=${resource.key}"
return uri
def get_folder_id(self, url):
try:
return self.extract_resource(url, SHARED_FOLDER)
except FileNotFoundError:
return None
def get_file_id(self, url):
try:
return self.extract_resource(url, SHARED_FILE)
except FileNotFoundError:
return None
def import_link(self, url) -> str:
file_resource = self.extract_resource(url, SHARED_FILE)
meta = self.get_meta(file_resource)
return f"{file_resource}/{meta['name']}"
"""
def folder_import(self, url) -> list[str]:
folder_id = self.extract_id(url, SHARED_FOLDER)
_, files = self.listdir(folder_id)
return files
"""
@@ -1,9 +1,12 @@
from django.shortcuts import resolve_url, redirect
from django.views.generic import FormView
from django.views.generic.detail import SingleObjectMixin
from library.views import CollectionMixin
from library.models import Work, Document
from library import forms
from polyphonic.library.views import CollectionMixin
from polyphonic.library.models import Work, Document
from polyphonic.library import forms
from byostorage.models import UserStorage
from . import sync_work
class WorkGDriveView(CollectionMixin, SingleObjectMixin, FormView):
@@ -27,31 +30,30 @@ class WorkGDriveView(CollectionMixin, SingleObjectMixin, FormView):
def form_valid(self, form):
link = form.cleaned_data["link"]
storage = self.collection.storage.instance()
# storage = self.collection.storage.instance()
storage = UserStorage.objects.get(name="gdrive").instance()
self.object = self.get_object()
try:
folderid = storage.get_folder_id(link)
folderid = storage.get_folder_id(link)
if folderid:
self.object.meta_info.update_or_create(
name="folderid", defaults={"value": folderid}
)
return redirect("work_detail", self.collection.pk, self.kwargs["pk"])
except FileNotFoundError:
pass # not a folder id
sync_work(self.object)
try:
link = self.collection.storage.instance().import_link(link)
except AttributeError:
pass
except FileNotFoundError as e:
form.add_error("link", str(e))
return redirect("work_detail", self.collection.pk, self.kwargs["pk"])
link = storage.import_link(link)
if link is None:
form.add_error("link", "Not a valid link")
return self.form_invalid(form)
work = self.collection.works.get(pk=self.kwargs["pk"])
doc = Document(
work=work,
upload=f"{self.collection.storage.name}:{link}",
upload=f"gdrive:{link}",
# upload=f"{self.collection.storage.name}:{link}",
doctype=Document.DOCTYPE_PDF,
)
doc.save()
@@ -3,9 +3,9 @@ from django.conf import settings
from django.utils.module_loading import import_module
from django.db.models import QuerySet
from library.models import Work, Collection
from polyphonic.library.models import Work, Collection
instance = getattr(settings, "INDEXER", "library.indexer.whoosh")
instance = getattr(settings, "INDEXER", "polyphonic.library.indexer.whoosh")
class Indexer(Protocol):
@@ -3,7 +3,7 @@ from whoosh.analysis import StemmingAnalyzer, CharsetFilter
from whoosh.support.charset import accent_map
from whoosh.fields import Schema, TEXT, KEYWORD, NUMERIC
from whoosh.qparser import QueryParser
from whoosh.query import Term, NullQuery
from whoosh.query import Term, NullQuery, FuzzyTerm
from typing import Any
@@ -24,7 +24,7 @@ schema = Schema(
)
index_path = os.path.join(os.path.dirname(settings.BASE_DIR), "index")
index_path = settings.WHOOSH_INDEX
def create_index() -> Index:
@@ -61,20 +61,26 @@ def search(
pagesize: int = 20,
) -> tuple[list[dict], dict[str, Any]]:
meta = {}
query = query.lower()
qp = QueryParser("text", schema=schema)
q = qp.parse(query.lower())
meta["query"] = str(q)
q = qp.parse(query)
terms = NullQuery
limit = NullQuery
for c in collections:
terms = terms | Term("collection", c)
q = q & terms
limit |= Term("collection", c)
hits = []
ix = get_index()
with ix.searcher() as searcher:
results = searcher.search_page(q, page, pagesize)
results = searcher.search_page(q & limit, page, pagesize)
# if no results, do a prefix search
if results.results.is_empty():
qp.termclass = FuzzyTerm
q = qp.parse(query)
results = searcher.search_page(q & limit, page, pagesize)
for result in results:
hits.append(
dict(
@@ -85,7 +91,7 @@ def search(
collection_id=int(result["collection"]),
)
)
meta["query"] = str(q & limit)
meta["total"] = len(results)
return hits, meta
@@ -1,7 +1,7 @@
from django.core.management.base import BaseCommand
from library import models
from library.indexer import model_search, index_works, indexer
from polyphonic.library import models
from polyphonic.library.indexer import model_search, index_works, indexer
FORMATTER = "{w.name:50s} {w.edition:15s} {w.collection.name:15s}"
@@ -1,6 +1,6 @@
from django.core.management.base import BaseCommand, CommandError
from library.models import Work, Collection
from library.gdrive import sync_work, sync_collection
from polyphonic.library.models import Work, Collection
from polyphonic.library.gdrive import sync_work, sync_collection
class Command(BaseCommand):
@@ -4,7 +4,7 @@ import byostorage.user
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
import library.models
import polyphonic.library.models
class Migration(migrations.Migration):
@@ -35,7 +35,7 @@ class Migration(migrations.Migration):
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('doctype', models.PositiveSmallIntegerField(choices=[(1, 'PDF'), (2, 'Audio'), (3, 'Video'), (4, 'Source')], default=1)),
('upload', models.FileField(storage=byostorage.user.BYOStorage(), upload_to=library.models.doc_upload_filename)),
('upload', models.FileField(storage=byostorage.user.BYOStorage(), upload_to=polyphonic.library.models.doc_upload_filename)),
('created', models.DateTimeField(auto_now_add=True)),
('version', models.CharField(blank=True, max_length=30)),
],
@@ -3,7 +3,7 @@
import byostorage.cached
import byostorage.user
from django.db import migrations, models
import library.models
import polyphonic.library.models
class Migration(migrations.Migration):
@@ -26,7 +26,7 @@ class Migration(migrations.Migration):
migrations.AlterField(
model_name='document',
name='upload',
field=models.FileField(storage=byostorage.cached.CachedStorage(byostorage.user.BYOStorage()), upload_to=library.models.doc_upload_filename),
field=models.FileField(storage=byostorage.cached.CachedStorage(byostorage.user.BYOStorage()), upload_to=polyphonic.library.models.doc_upload_filename),
),
migrations.AlterField(
model_name='work',
@@ -11,8 +11,8 @@ import os.path
from byostorage.user import BYOStorage
from byostorage.cached import CachedStorage
from library.music_tags import MusicTag, auto_tag
from interface.utils import sign_data
from polyphonic.library.music_tags import MusicTag, auto_tag
from polyphonic.interface.utils import sign_data
import logging
@@ -19,6 +19,10 @@ cb Double Bass
mall Mallet Percussion
vln Violin
vla Viola
kit Drumkit
asax Alto Sax
tsax Tenor Sax
bsax Bari Sax
acc Accordion
afl Alto flute
@@ -1,4 +1,5 @@
{% extends "interface/project_base.html" %}
{% load polyphonic %}
{% block page %}
<h3 class="title">Library collections for {% firstof request.user.first_name request.user.username %}</h3>
@@ -10,7 +11,9 @@
<input class="input" name="q" type="text" placeholder="Find a work" value="{{ request.GET.filter }}"/>
</div>
<div class="control">
<a class="button" href="?"><i class="fas fa-times"></i></a>
<a class="button" href="?">
{% icon "close" %}
</a>
</div>
</div>
</form>
@@ -47,4 +50,5 @@
<div>
<small>{{ ensemble.ensemble_code }}</small>
</div>
{% endblock %}
@@ -1,11 +1,13 @@
{% extends "interface/project_base.html" %}
{% load polyphonic %}
{% block admin %}
<a href="#" onclick="saveTags()" class="button is-link">
<span class="icon"><i class="fas fa-save"></i></span>
{% icon "save" %}
<span>Save</span>
</a>
<a href="{% url 'work_detail' collection=collection.pk pk=object.work_id %}" class="button is-link is-light">
{% icon "backspace" %}
<span>Cancel</span>
</a>
{% endblock %}
@@ -76,7 +78,11 @@
</div>
<ul id="unassigned-area">
{% for tag, inst in document.work.music_tags %}
<li class="is-clickable" onclick="assignInstrument('{{tag}}', this)")>{{ inst }}</li>
<li>
<span class="is-clickable" onclick="assignInstrument('{{tag}}', this)")>{{ inst }}</span>
&nbsp;
<span class="is-clickable" onclick="addNumberedInstrument('{{tag}}', this)">...</span>
</li>
{% endfor %}
</ul>
<a onclick="document.getElementById('add-modal').classList.add('is-active')">Add Tag</a>
@@ -275,6 +281,16 @@
document.getElementById('add-instrument-name').value = "";
document.getElementById('add-instrument-variant').value = "";
}
function addNumberedInstrument(tag, e) {
let modal = document.getElementById('add-modal');
document.getElementById("add-instrument-name").value = data.instruments[tag];
document.getElementById("add-instrument-variant").value = 1;
document.getElementById("add-instrument-variant").focus();
modal.classList.add('is-active');
}
function addInstrument() {
let name = document.getElementById('add-instrument-name');
@@ -326,7 +342,7 @@
let setStart = document.createElement('span');
setStart.className = "icon is-action";
setStart.innerHTML = '<i class="fas fa-sort-amount-down" title="Set start page"></i>';
setStart.innerHTML = '<span class="material-symbols-outlined" title="Set start page">vertical_align_top</span>';
setStart.addEventListener('click', () => setTagStart(el));
el.appendChild(setStart);
@@ -336,22 +352,20 @@
let name = document.createElement('b');
name.innerHTML = get_instrument(tag);
label.appendChild(name);
el.appendChild(label);
let del = document.createElement('span');
del.className = "icon is-action";
del.innerHTML = '<i class="fas fa-trash-alt" title="Remove this tag"></i>';
del.innerHTML = '<span class="material-symbols-outlined" title="Remove this tag">delete</span>';
del.addEventListener('click', () => {
el.remove();
dirty=true;
});
label.appendChild(del)
el.appendChild(label);
el.appendChild(del)
let setEnd = document.createElement('span');
setEnd.className = "icon is-action";
setEnd.innerHTML = '<i class="fas fa-sort-amount-up" title="Set end page"></i>';
setEnd.innerHTML = '<span class="material-symbols-outlined" title="Set end page">vertical_align_bottom</span>';
setEnd.addEventListener('click', () => setTagEnd(el));
el.appendChild(setEnd);
@@ -1,7 +1,10 @@
{% load path_filters %}
{% load polyphonic %}
<tr>
<td><a href="{{ doc.upload.url }}" target="_blank">
{{ doc.upload.name|basename }}</a></td>
<td style="white-space: nowrap">
<a href="{{ doc.upload.url }}" target="_blank">
{{ doc.upload.name|basename }}</a>
</td>
<td>
{% for section in doc.sections.all %}
<a class="tag is-{{ section.bulma_class }}" target="_blank" href="{% url 'part_download' collection.pk section.pk section.filename %}">{{ section.name }}</a>
@@ -10,10 +13,13 @@
<td class="has-text-right" style="white-space: nowrap;">
{% if request.is_admin %}
{% if doc.doctype == 1 %}
<a href="{% url 'document_annotate' collection.pk doc.pk %}"><i class="fas fa-tags"
title="Manage Tags"></i></a>
<a href="{% url 'document_annotate' collection.pk doc.pk %}" title="Annotate">
{% icon "toc" %}
</a>
{% endif %}
<a href="{% url 'document_delete' collection.pk doc.pk %}"><i class="fas fa-trash-alt" title="Delete Document"></i></a>
<a href="{% url 'document_delete' collection.pk doc.pk %}" title="Delete">
{% icon "delete" %}
</a>
{% endif %}
</td>
</tr>
</tr>

Some files were not shown because too many files have changed in this diff Show More