diff --git a/polyphonic/config/settings/dev.py b/polyphonic/config/settings/dev.py index 68060a0..a553005 100644 --- a/polyphonic/config/settings/dev.py +++ b/polyphonic/config/settings/dev.py @@ -1,4 +1,5 @@ from .base import * # noqa +from os import environ DEBUG = True @@ -8,3 +9,20 @@ SECRET_KEY = "DO NOT USE IN PRODUCTION" INSTALLED_APPS.append("debug_toolbar") # noqa MIDDLEWARE.insert(1, "debug_toolbar.middleware.DebugToolbarMiddleware") # noqa INTERNAL_IPS = ["127.0.0.1"] + +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": environ.get("DEBUG_LEVEL", "WARNING"), + }, + }, + "loggers": { + "polyphonic": { + "handlers": ["console"], + "level": environ.get("DEBUG_LEVEL", "WARNING"), + } + }, +} diff --git a/polyphonic/library/gdrive/__init__.py b/polyphonic/library/gdrive/__init__.py index 30c10c1..0f1df82 100644 --- a/polyphonic/library/gdrive/__init__.py +++ b/polyphonic/library/gdrive/__init__.py @@ -1,4 +1,5 @@ from polyphonic.library.models import Collection, Work, WorkMeta, Document +from byostorage.models import UserStorage import logging @@ -6,12 +7,11 @@ logger = logging.getLogger(__name__) def sync_work(work: Work): - logger.info("Syncing '%s'", work.name) folder_id = work.meta_info.get(name="folderid").value - storage = work.collection.storage.instance() - prefix = work.collection.storage.name - _, files = storage.listdir(folder_id) + logger.info("Syncing '%s' from %r", work.name, folder_id) + + storage = UserStorage.objects.get(name="gdrive").instance() existing = set( [ @@ -21,6 +21,9 @@ def sync_work(work: Work): ) logger.debug("%d existing documents", len(existing)) + _, files = storage.listdir(folder_id) + logger.debug("Remote files: %r", files) + for file in files: if file.id in existing: logger.debug("%30s: Skipping existing (%s)", file.name, file.id) @@ -32,7 +35,7 @@ def sync_work(work: Work): continue logger.info("%40s: Adding", file.name) - doc = work.docs.create(upload=f"{prefix}:{file}", doctype=Document.DOCTYPE_PDF) + doc = work.docs.create(upload=f"gdrive:{file}", doctype=Document.DOCTYPE_PDF) doc.auto_tag() for uri in existing: @@ -45,8 +48,10 @@ def sync_collection(collection: Collection, sync_existing: bool = False): if not collection.storage.storage.endswith("GDriveLinkStorage"): raise RuntimeError("Not a gdrive storage") - if not collection.prefix: - raise KeyError("Prefix must store folder id") + try: + folder_id = collection.settings["folder_id"] + except KeyError: + raise KeyError("Missing 'folder_id' in settings") existing = dict( WorkMeta.objects.filter( @@ -55,9 +60,12 @@ def sync_collection(collection: Collection, sync_existing: bool = False): ) storage = collection.storage.instance() - folders, _ = storage.listdir(collection.prefix) + folders, _ = storage.listdir(folder_id) for folder in folders: + if folder[0] == "_": + continue + if folder.id in existing: if sync_existing: logger.info("%40s: Syncing (%s)", folder.name, folder.id[:12]) diff --git a/polyphonic/library/gdrive/storage.py b/polyphonic/library/gdrive/storage.py index 4f76f63..548bbbf 100644 --- a/polyphonic/library/gdrive/storage.py +++ b/polyphonic/library/gdrive/storage.py @@ -8,7 +8,7 @@ import logging logger = logging.getLogger(__name__) -SHARED_FOLDER = re.compile(r"https://drive.google.com/drive/folders/(\w+)") +SHARED_FOLDER = re.compile(r"https://drive.google.com/drive[u0-9\/]+folders/([\w\-]+)") SHARED_FILE = re.compile(r"https://drive.google.com/file/d/([\w\-]+)") FILES_API = "https://www.googleapis.com/drive/v3/files" @@ -50,12 +50,11 @@ class GDriveLinkStorage(Storage): return data def listdir(self, path) -> tuple[list[str], list[str]]: - # used to test for valid connection parameters - should do something to validate API key here + logger.debug("listdir: %s", path) if path == "": return [], [] - logger.debug("LISTDIR: %s", path) folder_id = self.parse_id(path) url = f"{FILES_API}?q='{folder_id}'+in+parents&key={self.api_key}" data = self.get_json(url) @@ -106,13 +105,13 @@ class GDriveLinkStorage(Storage): except FileNotFoundError: return None - -""" def import_link(self, url) -> str: file_id = self.extract_id(url, SHARED_FILE) meta = self.get_meta(file_id) return f"{file_id}/{meta['name']}" + +""" def folder_import(self, url) -> list[str]: folder_id = self.extract_id(url, SHARED_FOLDER) _, files = self.listdir(folder_id) diff --git a/polyphonic/library/gdrive/views.py b/polyphonic/library/gdrive/views.py index 8192422..27e4d6e 100644 --- a/polyphonic/library/gdrive/views.py +++ b/polyphonic/library/gdrive/views.py @@ -4,6 +4,9 @@ from django.views.generic.detail import SingleObjectMixin from polyphonic.library.views import CollectionMixin from polyphonic.library.models import Work, Document from polyphonic.library import forms +from byostorage.models import UserStorage + +from . import sync_work class WorkGDriveView(CollectionMixin, SingleObjectMixin, FormView): @@ -27,31 +30,30 @@ class WorkGDriveView(CollectionMixin, SingleObjectMixin, FormView): def form_valid(self, form): link = form.cleaned_data["link"] - storage = self.collection.storage.instance() + # storage = self.collection.storage.instance() + storage = UserStorage.objects.get(name="gdrive").instance() self.object = self.get_object() - try: - folderid = storage.get_folder_id(link) + folderid = storage.get_folder_id(link) + if folderid: self.object.meta_info.update_or_create( name="folderid", defaults={"value": folderid} ) - return redirect("work_detail", self.collection.pk, self.kwargs["pk"]) - except FileNotFoundError: - pass # not a folder id + sync_work(self.object) - try: - link = self.collection.storage.instance().import_link(link) - except AttributeError: - pass - except FileNotFoundError as e: - form.add_error("link", str(e)) + return redirect("work_detail", self.collection.pk, self.kwargs["pk"]) + + link = storage.import_link(link) + if link is None: + form.add_error("link", "Not a valid link") return self.form_invalid(form) work = self.collection.works.get(pk=self.kwargs["pk"]) doc = Document( work=work, - upload=f"{self.collection.storage.name}:{link}", + upload=f"gdrive:{link}", + # upload=f"{self.collection.storage.name}:{link}", doctype=Document.DOCTYPE_PDF, ) doc.save() diff --git a/polyphonic/library/templates/library/work_detail.html b/polyphonic/library/templates/library/work_detail.html index dda7405..d9d1af6 100644 --- a/polyphonic/library/templates/library/work_detail.html +++ b/polyphonic/library/templates/library/work_detail.html @@ -129,16 +129,19 @@ {% if request.is_admin %}

Add Files

- {% if "gdrive" in methods %} -
- Link Google Drive Files
-
- {% endif %} {% if "upload" in methods %}
{% csrf_token %}
{% endif %} + {% if "gdrive" in methods %} +
+ + + Link Google Drive Files +
+
+ {% endif %}
{% endif %} diff --git a/polyphonic/library/views/__init__.py b/polyphonic/library/views/__init__.py index 6feb07a..995ba22 100644 --- a/polyphonic/library/views/__init__.py +++ b/polyphonic/library/views/__init__.py @@ -320,9 +320,11 @@ class WorkDetailView(CollectionMixin, DetailView): methods = set(["upload"]) match self.collection.storage.storage: - case "library.storage.GDriveLinkStorage": + case "library.gdrive.storage.GDriveLinkStorage": methods.discard("upload") methods.add("gdrive") + case _: + methods.add("gdrive") context["methods"] = methods return context @@ -336,8 +338,7 @@ class WorkUpdateView(CollectionMixin, UpdateView): def form_valid(self, form): response = super().form_valid(form) - ix = indexer.get_index() - indexer.index_works(ix, [self.object]) + index_works([self.object]) return response diff --git a/pyproject.toml b/pyproject.toml index f35e799..61d72e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,12 +26,22 @@ dependencies = [ django-debug-toolbar = "5.2" ruff = "^0.15.12" coverage = "^7.14.0" +django-types = "^0.24.0" [tool.poetry.scripts] poly-tool = "polyphonic.manage:main" [tool.ruff] extend-exclude = ["**/migrations/"] +lint.extend-select = [ + #"DJ", # flake8-django: Django-specific bugs + #"E", # pycodestyle errors + "F", # Pyflakes + #"W", # pycodestyle warnings + #"I", # isort + #"UP", # pyupgrade + #"B", # flake8-bugbear +] [build-system] requires = ["poetry-core>=2.0.0,<3.0.0"]