From cec64ecd2f09ef2a3de3369c02b073fbeb691b57 Mon Sep 17 00:00:00 2001 From: Tris Forster Date: Sun, 24 May 2026 13:51:23 +1000 Subject: [PATCH] Working with manual sync --- polyphonic/config/settings/dev.py | 18 ++++++++++++++++++ polyphonic/library/gdrive/__init__.py | 24 ++++++++++++++++-------- polyphonic/library/gdrive/storage.py | 5 ++--- polyphonic/library/gdrive/views.py | 6 +++++- 4 files changed, 41 insertions(+), 12 deletions(-) diff --git a/polyphonic/config/settings/dev.py b/polyphonic/config/settings/dev.py index 68060a0..a553005 100644 --- a/polyphonic/config/settings/dev.py +++ b/polyphonic/config/settings/dev.py @@ -1,4 +1,5 @@ from .base import * # noqa +from os import environ DEBUG = True @@ -8,3 +9,20 @@ SECRET_KEY = "DO NOT USE IN PRODUCTION" INSTALLED_APPS.append("debug_toolbar") # noqa MIDDLEWARE.insert(1, "debug_toolbar.middleware.DebugToolbarMiddleware") # noqa INTERNAL_IPS = ["127.0.0.1"] + +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": environ.get("DEBUG_LEVEL", "WARNING"), + }, + }, + "loggers": { + "polyphonic": { + "handlers": ["console"], + "level": environ.get("DEBUG_LEVEL", "WARNING"), + } + }, +} diff --git a/polyphonic/library/gdrive/__init__.py b/polyphonic/library/gdrive/__init__.py index 30c10c1..0f1df82 100644 --- a/polyphonic/library/gdrive/__init__.py +++ b/polyphonic/library/gdrive/__init__.py @@ -1,4 +1,5 @@ from polyphonic.library.models import Collection, Work, WorkMeta, Document +from byostorage.models import UserStorage import logging @@ -6,12 +7,11 @@ logger = logging.getLogger(__name__) def sync_work(work: Work): - logger.info("Syncing '%s'", work.name) folder_id = work.meta_info.get(name="folderid").value - storage = work.collection.storage.instance() - prefix = work.collection.storage.name - _, files = storage.listdir(folder_id) + logger.info("Syncing '%s' from %r", work.name, folder_id) + + storage = UserStorage.objects.get(name="gdrive").instance() existing = set( [ @@ -21,6 +21,9 @@ def sync_work(work: Work): ) logger.debug("%d existing documents", len(existing)) + _, files = storage.listdir(folder_id) + logger.debug("Remote files: %r", files) + for file in files: if file.id in existing: logger.debug("%30s: Skipping existing (%s)", file.name, file.id) @@ -32,7 +35,7 @@ def sync_work(work: Work): continue logger.info("%40s: Adding", file.name) - doc = work.docs.create(upload=f"{prefix}:{file}", doctype=Document.DOCTYPE_PDF) + doc = work.docs.create(upload=f"gdrive:{file}", doctype=Document.DOCTYPE_PDF) doc.auto_tag() for uri in existing: @@ -45,8 +48,10 @@ def sync_collection(collection: Collection, sync_existing: bool = False): if not collection.storage.storage.endswith("GDriveLinkStorage"): raise RuntimeError("Not a gdrive storage") - if not collection.prefix: - raise KeyError("Prefix must store folder id") + try: + folder_id = collection.settings["folder_id"] + except KeyError: + raise KeyError("Missing 'folder_id' in settings") existing = dict( WorkMeta.objects.filter( @@ -55,9 +60,12 @@ def sync_collection(collection: Collection, sync_existing: bool = False): ) storage = collection.storage.instance() - folders, _ = storage.listdir(collection.prefix) + folders, _ = storage.listdir(folder_id) for folder in folders: + if folder[0] == "_": + continue + if folder.id in existing: if sync_existing: logger.info("%40s: Syncing (%s)", folder.name, folder.id[:12]) diff --git a/polyphonic/library/gdrive/storage.py b/polyphonic/library/gdrive/storage.py index a17715e..548bbbf 100644 --- a/polyphonic/library/gdrive/storage.py +++ b/polyphonic/library/gdrive/storage.py @@ -8,7 +8,7 @@ import logging logger = logging.getLogger(__name__) -SHARED_FOLDER = re.compile(r"https://drive.google.com/drive/folders/(\w+)") +SHARED_FOLDER = re.compile(r"https://drive.google.com/drive[u0-9\/]+folders/([\w\-]+)") SHARED_FILE = re.compile(r"https://drive.google.com/file/d/([\w\-]+)") FILES_API = "https://www.googleapis.com/drive/v3/files" @@ -50,12 +50,11 @@ class GDriveLinkStorage(Storage): return data def listdir(self, path) -> tuple[list[str], list[str]]: - # used to test for valid connection parameters - should do something to validate API key here + logger.debug("listdir: %s", path) if path == "": return [], [] - logger.debug("LISTDIR: %s", path) folder_id = self.parse_id(path) url = f"{FILES_API}?q='{folder_id}'+in+parents&key={self.api_key}" data = self.get_json(url) diff --git a/polyphonic/library/gdrive/views.py b/polyphonic/library/gdrive/views.py index 7bf9aff..27e4d6e 100644 --- a/polyphonic/library/gdrive/views.py +++ b/polyphonic/library/gdrive/views.py @@ -6,6 +6,8 @@ from polyphonic.library.models import Work, Document from polyphonic.library import forms from byostorage.models import UserStorage +from . import sync_work + class WorkGDriveView(CollectionMixin, SingleObjectMixin, FormView): model = Work @@ -37,11 +39,13 @@ class WorkGDriveView(CollectionMixin, SingleObjectMixin, FormView): self.object.meta_info.update_or_create( name="folderid", defaults={"value": folderid} ) + sync_work(self.object) + return redirect("work_detail", self.collection.pk, self.kwargs["pk"]) link = storage.import_link(link) if link is None: - form.add_error("link", str(e)) + form.add_error("link", "Not a valid link") return self.form_invalid(form) work = self.collection.works.get(pk=self.kwargs["pk"])