Merge pull request 'gdrive' (#15) from gdrive into master

Reviewed-on: #15

polyphonic/config/settings/dev.py

@@ -1,4 +1,5 @@
 from .base import *  # noqa
+from os import environ
 
 DEBUG = True
 
@@ -8,3 +9,20 @@ SECRET_KEY = "DO NOT USE IN PRODUCTION"
 INSTALLED_APPS.append("debug_toolbar")  # noqa
 MIDDLEWARE.insert(1, "debug_toolbar.middleware.DebugToolbarMiddleware")  # noqa
 INTERNAL_IPS = ["127.0.0.1"]
+
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "level": environ.get("DEBUG_LEVEL", "WARNING"),
+        },
+    },
+    "loggers": {
+        "polyphonic": {
+            "handlers": ["console"],
+            "level": environ.get("DEBUG_LEVEL", "WARNING"),
+        }
+    },
+}

polyphonic/library/gdrive/__init__.py

@@ -1,4 +1,5 @@
 from polyphonic.library.models import Collection, Work, WorkMeta, Document
+from byostorage.models import UserStorage
 
 import logging
 
@@ -6,12 +7,11 @@ logger = logging.getLogger(__name__)
 
 
 def sync_work(work: Work):
-    logger.info("Syncing '%s'", work.name)
     folder_id = work.meta_info.get(name="folderid").value
 
-    storage = work.collection.storage.instance()
+    logger.info("Syncing '%s' from %r", work.name, folder_id)
-    prefix = work.collection.storage.name
+
-    _, files = storage.listdir(folder_id)
+    storage = UserStorage.objects.get(name="gdrive").instance()
 
     existing = set(
         [
@@ -21,6 +21,9 @@ def sync_work(work: Work):
     )
     logger.debug("%d existing documents", len(existing))
 
+    _, files = storage.listdir(folder_id)
+    logger.debug("Remote files: %r", files)
+
     for file in files:
         if file.id in existing:
             logger.debug("%30s: Skipping existing (%s)", file.name, file.id)
@@ -32,7 +35,7 @@ def sync_work(work: Work):
             continue
 
         logger.info("%40s: Adding", file.name)
-        doc = work.docs.create(upload=f"{prefix}:{file}", doctype=Document.DOCTYPE_PDF)
+        doc = work.docs.create(upload=f"gdrive:{file}", doctype=Document.DOCTYPE_PDF)
         doc.auto_tag()
 
     for uri in existing:
@@ -45,8 +48,10 @@ def sync_collection(collection: Collection, sync_existing: bool = False):
     if not collection.storage.storage.endswith("GDriveLinkStorage"):
         raise RuntimeError("Not a gdrive storage")
 
-    if not collection.prefix:
+    try:
-        raise KeyError("Prefix must store folder id")
+        folder_id = collection.settings["folder_id"]
+    except KeyError:
+        raise KeyError("Missing 'folder_id' in settings")
 
     existing = dict(
         WorkMeta.objects.filter(
@@ -55,9 +60,12 @@ def sync_collection(collection: Collection, sync_existing: bool = False):
     )
 
     storage = collection.storage.instance()
-    folders, _ = storage.listdir(collection.prefix)
+    folders, _ = storage.listdir(folder_id)
 
     for folder in folders:
+        if folder[0] == "_":
+            continue
+
         if folder.id in existing:
             if sync_existing:
                 logger.info("%40s: Syncing (%s)", folder.name, folder.id[:12])

polyphonic/library/gdrive/storage.py

@@ -8,7 +8,7 @@ import logging
 
 logger = logging.getLogger(__name__)
 
-SHARED_FOLDER = re.compile(r"https://drive.google.com/drive/folders/(\w+)")
+SHARED_FOLDER = re.compile(r"https://drive.google.com/drive[u0-9\/]+folders/([\w\-]+)")
 SHARED_FILE = re.compile(r"https://drive.google.com/file/d/([\w\-]+)")
 
 FILES_API = "https://www.googleapis.com/drive/v3/files"
@@ -50,12 +50,11 @@ class GDriveLinkStorage(Storage):
         return data
 
     def listdir(self, path) -> tuple[list[str], list[str]]:
-
         # used to test for valid connection parameters - should do something to validate API key here
+        logger.debug("listdir: %s", path)
         if path == "":
             return [], []
 
-        logger.debug("LISTDIR: %s", path)
         folder_id = self.parse_id(path)
         url = f"{FILES_API}?q='{folder_id}'+in+parents&key={self.api_key}"
         data = self.get_json(url)
@@ -106,13 +105,13 @@ class GDriveLinkStorage(Storage):
         except FileNotFoundError:
             return None
 
-
-"""
     def import_link(self, url) -> str:
         file_id = self.extract_id(url, SHARED_FILE)
         meta = self.get_meta(file_id)
         return f"{file_id}/{meta['name']}"
 
+
+"""
     def folder_import(self, url) -> list[str]:
         folder_id = self.extract_id(url, SHARED_FOLDER)
         _, files = self.listdir(folder_id)

polyphonic/library/gdrive/views.py

@@ -4,6 +4,9 @@ from django.views.generic.detail import SingleObjectMixin
 from polyphonic.library.views import CollectionMixin
 from polyphonic.library.models import Work, Document
 from polyphonic.library import forms
+from byostorage.models import UserStorage
+
+from . import sync_work
 
 
 class WorkGDriveView(CollectionMixin, SingleObjectMixin, FormView):
@@ -27,31 +30,30 @@ class WorkGDriveView(CollectionMixin, SingleObjectMixin, FormView):
     def form_valid(self, form):
         link = form.cleaned_data["link"]
 
-        storage = self.collection.storage.instance()
+        # storage = self.collection.storage.instance()
+        storage = UserStorage.objects.get(name="gdrive").instance()
         self.object = self.get_object()
 
-        try:
+        folderid = storage.get_folder_id(link)
-            folderid = storage.get_folder_id(link)
+        if folderid:
             self.object.meta_info.update_or_create(
                 name="folderid", defaults={"value": folderid}
             )
-            return redirect("work_detail", self.collection.pk, self.kwargs["pk"])
+            sync_work(self.object)
-        except FileNotFoundError:
-            pass  # not a folder id
 
-        try:
+            return redirect("work_detail", self.collection.pk, self.kwargs["pk"])
-            link = self.collection.storage.instance().import_link(link)
+
-        except AttributeError:
+        link = storage.import_link(link)
-            pass
+        if link is None:
-        except FileNotFoundError as e:
+            form.add_error("link", "Not a valid link")
-            form.add_error("link", str(e))
             return self.form_invalid(form)
 
         work = self.collection.works.get(pk=self.kwargs["pk"])
 
         doc = Document(
             work=work,
-            upload=f"{self.collection.storage.name}:{link}",
+            upload=f"gdrive:{link}",
+            # upload=f"{self.collection.storage.name}:{link}",
             doctype=Document.DOCTYPE_PDF,
         )
         doc.save()

polyphonic/library/templates/library/work_detail.html

@@ -129,16 +129,19 @@
             {% if request.is_admin %}
             <div class="column is-one-quarter">
                 <h4 class="is-size-5">Add Files</h4>
-                {% if "gdrive" in methods %}
-                <div class="has-text-centered mt-3">
-                  <a class="button button-primary" href="{% url 'work_gdrive' collection.pk object.pk %}">Link Google Drive Files</a><br/>
-                </div>
-                {% endif %}
                 {% if "upload" in methods %}
                 <form action="{% url 'document_add' collection.pk object.pk %}" class="dropzone" id="doc-upload" style="-moz-user-select: none">
                     {% csrf_token %}
                 </form>
                 {% endif %}
+                {% if "gdrive" in methods %}
+                <div class="has-text-centered mt-3">
+                  <a class="button button-primary is-size-7" href="{% url 'work_gdrive' collection.pk object.pk %}">
+                    <span class="icon"><i class="fa-brands fa-google-drive"></i></span>
+                    Link Google Drive Files
+                  </a><br/>
+                </div>
+                {% endif %}
             </div>
             {% endif %}
         </div>

polyphonic/library/views/__init__.py

@@ -320,9 +320,11 @@ class WorkDetailView(CollectionMixin, DetailView):
 
         methods = set(["upload"])
         match self.collection.storage.storage:
-            case "library.storage.GDriveLinkStorage":
+            case "library.gdrive.storage.GDriveLinkStorage":
                 methods.discard("upload")
                 methods.add("gdrive")
+            case _:
+                methods.add("gdrive")
         context["methods"] = methods
 
         return context
@@ -336,8 +338,7 @@ class WorkUpdateView(CollectionMixin, UpdateView):
     def form_valid(self, form):
         response = super().form_valid(form)
 
-        ix = indexer.get_index()
+        index_works([self.object])
-        indexer.index_works(ix, [self.object])
 
         return response
 

pyproject.toml

@@ -26,12 +26,22 @@ dependencies = [
 django-debug-toolbar = "5.2"
 ruff = "^0.15.12"
 coverage = "^7.14.0"
+django-types = "^0.24.0"
 
 [tool.poetry.scripts]
 poly-tool = "polyphonic.manage:main"
 
 [tool.ruff]
 extend-exclude = ["**/migrations/"]
+lint.extend-select = [
+  #"DJ", # flake8-django: Django-specific bugs
+  #"E", # pycodestyle errors
+  "F", # Pyflakes
+  #"W", # pycodestyle warnings
+  #"I", # isort
+  #"UP", # pyupgrade
+  #"B", # flake8-bugbear
+]
 
 [build-system]
 requires = ["poetry-core>=2.0.0,<3.0.0"]