From 872d16abd6bae59dbf8239e3e9f562a0930ba27e Mon Sep 17 00:00:00 2001 From: fraoustin Date: Wed, 9 Mar 2016 07:01:11 +0100 Subject: [PATCH] add cache --- piwigotools/__init__.py | 69 +++++++++++++++++++++++++++-------------- piwigotools/main.py | 32 ++++++++++--------- 2 files changed, 63 insertions(+), 38 deletions(-) diff --git a/piwigotools/__init__.py b/piwigotools/__init__.py index 0934a63..5009a97 100644 --- a/piwigotools/__init__.py +++ b/piwigotools/__init__.py @@ -14,6 +14,9 @@ import requests import piwigo +FORCE_PLAN = False +FORCE_IMAGES = False + class LoginException(Exception): def __str__(self): @@ -41,6 +44,8 @@ class Piwigo(piwigo.Piwigo): def __init__(self, url): piwigo.Piwigo.__init__(self, url) self._login = False + self._plan = {} + self._images = {} def login(self, username, password): """ @@ -60,9 +65,11 @@ class Piwigo(piwigo.Piwigo): @property def plan(self): - plan = { i["name"] : i["id"] for i in self.pwg.categories.getList(recursive=True, fullname=True)['categories'] } - plan[""] = 0 - return plan + if FORCE_PLAN or not len(self._plan): + plan = { i["name"] : i["id"] for i in self.pwg.categories.getList(recursive=True, fullname=True)['categories'] } + plan[""] = 0 + self._plan = plan + return self._plan def _checkarg(fn): def checking(self, *args, **kw): @@ -114,21 +121,24 @@ class Piwigo(piwigo.Piwigo): """ return list of file name image for path """ - try: - kw["cat_id"]= self.idcategory(path) - except PiwigoExistException: - return {} - kw["per_page"] = 200 - kw["page"] = 0 - imgs = {} - loop = True - while loop: - req = self.pwg.categories.getImages(**kw) - for img in req["images"]: - imgs[img["file"]] = img - if req["paging"]["count"] < req["paging"]["per_page"]: - loop = False - return imgs + if FORCE_IMAGES or path not in self._images: + try: + kw["cat_id"]= self.idcategory(path) + except PiwigoExistException: + return {} + kw["per_page"] = 200 + kw["page"] = 0 + imgs = {} + loop = True + while loop: + req = self.pwg.categories.getImages(**kw) + for img in req["images"]: + imgs[img["file"]] = img + if req["paging"]["count"] < req["paging"]["per_page"]: + loop = False + kw["page"] = kw["page"] + 1 + self._images[path] = imgs + return self._images[path] @_checkarg def sublevels(self, path, **kw): @@ -149,11 +159,13 @@ class Piwigo(piwigo.Piwigo): @_checkarg def idimage(self, path): - if not self.isimage(path): - raise PiwigoExistException("image %s not exist" % path) img = path.split(' / ')[-1] path = ' / '.join(path.split(' / ')[:-1]) - return self.images(path)[img]["id"] + try: + return self.images(path)[img]["id"] + except: + raise PiwigoExistException("image %s not exist" % path) + @_checkarg @_checklogin @@ -166,7 +178,9 @@ class Piwigo(piwigo.Piwigo): if parent and not self.iscategory(parent): raise PiwigoExistException("category %s not exist" % parent) if parent : kw['parent'] = self.plan[parent] - self.pwg.categories.add(**kw) + id = self.pwg.categories.add(**kw)['id'] + if not FORCE_PLAN: + self._plan[path] = id return self.idcategory(path) @_checkarg @@ -190,6 +204,8 @@ class Piwigo(piwigo.Piwigo): remove (delete) category """ self.pwg.categories.delete(category_id=self.idcategory(path), pwg_token=self.token, **kw) + if not FORCE_PLAN and path in self._plan: + del self._plan[path] return True @_checkarg @@ -201,9 +217,12 @@ class Piwigo(piwigo.Piwigo): kw["image"] = image if len(path): if not self.iscategory(path): - raise PiwigoExistException("category %s not exist" % parent) + raise PiwigoExistException("category %s not exist" % path) kw['category'] = self.idcategory(path) - return self.pwg.images.addSimple(**kw)['image_id'] + res = self.pwg.images.addSimple(**kw)['image_id'] + if ' / '.join(path.split(' / ')[:-1]) in self._images: + del self._images[' / '.join(path.split(' / ')[:-1])] + return res @_checkarg @_checklogin @@ -231,4 +250,6 @@ class Piwigo(piwigo.Piwigo): if not self.isimage(path): raise PiwigoException("image %s not exist" % path) self.pwg.images.delete(image_id= self.idimage(path), pwg_token=self.token) + if ' / '.join(path.split(' / ')[:-1]) in self._images: + del self._images[' / '.join(path.split(' / ')[:-1])] return True diff --git a/piwigotools/main.py b/piwigotools/main.py index a66ed41..90280ed 100644 --- a/piwigotools/main.py +++ b/piwigotools/main.py @@ -212,6 +212,8 @@ def main(): if not os.path.isdir(options.source): raise Exception("%s is not directory" % options.source) piwigo.iscategory(options.category) + if len(options.category) and options.category[-1] != '/' and options.category[:-3] != ' / ': + options.category = options.category+ ' / ' # treatment run = Run(verb, options.thread) kw = purge_kw(options.__dict__,('user','password','url','source','category','thread')) @@ -221,22 +223,24 @@ def main(): for ext in options.extension.split(',')[1:]: filtering = filtering + fnmatch.filter(filenames, ext) for filename in filtering: - path = os.path.abspath(os.path.join(root, filename))[len(options.source)+1:] - if not piwigo.isimage(path.replace(os.sep, ' / ')): - category = ' / '.join(path.split(os.sep)[:-1]) + pathrel = os.path.abspath(os.path.join(root, filename))[len(options.source)+1:] + pathabs = os.path.abspath(os.path.join(root, filename)) + category = options.category + ' / '.join(pathrel.split(os.sep)[:-1]) + if not piwigo.isimage(category + ' / ' + filename): run.add(piwigo.makedirs,[category,], kw) - run.add(piwigo.upload,[path, category], kw) + run.add(piwigo.upload,[pathabs, category], kw) # piwigo -> local - for category, item in piwigo.plan.iteritems(): - path = os.path.join(options.source, *category.split(' / ')) - if not os.path.exists(path): - os.makedirs(path) - for img in piwigo.images(category): - pathimg = os.path.join(path, img) - if not os.path.exists(pathimg): - run.add(piwigo.download, - ["%s / %s" % (category, str(img)), pathimg], - kw) + #for category, item in piwigo.plan.iteritems(): + # if options.category == category[0:len(options.category)]: + # path = os.path.join(options.source, *category[len(options.category):].split(' / ')) + # if not os.path.exists(path): + # os.makedirs(path) + # for img in piwigo.images(category): + # pathimg = os.path.join(path, img) + # if not os.path.exists(pathimg): + # run.add(piwigo.download, + # ["%s / %s" % (category, str(img)), pathimg], + # kw) ana.stop() except Exception as e: ana.stop()