From b2bb209dfcf75f96fc8abc1b6b4eeb1d752e232d Mon Sep 17 00:00:00 2001 From: fraoustin Date: Thu, 28 Jan 2016 11:40:31 +0100 Subject: [PATCH 1/7] branch feature-sync From 61f33870086708479564f1736e993020deb2fcf4 Mon Sep 17 00:00:00 2001 From: fraoustin Date: Sun, 7 Feb 2016 19:31:22 +0100 Subject: [PATCH 2/7] advance on sync --- piwigotools/__init__.py | 5 +++-- piwigotools/interface.py | 11 ++++++----- piwigotools/main.py | 28 +++++++++++++++++++++++++--- 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/piwigotools/__init__.py b/piwigotools/__init__.py index c74e6e7..adf18f6 100644 --- a/piwigotools/__init__.py +++ b/piwigotools/__init__.py @@ -60,8 +60,9 @@ class Piwigo(piwigo.Piwigo): @property def plan(self): - return { i["name"] : i["id"] for i in self.pwg.categories.getList(recursive=True, fullname=True)['categories'] } - + plan = { i["name"] : i["id"] for i in self.pwg.categories.getList(recursive=True, fullname=True)['categories'] } + plan[""] = 0 + return plan def _checkarg(fn): def checking(self, *args, **kw): diff --git a/piwigotools/interface.py b/piwigotools/interface.py index dd377f2..86afc4e 100644 --- a/piwigotools/interface.py +++ b/piwigotools/interface.py @@ -70,11 +70,12 @@ class Run: ' ', progressbar.Timer()], maxval=self._qin.qsize()).start() - for thread in self._threads: - thread.start() - while not self._qout.full(): - time.sleep(0.1) # sleep 0.1s - pbar.update(self._qout.qsize()) + if self._qin.qsize(): + for thread in self._threads: + thread.start() + while not self._qout.full(): + time.sleep(0.1) # sleep 0.1s + pbar.update(self._qout.qsize()) pbar.finish() return self._qerr diff --git a/piwigotools/main.py b/piwigotools/main.py index 44d4eba..bfae269 100644 --- a/piwigotools/main.py +++ b/piwigotools/main.py @@ -35,7 +35,7 @@ VERBS = { "description" : "upload file in piwigo gallery", "arg" : { - "category" : {"type":"string", "default":"/", "help":"destination category of piwigo gallery"}, + "category" : {"type":"string", "default":"", "help":"destination category of piwigo gallery"}, "source" : {"type":"string", "default":"*.jpg", "help":"path of upload picture"}, "url" : {"type":"string", "default":"", "help":"url of piwigo gallery"}, "user" : {"type":"string", "default":"", "help":"user of piwigo gallery"}, @@ -49,7 +49,7 @@ VERBS = { "description" : "download image from piwigo gallery", "arg" : { - "category" : {"type":"string", "default":"/", "help":"source category of piwigo gallery"}, + "category" : {"type":"string", "default":"", "help":"source category of piwigo gallery"}, "dest" : {"type":"string", "default":".", "help":"path of destination"}, "url" : {"type":"string", "default":"", "help":"url of piwigo gallery"}, "user" : {"type":"string", "default":"", "help":"user of piwigo gallery"}, @@ -63,7 +63,7 @@ VERBS = { "description" : "synchronization between path and piwigo gallery", "arg" : { - "category" : {"type":"string", "default":"/", "help":"category of piwigo gallery"}, + "category" : {"type":"string", "default":"", "help":"category of piwigo gallery"}, "source" : {"type":"string", "default":".", "help":"path of picture"}, "url" : {"type":"string", "default":"", "help":"url of piwigo gallery"}, "user" : {"type":"string", "default":"", "help":"user of piwigo gallery"}, @@ -199,6 +199,28 @@ def main(): piwigo.logout() if run.error: parser.error(run.strerror) + if verb == "sync": + ana = Analyse('Analyze') + ana.start() + try: + piwigo = Piwigo(url=options.url) + piwigo.login(options.user, options.password) + # check + options.source = os.path.abspath(options.source) + if not os.path.isdir(options.source): + raise Exception("%s is not directory" % options.source) + piwigo.iscategory(options.category) + # treatment + run = Run(verb, options.thread) + kw = purge_kw(options.__dict__,('user','password','url','source','category','thread')) + ana.stop() + except Exception as e: + ana.stop() + raise e + run.start() + piwigo.logout() + if run.error: + parser.error(run.strerror) except Exception as e: parser.error(e) sys.exit(1) From 3063af348212819cb8ffb620185ad9e9a1fc768d Mon Sep 17 00:00:00 2001 From: fraoustin Date: Thu, 18 Feb 2016 20:48:34 +0100 Subject: [PATCH 3/7] add upload in sync --- piwigotools/__init__.py | 5 ++++- piwigotools/main.py | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/piwigotools/__init__.py b/piwigotools/__init__.py index adf18f6..0934a63 100644 --- a/piwigotools/__init__.py +++ b/piwigotools/__init__.py @@ -114,7 +114,10 @@ class Piwigo(piwigo.Piwigo): """ return list of file name image for path """ - kw["cat_id"]= self.idcategory(path) + try: + kw["cat_id"]= self.idcategory(path) + except PiwigoExistException: + return {} kw["per_page"] = 200 kw["page"] = 0 imgs = {} diff --git a/piwigotools/main.py b/piwigotools/main.py index bfae269..7915475 100644 --- a/piwigotools/main.py +++ b/piwigotools/main.py @@ -3,6 +3,7 @@ import sys import os, os.path import glob +import fnmatch import pprint try: @@ -69,6 +70,7 @@ VERBS = { "user" : {"type":"string", "default":"", "help":"user of piwigo gallery"}, "password" : {"type":"string", "default":"", "help":"password of piwigo gallery"}, "thread" : {"type":"int", "default":"1", "help":"number of thread"}, + "extension" : {"type":"string", "default":"*.JPG", "help":"extension for upload"}, }, }, "ws": @@ -213,6 +215,13 @@ def main(): # treatment run = Run(verb, options.thread) kw = purge_kw(options.__dict__,('user','password','url','source','category','thread')) + for root, dirnames, filenames in os.walk(options.source): + for filename in fnmatch.filter(filenames, options.extension): + path = os.path.abspath(os.path.join(root, filename))[len(options.source)+1:] + if not piwigo.isimage(path.replace(os.sep, ' / ')): + category = ' / '.join(path.split(os.sep)[:-1]) + run.add(piwigo.makedirs,[category,], kw) + run.add(piwigo.upload,[path, category], kw) ana.stop() except Exception as e: ana.stop() From a4277d7e8b0976a234db2b35323e393e7e21ca59 Mon Sep 17 00:00:00 2001 From: fraoustin Date: Fri, 19 Feb 2016 10:38:10 +0100 Subject: [PATCH 4/7] add upload in sync --- piwigotools/main.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/piwigotools/main.py b/piwigotools/main.py index 7915475..a66ed41 100644 --- a/piwigotools/main.py +++ b/piwigotools/main.py @@ -70,7 +70,7 @@ VERBS = { "user" : {"type":"string", "default":"", "help":"user of piwigo gallery"}, "password" : {"type":"string", "default":"", "help":"password of piwigo gallery"}, "thread" : {"type":"int", "default":"1", "help":"number of thread"}, - "extension" : {"type":"string", "default":"*.JPG", "help":"extension for upload"}, + "extension" : {"type":"string", "default":"*.JPG,*.jpg,*.PNG,*.png,*.JPEG,*.jpeg,*.GIF,*.gif", "help":"extension for upload"}, }, }, "ws": @@ -168,7 +168,7 @@ def main(): kw = purge_kw(options.__dict__,('user','password','url','dest','category','thread')) for img in piwigo.images(options.category, **kw): run.add(piwigo.download, - ["%s / %s" % (options.category, str(img)), "%s/%s" % (options.dest, str(img))], + ["%s / %s" % (options.category, str(img)), "%s%s%s" % (options.dest, os.path.sep, str(img))], kw) except Exception as e: ana.stop() @@ -215,13 +215,28 @@ def main(): # treatment run = Run(verb, options.thread) kw = purge_kw(options.__dict__,('user','password','url','source','category','thread')) + # local -> piwigo for root, dirnames, filenames in os.walk(options.source): - for filename in fnmatch.filter(filenames, options.extension): + filtering = fnmatch.filter(filenames, options.extension.split(',')[0]) + for ext in options.extension.split(',')[1:]: + filtering = filtering + fnmatch.filter(filenames, ext) + for filename in filtering: path = os.path.abspath(os.path.join(root, filename))[len(options.source)+1:] if not piwigo.isimage(path.replace(os.sep, ' / ')): category = ' / '.join(path.split(os.sep)[:-1]) run.add(piwigo.makedirs,[category,], kw) run.add(piwigo.upload,[path, category], kw) + # piwigo -> local + for category, item in piwigo.plan.iteritems(): + path = os.path.join(options.source, *category.split(' / ')) + if not os.path.exists(path): + os.makedirs(path) + for img in piwigo.images(category): + pathimg = os.path.join(path, img) + if not os.path.exists(pathimg): + run.add(piwigo.download, + ["%s / %s" % (category, str(img)), pathimg], + kw) ana.stop() except Exception as e: ana.stop() From 872d16abd6bae59dbf8239e3e9f562a0930ba27e Mon Sep 17 00:00:00 2001 From: fraoustin Date: Wed, 9 Mar 2016 07:01:11 +0100 Subject: [PATCH 5/7] add cache --- piwigotools/__init__.py | 69 +++++++++++++++++++++++++++-------------- piwigotools/main.py | 32 ++++++++++--------- 2 files changed, 63 insertions(+), 38 deletions(-) diff --git a/piwigotools/__init__.py b/piwigotools/__init__.py index 0934a63..5009a97 100644 --- a/piwigotools/__init__.py +++ b/piwigotools/__init__.py @@ -14,6 +14,9 @@ import requests import piwigo +FORCE_PLAN = False +FORCE_IMAGES = False + class LoginException(Exception): def __str__(self): @@ -41,6 +44,8 @@ class Piwigo(piwigo.Piwigo): def __init__(self, url): piwigo.Piwigo.__init__(self, url) self._login = False + self._plan = {} + self._images = {} def login(self, username, password): """ @@ -60,9 +65,11 @@ class Piwigo(piwigo.Piwigo): @property def plan(self): - plan = { i["name"] : i["id"] for i in self.pwg.categories.getList(recursive=True, fullname=True)['categories'] } - plan[""] = 0 - return plan + if FORCE_PLAN or not len(self._plan): + plan = { i["name"] : i["id"] for i in self.pwg.categories.getList(recursive=True, fullname=True)['categories'] } + plan[""] = 0 + self._plan = plan + return self._plan def _checkarg(fn): def checking(self, *args, **kw): @@ -114,21 +121,24 @@ class Piwigo(piwigo.Piwigo): """ return list of file name image for path """ - try: - kw["cat_id"]= self.idcategory(path) - except PiwigoExistException: - return {} - kw["per_page"] = 200 - kw["page"] = 0 - imgs = {} - loop = True - while loop: - req = self.pwg.categories.getImages(**kw) - for img in req["images"]: - imgs[img["file"]] = img - if req["paging"]["count"] < req["paging"]["per_page"]: - loop = False - return imgs + if FORCE_IMAGES or path not in self._images: + try: + kw["cat_id"]= self.idcategory(path) + except PiwigoExistException: + return {} + kw["per_page"] = 200 + kw["page"] = 0 + imgs = {} + loop = True + while loop: + req = self.pwg.categories.getImages(**kw) + for img in req["images"]: + imgs[img["file"]] = img + if req["paging"]["count"] < req["paging"]["per_page"]: + loop = False + kw["page"] = kw["page"] + 1 + self._images[path] = imgs + return self._images[path] @_checkarg def sublevels(self, path, **kw): @@ -149,11 +159,13 @@ class Piwigo(piwigo.Piwigo): @_checkarg def idimage(self, path): - if not self.isimage(path): - raise PiwigoExistException("image %s not exist" % path) img = path.split(' / ')[-1] path = ' / '.join(path.split(' / ')[:-1]) - return self.images(path)[img]["id"] + try: + return self.images(path)[img]["id"] + except: + raise PiwigoExistException("image %s not exist" % path) + @_checkarg @_checklogin @@ -166,7 +178,9 @@ class Piwigo(piwigo.Piwigo): if parent and not self.iscategory(parent): raise PiwigoExistException("category %s not exist" % parent) if parent : kw['parent'] = self.plan[parent] - self.pwg.categories.add(**kw) + id = self.pwg.categories.add(**kw)['id'] + if not FORCE_PLAN: + self._plan[path] = id return self.idcategory(path) @_checkarg @@ -190,6 +204,8 @@ class Piwigo(piwigo.Piwigo): remove (delete) category """ self.pwg.categories.delete(category_id=self.idcategory(path), pwg_token=self.token, **kw) + if not FORCE_PLAN and path in self._plan: + del self._plan[path] return True @_checkarg @@ -201,9 +217,12 @@ class Piwigo(piwigo.Piwigo): kw["image"] = image if len(path): if not self.iscategory(path): - raise PiwigoExistException("category %s not exist" % parent) + raise PiwigoExistException("category %s not exist" % path) kw['category'] = self.idcategory(path) - return self.pwg.images.addSimple(**kw)['image_id'] + res = self.pwg.images.addSimple(**kw)['image_id'] + if ' / '.join(path.split(' / ')[:-1]) in self._images: + del self._images[' / '.join(path.split(' / ')[:-1])] + return res @_checkarg @_checklogin @@ -231,4 +250,6 @@ class Piwigo(piwigo.Piwigo): if not self.isimage(path): raise PiwigoException("image %s not exist" % path) self.pwg.images.delete(image_id= self.idimage(path), pwg_token=self.token) + if ' / '.join(path.split(' / ')[:-1]) in self._images: + del self._images[' / '.join(path.split(' / ')[:-1])] return True diff --git a/piwigotools/main.py b/piwigotools/main.py index a66ed41..90280ed 100644 --- a/piwigotools/main.py +++ b/piwigotools/main.py @@ -212,6 +212,8 @@ def main(): if not os.path.isdir(options.source): raise Exception("%s is not directory" % options.source) piwigo.iscategory(options.category) + if len(options.category) and options.category[-1] != '/' and options.category[:-3] != ' / ': + options.category = options.category+ ' / ' # treatment run = Run(verb, options.thread) kw = purge_kw(options.__dict__,('user','password','url','source','category','thread')) @@ -221,22 +223,24 @@ def main(): for ext in options.extension.split(',')[1:]: filtering = filtering + fnmatch.filter(filenames, ext) for filename in filtering: - path = os.path.abspath(os.path.join(root, filename))[len(options.source)+1:] - if not piwigo.isimage(path.replace(os.sep, ' / ')): - category = ' / '.join(path.split(os.sep)[:-1]) + pathrel = os.path.abspath(os.path.join(root, filename))[len(options.source)+1:] + pathabs = os.path.abspath(os.path.join(root, filename)) + category = options.category + ' / '.join(pathrel.split(os.sep)[:-1]) + if not piwigo.isimage(category + ' / ' + filename): run.add(piwigo.makedirs,[category,], kw) - run.add(piwigo.upload,[path, category], kw) + run.add(piwigo.upload,[pathabs, category], kw) # piwigo -> local - for category, item in piwigo.plan.iteritems(): - path = os.path.join(options.source, *category.split(' / ')) - if not os.path.exists(path): - os.makedirs(path) - for img in piwigo.images(category): - pathimg = os.path.join(path, img) - if not os.path.exists(pathimg): - run.add(piwigo.download, - ["%s / %s" % (category, str(img)), pathimg], - kw) + #for category, item in piwigo.plan.iteritems(): + # if options.category == category[0:len(options.category)]: + # path = os.path.join(options.source, *category[len(options.category):].split(' / ')) + # if not os.path.exists(path): + # os.makedirs(path) + # for img in piwigo.images(category): + # pathimg = os.path.join(path, img) + # if not os.path.exists(pathimg): + # run.add(piwigo.download, + # ["%s / %s" % (category, str(img)), pathimg], + # kw) ana.stop() except Exception as e: ana.stop() From 8a8d26c1d4f81b7cba764d369ec3609bf66e4112 Mon Sep 17 00:00:00 2001 From: fraoustin Date: Fri, 15 Apr 2016 15:19:24 +0200 Subject: [PATCH 6/7] add parameters sync-up and sync-down del paramater sync --- piwigotools/main.py | 69 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 54 insertions(+), 15 deletions(-) diff --git a/piwigotools/main.py b/piwigotools/main.py index 90280ed..338053d 100644 --- a/piwigotools/main.py +++ b/piwigotools/main.py @@ -58,10 +58,25 @@ VERBS = { "thread" : {"type":"int", "default":"1", "help":"number of thread"}, }, }, - "sync": + "sync-up": { - "usage" : "usage for verb sync", - "description" : "synchronization between path and piwigo gallery", + "usage" : "usage for verb sync-up", + "description" : "synchronization local path to piwigo gallery", + "arg" : + { + "category" : {"type":"string", "default":"", "help":"category of piwigo gallery"}, + "source" : {"type":"string", "default":".", "help":"path of picture"}, + "url" : {"type":"string", "default":"", "help":"url of piwigo gallery"}, + "user" : {"type":"string", "default":"", "help":"user of piwigo gallery"}, + "password" : {"type":"string", "default":"", "help":"password of piwigo gallery"}, + "thread" : {"type":"int", "default":"1", "help":"number of thread"}, + "extension" : {"type":"string", "default":"*.JPG,*.jpg,*.PNG,*.png,*.JPEG,*.jpeg,*.GIF,*.gif", "help":"extension for upload"}, + }, + }, + "sync-down": + { + "usage" : "usage for verb sync-down", + "description" : "synchronization piwigo gallery to local path", "arg" : { "category" : {"type":"string", "default":"", "help":"category of piwigo gallery"}, @@ -201,7 +216,7 @@ def main(): piwigo.logout() if run.error: parser.error(run.strerror) - if verb == "sync": + if verb == "sync-up": ana = Analyse('Analyze') ana.start() try: @@ -229,18 +244,42 @@ def main(): if not piwigo.isimage(category + ' / ' + filename): run.add(piwigo.makedirs,[category,], kw) run.add(piwigo.upload,[pathabs, category], kw) + ana.stop() + except Exception as e: + ana.stop() + raise e + run.start() + piwigo.logout() + if run.error: + parser.error(run.strerror) + if verb == "sync-down": + ana = Analyse('Analyze') + ana.start() + try: + piwigo = Piwigo(url=options.url) + piwigo.login(options.user, options.password) + # check + options.source = os.path.abspath(options.source) + if not os.path.isdir(options.source): + raise Exception("%s is not directory" % options.source) + piwigo.iscategory(options.category) + if len(options.category) and options.category[-1] != '/' and options.category[:-3] != ' / ': + options.category = options.category+ ' / ' + # treatment + run = Run(verb, options.thread) + kw = purge_kw(options.__dict__,('user','password','url','source','category','thread')) # piwigo -> local - #for category, item in piwigo.plan.iteritems(): - # if options.category == category[0:len(options.category)]: - # path = os.path.join(options.source, *category[len(options.category):].split(' / ')) - # if not os.path.exists(path): - # os.makedirs(path) - # for img in piwigo.images(category): - # pathimg = os.path.join(path, img) - # if not os.path.exists(pathimg): - # run.add(piwigo.download, - # ["%s / %s" % (category, str(img)), pathimg], - # kw) + for category, item in piwigo.plan.iteritems(): + if options.category == category[0:len(options.category)]: + path = os.path.join(options.source, *category[len(options.category):].split(' / ')) + if not os.path.exists(path): + os.makedirs(path) + for img in piwigo.images(category): + pathimg = os.path.join(path, img) + if not os.path.exists(pathimg): + run.add(piwigo.download, + ["%s / %s" % (category, str(img)), pathimg], + kw) ana.stop() except Exception as e: ana.stop() From a055427713ec35a194230c35eb1661f8c8528f69 Mon Sep 17 00:00:00 2001 From: fraoustin Date: Fri, 15 Apr 2016 15:24:15 +0200 Subject: [PATCH 7/7] create version 0.1.0 --- CHANGES.rst | 5 +++++ piwigotools/__init__.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2a4ca7b..63f2036 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,8 @@ +0.1.0 +===== + +- add verb sync-up and sync-down + 0.0.2 ===== diff --git a/piwigotools/__init__.py b/piwigotools/__init__.py index 5009a97..04c26b8 100644 --- a/piwigotools/__init__.py +++ b/piwigotools/__init__.py @@ -5,7 +5,7 @@ Module piwigotools """ -__version_info__ = (0, 0, 2) +__version_info__ = (0, 1, 0) __version__ = '.'.join([str(val) for val in __version_info__]) import inspect