nncli

NextCloud Notes Command Line Interface
git clone git://git.danielmoch.com/nncli.git
Log | Files | Refs | LICENSE

commit cdd936c474693ac83cac8c1754e50e18fa727f5e
parent fb58581b652746ec10fe9bd9c4babdffa52a67ad
Author: Eric Davis <edavis@insanum.com>
Date:   Tue,  8 Jul 2014 17:31:45 -0700

simpler sync worker that always performs a two way sync

Diffstat:
Mnotes_db.py | 234++++++++++++++++++++++++++++++-------------------------------------------------
Msncli.py | 36+++++++++++-------------------------
2 files changed, 99 insertions(+), 171 deletions(-)

diff --git a/notes_db.py b/notes_db.py @@ -2,23 +2,12 @@ # copyright 2012 by Charl P. Botha <cpbotha@vxlabs.com> # new BSD license -import copy -import glob -import os -import json -from Queue import Queue, Empty -import re +import os, time, re, glob, json, copy +import utils import simplenote simplenote.NOTE_FETCH_LENGTH=100 from simplenote import Simplenote -from threading import Thread -import time -import utils - -class SyncError(RuntimeError): - pass - class ReadError(RuntimeError): pass @@ -29,8 +18,9 @@ class NotesDB(): """NotesDB will take care of the local notes database and syncing with SN. """ def __init__(self, config, log): - self.config = config - self.log = log + self.config = config + self.log = log + self.last_sync = 0 # set to zero to trigger a full sync # create db dir if it does not exist if not os.path.exists(self.config.get_config('db_path')): @@ -332,7 +322,7 @@ def set_note_deleted(self, key): n['deleted'] = 1 n['modifydate'] = time.time() self.flag_what_changed(n, 'deleted') - self.log('Note trashed') + self.log('Note trashed (key={0})'.format(key)) def set_note_content(self, key, content): n = self.notes[key] @@ -341,7 +331,7 @@ def set_note_content(self, key, content): n['content'] = content n['modifydate'] = time.time() self.flag_what_changed(n, 'content') - self.log('Note content updated') + self.log('Note content updated (key={0})'.format(key)) def set_note_tags(self, key, tags): n = self.notes[key] @@ -351,7 +341,7 @@ def set_note_tags(self, key, tags): n['tags'] = tags n['modifydate'] = time.time() self.flag_what_changed(n, 'tags') - self.log('Note tags updated') + self.log('Note tags updated (key={0})'.format(key)) def set_note_pinned(self, key, pinned): n = self.notes[key] @@ -366,7 +356,7 @@ def set_note_pinned(self, key, pinned): systemtags.remove('pinned') n['modifydate'] = time.time() self.flag_what_changed(n, 'systemtags') - self.log('Note pinned' if pinned else 'Note unpinned') + self.log('Note {0} (key={1})'.format('pinned' if pinned else 'unpinned', key)) def set_note_markdown(self, key, markdown): n = self.notes[key] @@ -381,7 +371,7 @@ def set_note_markdown(self, key, markdown): systemtags.remove('markdown') n['modifydate'] = time.time() self.flag_what_changed(n, 'systemtags') - self.log('Note markdown flagged' if markdown else 'Note markdown unflagged') + self.log('Note markdown {0} (key={1})'.format('flagged' if markdown else 'unflagged', key)) def helper_key_to_fname(self, k): return os.path.join(self.config.get_config('db_path'), k) + '.json' @@ -394,97 +384,7 @@ def helper_save_note(self, k, note): # record that we saved this to disc. note['savedate'] = time.time() - def sync_note(self, k, check_for_new): - """Sync a single note with the server. - - Update existing note in memory with the returned data. - This is a sychronous (blocking) call. - """ - - note = self.notes[k] - - # update if note has no key or it has been modified since last sync - if not note.get('key') or \ - float(note.get('modifydate')) > float(note.get('syncdate')): - - if not note.get('key'): - self.log('Sync worker: creating note (temp key = {0})'.format(k)) - - self.log('Sync worker: updating note {0}'.format(k)) - - # only send required fields - cn = copy.deepcopy(note) - if 'what_changed' in note: - del note['what_changed'] - - if 'minversion' in cn: - del cn['minversion'] - del cn['createdate'] - del cn['syncdate'] - del cn['savedate'] - - if 'what_changed' in cn: - if 'deleted' not in cn['what_changed']: - del cn['deleted'] - if 'systemtags' not in cn['what_changed']: - del cn['systemtags'] - if 'tags' not in cn['what_changed']: - del cn['tags'] - if 'content' not in cn['what_changed']: - del cn['content'] - del cn['what_changed'] - - uret = self.simplenote.update_note(cn) - - if uret[1] == 0: # success! - n = uret[0] - # if content was unchanged there'll be no content sent back - new_content = True if n.get('content', None) else False - # store when we've synced - n['syncdate'] = time.time() - note.update(n) - self.log('Sync worker: updated note {0}'.format(k)) - return (k, new_content) - else: - self.log('ERROR: Sync worker: update failed for note {0}'.format(k)) - return None - - else: # note has not been modified, check for an update on the server - - if not check_for_new: - return None - - self.log('Sync worker: checking for server update of note {0}'.format(k)) - - gret = self.simplenote.get_note(note['key']) - - if gret[1] == 0: # success! - n = gret[0] - if int(n.get('syncnum')) > int(note.get('syncnum')): - # store what we pulled down from the server - n['syncdate'] = time.time() - note.update(n) - self.log('Sync worker: server had an update for note {0}'.format(k)) - return (k, True) - else: - self.log('Sync worker: server in sync with note {0}'.format(k)) - return (k, False) - else: - self.log('ERROR: Sync worker: get failed for note {0}'.format(k)) - return None - - # sync worker thread... - def sync_worker(self): - self.log('Sync worker: started') - while True: - time.sleep(5) - now = time.time() - for k,n in self.notes.items(): - modifydate = float(n.get('modifydate', -1)) - if (now - modifydate) > 3: - self.sync_note(k, False) - - def sync_full(self): + def sync_notes(self, server_sync=True, full_sync=True): """Perform a full bi-directional sync with server. This follows the recipe in the SimpleNote 2.0 API documentation. @@ -521,7 +421,11 @@ def sync_full(self): server_keys = {} now = time.time() - self.log('Starting full sync') + sync_start_time = time.time() + sync_errors = 0 + + if server_sync and full_sync: + self.log("Starting full sync") # 1. for any note changed locally, including new notes: # save note to server, update note with response @@ -529,7 +433,43 @@ def sync_full(self): n = self.notes[local_key] if not n.get('key') or \ float(n.get('modifydate')) > float(n.get('syncdate')): - uret = self.simplenote.update_note(n) + + savedate = float(n.get('savedate')) + if float(n.get('modifydate')) > savedate or \ + float(n.get('syncdate')) > savedate: + # this will trigger a save to disk after sync algorithm + # we want this note saved even if offline or sync fails + local_updates[local_key] = True + + if not server_sync: + # the 'what_changed' field will be written to disk and + # picked up whenever the next full server sync occurs + continue + + # only send required fields + cn = copy.deepcopy(n) + if 'what_changed' in n: + del n['what_changed'] + + if 'minversion' in cn: + del cn['minversion'] + del cn['createdate'] + del cn['syncdate'] + del cn['savedate'] + + if 'what_changed' in cn: + if 'deleted' not in cn['what_changed']: + del cn['deleted'] + if 'systemtags' not in cn['what_changed']: + del cn['systemtags'] + if 'tags' not in cn['what_changed']: + del cn['tags'] + if 'content' not in cn['what_changed']: + del cn['content'] + del cn['what_changed'] + + uret = self.simplenote.update_note(cn) + if uret[1] == 0: # success # if this is a new note our local key is not valid anymore # merge the note we got back (content could be empty) @@ -540,33 +480,36 @@ def sync_full(self): n['syncdate'] = now self.notes[k] = n - # whatever the case may be, k is now updated local_updates[k] = True - if local_key != k: # if local_key was a different key it should be deleted local_deletes[local_key] = True + local_updates[local_key] = False self.log('Synced note to server (key={0})'.format(local_key)) else: self.log('ERROR: Failed to sync note to server (key={0})'.format(local_key)) + sync_errors += 1 # 2. get the note index - self.log('Retrieving note list from server') - nl = self.simplenote.get_note_list() - if nl[1] == 0: # success - self.log('Retrieved full note list from server') + if not server_sync: + nl = [] else: - self.log('ERROR: Could not get note list from server') - return 1 - nl = nl[0] + nl = self.simplenote.get_note_list(since=None if full_sync else self.last_sync) + + if nl[1] == 0: # success + nl = nl[0] + else: + self.log('ERROR: Failed to get note list from server') + sync_errors += 1 + nl = [] # 3. for each remote note # if remote syncnum > local syncnum || # a new note and key is not in local store # retrieve note, update note with response len_nl = len(nl) - sync_from_server_errors = 0 + sync_errors = 0 for note_index, n in enumerate(nl): k = n.get('key') server_keys[k] = True @@ -585,7 +528,7 @@ def sync_full(self): self.log('Synced newer note from server (key={0})'.format(k)) else: self.log('ERROR: Failed to sync newer note from server (key={0})'.format(k)) - sync_from_server_errors += 1 + sync_errors += 1 else: # this is a new note gret = self.simplenote.get_note(k) @@ -597,14 +540,16 @@ def sync_full(self): self.log('Synced new note from server (key={0})'.format(k)) else: self.log('ERROR: Failed syncing new note from server (key={0})'.format(k)) - sync_from_server_errors += 1 + sync_errors += 1 # 4. for each local note not in the index # PERMANENT DELETE, remove note from local store - for local_key in self.notes.keys(): - if local_key not in server_keys: - del self.notes[local_key] - local_deletes[local_key] = True + # Only do this when a full sync (i.e. entire index) is performed! + if server_sync and full_sync: + for local_key in self.notes.keys(): + if local_key not in server_keys: + del self.notes[local_key] + local_deletes[local_key] = True # sync done, now write changes to db_path @@ -613,31 +558,28 @@ def sync_full(self): self.helper_save_note(k, self.notes[k]) except WriteError, e: raise WriteError (str(e)) + self.log("Saved note to disk (key={0})".format(k)) for k in local_deletes.keys(): fn = self.helper_key_to_fname(k) if os.path.exists(fn): os.unlink(fn) + self.log("Deleted note from disk (key={0})".format(k)) + + if not sync_errors: + self.last_sync = sync_start_time - self.log('Full sync complete') + if server_sync and full_sync: + self.log("Full sync completed") - return sync_from_server_errors + return sync_errors - # save worker thread... - def save_worker(self): - self.log('Save worker: started') + # sync worker thread... + def sync_worker(self, do_sync): + time.sleep(1) # give some time to wait for GUI initialization + self.log('Sync worker: started') while True: + self.sync_notes(server_sync=do_sync, + full_sync=True if not self.last_sync else False) time.sleep(5) - #self.log('Save worker: checking for work') - for k,n in self.notes.items(): - savedate = float(n.get('savedate')) - if float(n.get('modifydate')) > savedate or \ - float(n.get('syncdate')) > savedate: - try: - # this will write the new savedate into the note - self.helper_save_note(k, n) - self.log('Save worker: saved note {0}'.format(k)) - except WriteError, e: - self.log('ERROR: Failed to write file to the filesystem!') - os._exit(1) diff --git a/sncli.py b/sncli.py @@ -7,7 +7,7 @@ import utils, temp from config import Config from simplenote import Simplenote -from notes_db import NotesDB, SyncError, ReadError, WriteError +from notes_db import NotesDB, ReadError, WriteError from logging.handlers import RotatingFileHandler class sncli: @@ -37,14 +37,8 @@ def __init__(self): self.log(str(e)) sys.exit(1) - def sync_full(self): - self.ndb.sync_full() - - def gui_sync_full_threaded(self): - thread.start_new_thread(self.ndb.sync_full, ()) - - def gui_sync_full_initial(self, loop, arg): - self.gui_sync_full_threaded() + def sync_notes(self): + self.ndb.sync_notes() def get_editor(self): editor = self.config.get_config('editor') @@ -175,14 +169,14 @@ def gui_tags_input(self, tags): self.gui_body_focus() self.master_frame.keypress = self.gui_frame_keypress if tags != None: - if self.gui_body_get().__class__ != view_titles.ViewTitles: + if self.gui_body_get().__class__ == view_titles.ViewTitles: note = self.view_titles.note_list[self.view_titles.focus_position].note else: # self.gui_body_get().__class__ == view_note.ViewNote: note = self.view_note.note self.ndb.set_note_tags(note['key'], tags) - if self.gui_body_get().__class__ != view_titles.ViewTitles: + if self.gui_body_get().__class__ == view_titles.ViewTitles: self.view_titles.update_note_title(None) else: # self.gui_body_get().__class__ == view_note.ViewNote: self.view_note.update_note(note['key']) @@ -218,7 +212,7 @@ def gui_frame_keypress(self, size, key): self.gui_switch_frame_body(self.view_help) elif key == self.config.get_keybind('sync'): - self.gui_sync_full_threaded() + self.ndb.last_sync = 0 elif key == self.config.get_keybind('view_log'): self.gui_switch_frame_body(self.view_log) @@ -601,13 +595,8 @@ def gui_init_view(self, loop, arg): self.master_frame.keypress = self.gui_frame_keypress self.gui_body_set(self.view_titles) - self.thread_save.start() self.thread_sync.start() - if self.do_sync: - # start full sync after initial view is up - self.sncli_loop.set_alarm_in(1, self.gui_sync_full_initial, None) - def gui_clear(self): self.sncli_loop.widget = urwid.Filler(urwid.Text(u'')) self.sncli_loop.draw_screen() @@ -623,8 +612,7 @@ def gui_stop(self): def gui(self, do_sync): - self.do_gui = True - self.do_sync = do_sync + self.do_gui = True self.last_view = [] self.status_bar = self.config.get_config('status_bar') @@ -632,10 +620,8 @@ def gui(self, do_sync): self.log_alarms = 0 self.log_lock = threading.Lock() - self.thread_save = threading.Thread(target=self.ndb.save_worker) - self.thread_save.setDaemon(True) - - self.thread_sync = threading.Thread(target=self.ndb.sync_worker) + self.thread_sync = threading.Thread(target=self.ndb.sync_worker, + args=[do_sync]) self.thread_sync.setDaemon(True) self.view_titles = \ @@ -762,7 +748,7 @@ def save_new_note(content): if content and content != u'\n': self.log(u'New note created') self.ndb.create_note(content) - self.ndb.sync_full() + self.ndb.sync_notes() if from_stdin: @@ -832,7 +818,7 @@ def main(argv): def sncli_start(sync): sn = sncli() - if sync: sn.sync_full() + if sync: sn.sync_notes() return sn if args[0] == 'list':