API: allow requesting notes list in chunks

lib/Controller/ChunkCursor.php

@@ -0,0 +1,41 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OCA\Notes\Controller;
+
+use OCA\Notes\Service\MetaNote;
+
+class ChunkCursor {
+	/** @var \DateTime */
+	public $timeStart;
+	/** @var integer */
+	public $noteLastUpdate;
+	/** @var integer */
+	public $noteId;
+
+	public static function fromString(string $str) : ?ChunkCursor {
+		if (preg_match('/^(\d+)-(\d+)-(\d+)$/', $str, $matches)) {
+			$cc = new static();
+			$cc->timeStart = new \DateTime();
+			$cc->timeStart->setTimestamp((int)$matches[1]);
+			$cc->noteLastUpdate = (int)$matches[2];
+			$cc->noteId = (int)$matches[3];
+			return $cc;
+		} else {
+			return null;
+		}
+	}
+
+	public static function fromNote(\DateTime $timeStart, MetaNote $m) : ChunkCursor {
+		$cc = new static();
+		$cc->timeStart = $timeStart;
+		$cc->noteLastUpdate = $m->meta->getLastUpdate();
+		$cc->noteId = $m->note->getId();
+		return $cc;
+	}
+
+	public function toString() : string {
+		return $this->timeStart->getTimestamp() . '-' . $this->noteLastUpdate . '-' . $this->noteId;
+	}
+}

lib/Controller/Helper.php

@@ -8,6 +8,7 @@
 use OCA\Notes\Db\Meta;
 use OCA\Notes\Service\Note;
 use OCA\Notes\Service\NotesService;
+use OCA\Notes\Service\MetaNote;
 use OCA\Notes\Service\MetaService;
 use OCA\Notes\Service\Util;
 
@@ -71,28 +72,60 @@ public function getNoteData(Note $note, array $exclude = [], Meta $meta = null)
 	public function getNotesAndCategories(
 		int $pruneBefore,
 		array $exclude,
-		string $category = null
+		string $category = null,
+		int $chunkSize = 0,
+		string $chunkCursorStr = null
 	) : array {
 		$userId = $this->getUID();
+		$chunkCursor = $chunkCursorStr ? ChunkCursor::fromString($chunkCursorStr) : null;
+		$lastUpdate = $chunkCursor->timeStart ?? new \DateTime();
 		$data = $this->notesService->getAll($userId);
-		$notes = $data['notes'];
-		$metas = $this->metaService->updateAll($userId, $notes);
+		$metaNotes = $this->metaService->getAll($userId, $data['notes']);
+
+		// if a category is requested, then ignore all other notes
 		if ($category !== null) {
-			$notes = array_values(array_filter($notes, function ($note) use ($category) {
-				return $note->getCategory() === $category;
-			}));
+			$metaNotes = array_filter($metaNotes, function (MetaNote $m) use ($category) {
+				return $m->note->getCategory() === $category;
+			});
 		}
-		$notesData = array_map(function ($note) use ($metas, $pruneBefore, $exclude) {
-			$meta = $metas[$note->getId()];
-			if ($pruneBefore && $meta->getLastUpdate() < $pruneBefore) {
-				return [ 'id' => $note->getId() ];
-			} else {
-				return $this->getNoteData($note, $exclude, $meta);
-			}
-		}, $notes);
+
+		// list of notes that should be sent to the client
+		$fullNotes = array_filter($metaNotes, function (MetaNote $m) use ($pruneBefore, $chunkCursor) {
+			$isPruned = $pruneBefore && $m->meta->getLastUpdate() < $pruneBefore;
+			$noteLastUpdate = (int)$m->meta->getLastUpdate();
+			$isBeforeCursor = $chunkCursor && (
+				$noteLastUpdate < $chunkCursor->noteLastUpdate
+				|| ($noteLastUpdate === $chunkCursor->noteLastUpdate
+				&& $m->note->getId() <= $chunkCursor->noteId)
+			);
+			return !$isPruned && !$isBeforeCursor;
+		});
+
+		// sort the list for slicing the next chunk
+		uasort($fullNotes, function (MetaNote $a, MetaNote $b) {
+			return $a->meta->getLastUpdate() <=> $b->meta->getLastUpdate()
+				?: $a->note->getId() <=> $b->note->getId();
+		});
+
+		// slice the next chunk
+		$chunkedNotes = $chunkSize ? array_slice($fullNotes, 0, $chunkSize, true) : $fullNotes;
+
+		// if the chunk does not contain all remaining notes, then generate new chunk cursor
+		$newChunkCursor = count($chunkedNotes) !== count($fullNotes)
+			? ChunkCursor::fromNote($lastUpdate, end($chunkedNotes))
+			: null;
+
+		// load data for the current chunk
+		$notesData = array_map(function (MetaNote $m) use ($exclude) {
+			return $this->getNoteData($m->note, $exclude, $m->meta);
+		}, $chunkedNotes);
+
 		return [
-			'notes' => $notesData,
 			'categories' => $data['categories'],
+			'notesAll' => $metaNotes,
+			'notesData' => $notesData,
+			'lastUpdate' => $lastUpdate,
+			'chunkCursor' => $newChunkCursor,
 		];
 	}
 

lib/Controller/NotesApiController.php

@@ -5,6 +5,7 @@
 namespace OCA\Notes\Controller;
 
 use OCA\Notes\Service\NotesService;
+use OCA\Notes\Service\MetaNote;
 use OCA\Notes\Service\MetaService;
 use OCA\Notes\Service\SettingsService;
 
@@ -45,16 +46,36 @@ public function __construct(
 	 * @CORS
 	 * @NoCSRFRequired
 	 */
-	public function index(?string $category = null, string $exclude = '', int $pruneBefore = 0) : JSONResponse {
-		return $this->helper->handleErrorResponse(function () use ($category, $exclude, $pruneBefore) {
+	public function index(
+		?string $category = null,
+		string $exclude = '',
+		int $pruneBefore = 0,
+		int $chunkSize = 0,
+		string $chunkCursor = null
+	) : JSONResponse {
+		return $this->helper->handleErrorResponse(function () use (
+			$category,
+			$exclude,
+			$pruneBefore,
+			$chunkSize,
+			$chunkCursor
+		) {
 			$exclude = explode(',', $exclude);
-			$now = new \DateTime(); // this must be before loading notes if there are concurrent changes possible
-			$data = $this->helper->getNotesAndCategories($pruneBefore, $exclude, $category);
-			$notesData = $data['notes'];
-			$etag = md5(json_encode($notesData));
-			return (new JSONResponse($notesData))
-				->setLastModified($now)
-				->setETag($etag);
+			$data = $this->helper->getNotesAndCategories($pruneBefore, $exclude, $category, $chunkSize, $chunkCursor);
+			$notesData = $data['notesData'];
+			if (!$data['chunkCursor']) {
+				// if last chunk, then send all notes (pruned)
+				$notesData += array_map(function (MetaNote $m) {
+					return [ 'id' => $m->note->getId() ];
+				}, $data['notesAll']);
+			}
+			$response = new JSONResponse(array_values($notesData));
+			$response->setLastModified($data['lastUpdate']);
+			$response->setETag(md5(json_encode($notesData)));
+			if ($data['chunkCursor']) {
+				$response->addHeader('X-Notes-Chunk-Cursor', $data['chunkCursor']->toString());
+			}
+			return $response;
 		});
 	}
 

lib/Controller/NotesController.php

@@ -55,7 +55,6 @@ public function __construct(
 	public function index(int $pruneBefore = 0) : JSONResponse {
 		return $this->helper->handleErrorResponse(function () use ($pruneBefore) {
 			$userId = $this->helper->getUID();
-			$now = new \DateTime(); // this must be before loading notes if there are concurrent changes possible
 			$settings = $this->settingsService->getAll($userId);
 
 			$lastViewedNote = (int) $this->settings->getUserValue(
@@ -64,32 +63,33 @@ public function index(int $pruneBefore = 0) : JSONResponse {
 				'notesLastViewedNote'
 			);
 			$errorMessage = null;
-			$notes = null;
-			$categories = null;
+			$nac = null;
 
 			try {
 				$nac = $this->helper->getNotesAndCategories($pruneBefore, [ 'etag', 'content' ]);
-				[ 'notes' => $notes, 'categories' => $categories ] = $nac;
 			} catch (\Throwable $e) {
 				$this->helper->logException($e);
 				$errorMessage = $this->l10n->t('Reading notes from filesystem has failed.').' ('.get_class($e).')';
 			}
 
-			if ($errorMessage === null && $lastViewedNote && is_array($notes) && !count($notes)) {
+			if ($errorMessage === null && $lastViewedNote
+				&& is_array($nac) && is_array($nac['notesAll']) && !count($nac['notesAll'])
+			) {
 				$this->settings->deleteUserValue($userId, $this->appName, 'notesLastViewedNote');
 				$lastViewedNote = 0;
 			}
 
 			$result = [
-				'notes' => $notes,
-				'categories' => $categories,
+				'notesData' => array_values($nac['notesData'] ?? null),
+				'noteIds' => array_keys($nac['notesAll']),
+				'categories' => $nac['categories'] ?? null,
 				'settings' => $settings,
 				'lastViewedNote' => $lastViewedNote,
 				'errorMessage' => $errorMessage,
 			];
 			$etag = md5(json_encode($result));
 			return (new JSONResponse($result))
-				->setLastModified($now)
+				->setLastModified($nac['lastUpdate'] ?? null)
 				->setETag($etag)
 			;
 		});

lib/Service/MetaNote.php

@@ -0,0 +1,19 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OCA\Notes\Service;
+
+use OCA\Notes\Db\Meta;
+
+class MetaNote {
+	/** @var Note */
+	public $note;
+	/** @var Meta */
+	public $meta;
+
+	public function __construct(Note $note, Meta $meta) {
+		$this->note = $note;
+		$this->meta = $meta;
+	}
+}

lib/Service/MetaService.php

@@ -26,7 +26,7 @@
  * time and the next synchronization time. However, this is totally sufficient
  * for this purpose.
  *
- * Therefore, on synchronization, the method `MetaService.updateAll` is called.
+ * Therefore, on synchronization, the method `MetaService.getAll` is called.
  * It generates an ETag for each note and compares it with the ETag from
  * `notes_meta` database table in order to detect changes (or creates an entry
  * if not existent). If there are changes, the ETag is updated and `LastUpdate`
@@ -61,35 +61,35 @@ public function deleteByNote(int $id) : void {
 		$this->metaMapper->deleteByNote($id);
 	}
 
-	public function updateAll(string $userId, array $notes, bool $forceUpdate = false) : array {
+	public function getAll(string $userId, array $notes, bool $forceUpdate = false) : array {
 		// load data
 		$metas = $this->metaMapper->getAll($userId);
 		$metas = $this->getIndexedArray($metas, 'fileId');
-		$notes = $this->getIndexedArray($notes, 'id');
+		$result = [];
 
 		// delete obsolete notes
 		foreach ($metas as $id => $meta) {
 			if (!array_key_exists($id, $notes)) {
 				// DELETE obsolete notes
 				$this->metaMapper->delete($meta);
-				unset($metas[$id]);
 			}
 		}
 
 		// insert/update changes
 		foreach ($notes as $id => $note) {
 			if (!array_key_exists($id, $metas)) {
 				// INSERT new notes
-				$metas[$note->getId()] = $this->createMeta($userId, $note);
+				$meta = $this->createMeta($userId, $note);
 			} else {
 				// UPDATE changed notes
 				$meta = $metas[$id];
 				if ($this->updateIfNeeded($meta, $note, $forceUpdate)) {
 					$this->metaMapper->update($meta);
 				}
 			}
+			$result[$id] = new MetaNote($note, $meta);
 		}
-		return $metas;
+		return $result;
 	}
 
 	public function update(string $userId, Note $note) : Meta {

lib/Service/NotesService.php

@@ -26,9 +26,7 @@ public function __construct(
 	public function getAll(string $userId) : array {
 		$notesFolder = $this->getNotesFolder($userId);
 		$data = $this->gatherNoteFiles($notesFolder);
-		$fileIds = array_map(function (File $file) : int {
-			return $file->getId();
-		}, $data['files']);
+		$fileIds = array_keys($data['files']);
 		// pre-load tags for all notes (performance improvement)
 		$this->noteUtil->getTagService()->loadTags($fileIds);
 		$notes = array_map(function (File $file) use ($notesFolder) : Note {
@@ -159,10 +157,10 @@ private static function gatherNoteFiles(Folder $folder, string $categoryPrefix =
 				$subCategory = $categoryPrefix . $node->getName();
 				$data['categories'][] = $subCategory;
 				$data_sub = self::gatherNoteFiles($node, $subCategory . '/');
-				$data['files'] = array_merge($data['files'], $data_sub['files']);
-				$data['categories'] = array_merge($data['categories'], $data_sub['categories']);
+				$data['files'] = $data['files'] + $data_sub['files'];
+				$data['categories'] = $data['categories'] + $data_sub['categories'];
 			} elseif (self::isNote($node)) {
-				$data['files'][] = $node;
+				$data['files'][$node->getId()] = $node;
 			}
 		}
 		return $data;

src/NotesService.js

@@ -74,8 +74,8 @@ export const fetchNotes = () => {
 		.then(response => {
 			store.commit('setSettings', response.data.settings)
 			store.commit('setCategories', response.data.categories)
-			if (response.data.notes !== null) {
-				store.dispatch('updateNotes', response.data.notes)
+			if (response.data.noteIds !== null) {
+				store.dispatch('updateNotes', { noteIds: response.data.noteIds, notes: response.data.notesData })
 			}
 			if (response.data.errorMessage) {
 				showError(t('notes', 'Error from Nextcloud server: {msg}', { msg: response.data.errorMessage }))

src/store/notes.js

@@ -123,20 +123,15 @@ const mutations = {
 }
 
 const actions = {
-	updateNotes(context, notes) {
-		const noteIds = {}
+	updateNotes(context, { noteIds, notes }) {
 		// add/update new notes
 		for (const note of notes) {
-			noteIds[note.id] = true
 			// TODO check for parallel (local) changes!
-			// only update, if note has changes (see API "pruneBefore")
-			if (note.title !== undefined) {
-				context.commit('updateNote', note)
-			}
+			context.commit('updateNote', note)
 		}
 		// remove deleted notes
 		context.state.notes.forEach(note => {
-			if (noteIds[note.id] === undefined) {
+			if (!noteIds.includes(note.id)) {
 				context.commit('removeNote', note.id)
 			}
 		})