Migrate offline files to PMTiles (#1963)

Resolves #1961

* Main commit to see how PMTiles works

* Fixed issue in android 13 when opening a file, added logs.

* Updated outofdate lock file

* Add ability to load a style.json file from device.

* Update backend

* Remove unneded file, revert appveyor

* Add missing logic in frontend

* Add xml method doc

* Make custom protocol use pmtiles first

* Fix pmtiles not showing

* Add a flag to signal if pm tiles were ever downloaded, fix lint

* Fix tests

* Move mbtiles code to a separate class to allow easier removal in the future.

* Fix tests

IsraelHiking.API/Controllers/FilesController.cs

@@ -120,19 +120,20 @@ private async Task<DataContainerPoco> ConvertToDataContainer(byte[] data, string
         /// Get a list of files that need to be downloaded since they are out dated
         /// </summary>
         /// <param name="lastModified"></param>
+        /// <param name="pmtiles">This parameter was added in 1.2024</param>
         /// <returns></returns>
         [HttpGet]
         [Route("offline")]
         [Authorize]
-        public async Task<IActionResult> GetOfflineFiles([FromQuery] DateTime lastModified)
+        public async Task<IActionResult> GetOfflineFiles([FromQuery] DateTime lastModified, [FromQuery] bool pmtiles)
         {
             if (!await _receiptValidationGateway.IsEntitled(User.Identity?.Name))
             {
                 _logger.LogInformation($"Unable to get the list of offline files for user: {User.Identity?.Name} since the user is not entitled, date: {lastModified}");
                 return Forbid();
             }
-            _logger.LogInformation($"Getting the list of offline files for user: {User.Identity?.Name}, date: {lastModified}");
-            return Ok(_offlineFilesService.GetUpdatedFilesList(lastModified));
+            _logger.LogInformation($"Getting the list of offline files for user: {User.Identity?.Name}, date: {lastModified}, pmtiles: {pmtiles}");
+            return Ok(_offlineFilesService.GetUpdatedFilesList(lastModified, pmtiles));
         }
 
         /// <summary>

IsraelHiking.API/Services/IOfflineFilesService.cs

@@ -20,7 +20,8 @@ public interface IOfflineFilesService
         /// Get a list of files that have been updated since a given date
         /// </summary>
         /// <param name="lastModifiedDate">The date to check against</param>
+        /// <param name="pmtiles"></param>
         /// <returns>A list of file names</returns>
-        Dictionary<string, DateTime> GetUpdatedFilesList(DateTime lastModifiedDate);
+        Dictionary<string, DateTime> GetUpdatedFilesList(DateTime lastModifiedDate, bool pmtiles);
     }
 }

IsraelHiking.API/Services/OfflineFilesService.cs

@@ -37,7 +37,7 @@ public OfflineFilesService(IFileSystemHelper fileSystemHelper,
         }
 
         /// <inheritdoc/>
-        public Dictionary<string, DateTime> GetUpdatedFilesList(DateTime lastModifiedDate)
+        public Dictionary<string, DateTime> GetUpdatedFilesList(DateTime lastModifiedDate, bool pmtiles)
         {
             var filesDictionary = new Dictionary<string, DateTime>();
             var contents = _fileProvider.GetDirectoryContents(string.Empty);
@@ -51,7 +51,7 @@ public Dictionary<string, DateTime> GetUpdatedFilesList(DateTime lastModifiedDat
                 {
                     continue;
                 }
-                if (content.Name.EndsWith(".mbtiles") || content.Name.StartsWith("style"))
+                if (content.Name.EndsWith(pmtiles ? ".pmtiles" : ".mbtiles") || content.Name.StartsWith("style"))
                 {
                     filesDictionary[content.Name] = content.LastModified.DateTime;
                 }

IsraelHiking.Web/package.json

@@ -82,6 +82,7 @@
     "photoswipe": "^5.4.2",
     "piexifjs": "1.0.6",
     "platform": "^1.3.6",
+    "pmtiles": "^2.10.0",
     "proj4": "^2.9.0",
     "rxjs": "^7.8.1",
     "tailwindcss": "^3.3.3",

IsraelHiking.Web/src/application/application.module.ts

@@ -95,6 +95,8 @@ import { AudioPlayerFactory } from "./services/audio-player.factory";
 import { GlobalErrorHandler } from "./services/global-error.handler";
 import { OverpassTurboService } from "./services/overpass-turbo.service";
 import { ImageAttributionService } from "./services/image-attribution.service";
+import { PmTilesService } from "./services/pmtiles.service";
+import { MBTilesService } from "./services/mbtiles.service";
 // interactions
 import { RouteEditPoiInteraction } from "./components/intercations/route-edit-poi.interaction";
 import { RouteEditRouteInteraction } from "./components/intercations/route-edit-route.interaction";
@@ -306,6 +308,8 @@ const initializeApplication = (injector: Injector) => async () => {
             OfflineFilesDownloadService,
             OverpassTurboService,
             ImageAttributionService,
+            PmTilesService,
+            MBTilesService,
             AudioPlayerFactory,
             FileSystemWrapper,
             // eslint-disable-next-line

IsraelHiking.Web/src/application/components/dialogs/files-shares-dialog.component.ts

@@ -57,15 +57,20 @@ export class FilesSharesDialogComponent extends BaseMapComponent {
             }
             return;
         }
-        if (file.name.endsWith(".mbtiles") && offlineState.isOfflineAvailable) {
+        if (file.name.endsWith(".pmtiles")) {
             this.toastService.info(this.resources.openingAFilePleaseWait);
-            const dbFileName = file.name.replace(".mbtiles", ".db");
-            await this.fileService.storeFileToCache(dbFileName, file);
-            await this.databaseService.moveDownloadedDatabaseFile(dbFileName);
+            await this.fileService.storeFileToCache(file.name, file);
+            await this.fileService.moveFileFromCacheToDataDirectory(file.name);
             this.toastService.confirm({ type: "Ok", message: this.resources.finishedOpeningTheFile });
             this.store.dispatch(new SetOfflineMapsLastModifiedDateAction(new Date(file.lastModified)));
             return;
         }
+        if (file.name.endsWith(".json")) {
+            this.toastService.info(this.resources.openingAFilePleaseWait);
+            await this.fileService.writeStyle(file.name, await this.fileService.getFileContent(file));
+            this.toastService.confirm({ type: "Ok", message: this.resources.finishedOpeningTheFile });
+            return;
+        }
         try {
             await this.fileService.addRoutesFromFile(file);
             this.matDialogRef.close();

IsraelHiking.Web/src/application/models/state/offline-state.d.ts

@@ -19,4 +19,10 @@
      * A Queue to represent the IDs of items waiting to be uploaded to the server
      */
     uploadPoiQueue: string[];
+    /**
+     * Marks if PMTiles were ever downloaded.
+     * This flag is used once to allow downloading all the files from the server.
+     * It was added 1.2024
+     */
+    isPmtilesDownloaded: boolean;
 };

IsraelHiking.Web/src/application/reducers/initial-state.ts

@@ -159,7 +159,8 @@ export const initialState =
             lastModifiedDate: null,
             poisLastModifiedDate: null,
             shareUrlsLastModifiedDate: null,
-            uploadPoiQueue: []
+            uploadPoiQueue: [],
+            isPmtilesDownloaded: false
         },
         uiComponentsState: {
             drawingVisible: false,

IsraelHiking.Web/src/application/reducers/offline.reducer.ts

@@ -54,6 +54,7 @@ export class OfflineReducer {
     public setOfflineMpasLastModifiedDate(ctx: StateContext<OfflineState>, action: SetOfflineMapsLastModifiedDateAction) {
         ctx.setState(produce(ctx.getState(), lastState => {
             lastState.lastModifiedDate = action.lastModifiedDate;
+            lastState.isPmtilesDownloaded = true;
             return lastState;
         }));
     }

IsraelHiking.Web/src/application/services/database.service.ts

@@ -1,14 +1,14 @@
 import { Injectable } from "@angular/core";
 import { Store } from "@ngxs/store";
 import { debounceTime } from "rxjs/operators";
-import { CapacitorSQLite, SQLiteDBConnection, SQLiteConnection} from "@capacitor-community/sqlite";
-import { gunzipSync } from "fflate";
 import Dexie from "dexie";
 import deepmerge from "deepmerge";
 import maplibregl from "maplibre-gl";
 
 import { LoggingService } from "./logging.service";
 import { RunningContextService } from "./running-context.service";
+import { PmTilesService } from "./pmtiles.service";
+import { MBTilesService } from "./mbtiles.service";
 import { POPULARITY_HEATMAP, initialState } from "../reducers/initial-state";
 import { ClearHistoryAction } from "../reducers/routes.reducer";
 import { SetSelectedPoiAction, SetSidebarAction } from "../reducers/poi.reducer";
@@ -41,25 +41,22 @@ export class DatabaseService {
     private imagesDatabase: Dexie;
     private shareUrlsDatabase: Dexie;
     private tracesDatabase: Dexie;
-    private sourceDatabases: Map<string, Promise<SQLiteDBConnection>>;
     private updating: boolean;
-    private sqlite: SQLiteConnection;
 
     constructor(private readonly loggingService: LoggingService,
                 private readonly runningContext: RunningContextService,
+                private readonly pmTilesService: PmTilesService,
+                private readonly mbtilesService: MBTilesService,
                 private readonly store: Store) {
         this.updating = false;
-        this.sourceDatabases = new Map<string, Promise<SQLiteDBConnection>>();
     }
 
     public async initialize() {
+        this.mbtilesService.initialize();
         this.stateDatabase = new Dexie(DatabaseService.STATE_DB_NAME);
         this.stateDatabase.version(1).stores({
             state: "id"
         });
-        if (this.runningContext.isCapacitor) {
-            this.sqlite = new SQLiteConnection(CapacitorSQLite);
-        }
         this.poisDatabase = new Dexie(DatabaseService.POIS_DB_NAME);
         this.poisDatabase.version(1).stores({
             pois: DatabaseService.POIS_ID_COLUMN + "," + DatabaseService.POIS_LOCATION_COLUMN,
@@ -79,11 +76,11 @@ export class DatabaseService {
         this.tracesDatabase.version(1).stores({
             traces: "id",
         });
-        this.initCustomTileLoadFunction();
         if (this.runningContext.isIFrame) {
             this.store.reset(initialState);
             return;
         }
+        this.initCustomTileLoadFunction();
         let storedState = initialState;
         const dbState = await this.stateDatabase.table(DatabaseService.STATE_TABLE_NAME).get(DatabaseService.STATE_DOC_ID);
         if (dbState != null) {
@@ -111,6 +108,8 @@ export class DatabaseService {
                     const message = `Tile is not in DB: ${params.url}`;
                     callback(new Error(message));
                 }
+            }).catch((err) => {
+                callback(err);
             });
             return { cancel: () => { } };
         });
@@ -123,27 +122,7 @@ export class DatabaseService {
         this.store.dispatch(new SetSidebarAction(false));
         const finalState = this.store.snapshot() as ApplicationState;
         await this.updateState(finalState);
-        for (const dbKey of this.sourceDatabases.keys()) {
-            await this.closeDatabase(dbKey);
-        }
-    }
-
-    public async closeDatabase(dbKey: string) {
-        this.loggingService.info("[Database] Closing " + dbKey);
-        if (!this.sourceDatabases.has(dbKey)) {
-            this.loggingService.info(`[Database] ${dbKey} was never opened`);
-            return;
-        }
-        try {
-            const db = await this.sourceDatabases.get(dbKey);
-            await db.close();
-            this.loggingService.info("[Database] Closed succefully: " + dbKey);
-            await this.sqlite.closeConnection(dbKey + ".db", true);
-            this.loggingService.info("[Database] Connection closed succefully: " + dbKey);
-            this.sourceDatabases.delete(dbKey);
-        } catch (ex) {
-            this.loggingService.error(`[Database] Unable to close ${dbKey}, ${(ex as Error).message}`);
-        }
+        await this.mbtilesService.uninitialize();
     }
 
     private async updateState(state: ApplicationState) {
@@ -163,72 +142,13 @@ export class DatabaseService {
         }
     }
 
-    private getSourceNameFromUrl(url: string) {
-        return url.replace("custom://", "").split("/")[0];
-    }
-
     public async getTile(url: string): Promise<ArrayBuffer> {
-        const splitUrl = url.split("/");
-        const dbName = this.getSourceNameFromUrl(url);
-        const z = +splitUrl[splitUrl.length - 3];
-        const x = +splitUrl[splitUrl.length - 2];
-        const y = +(splitUrl[splitUrl.length - 1].split(".")[0]);
-
-        return this.getTileFromDatabase(dbName, z, x, y);
-    }
-
-    private async getTileFromDatabase(dbName: string, z: number, x: number, y: number): Promise<ArrayBuffer> {
-        const db = await this.getDatabase(dbName);
-
-        const params = [z, x, Math.pow(2, z) - y - 1];
-        const queryresults = await db.query("SELECT HEX(tile_data) as tile_data_hex FROM tiles " +
-                "WHERE zoom_level = ? AND tile_column = ? AND tile_row = ? limit 1",
-                params);
-        if (queryresults.values.length !== 1) {
-            throw new Error("Unable to get tile from database");
-        }
-        const hexData = queryresults.values[0].tile_data_hex;
-        let binData = new Uint8Array(hexData.match(/.{1,2}/g).map((byte: string) => parseInt(byte, 16)));
-        const isGzipped = binData[0] === 0x1f && binData[1] === 0x8b;
-        if (isGzipped) {
-            binData = gunzipSync(binData);
-        }
-        return binData.buffer;
-    }
-
-    private async getDatabase(dbName: string): Promise<SQLiteDBConnection> {
-        if (this.sourceDatabases.has(dbName)) {
-            try {
-                const db = await this.sourceDatabases.get(dbName);
-                return db;
-            } catch (ex) {
-                this.loggingService.error(`[Database] There's a problem with the connection to ${dbName}, ${(ex as Error).message}`);
-            }
-        }
-        this.loggingService.info(`[Database] Creating connection to ${dbName}`);
-        this.sourceDatabases.set(dbName, this.createConnection(dbName));
-        return this.sourceDatabases.get(dbName);
-    }
-
-    private async createConnection(dbName: string) {
         try {
-            const dbPromise = this.sqlite.createConnection(dbName + ".db", false, "no-encryption", 1, true);
-            const db = await dbPromise;
-            this.loggingService.info(`[Database] Connection created succefully to ${dbName}`);
-            await db.open();
-            this.loggingService.info(`[Database] Connection opened succefully: ${dbName}`);
-            return db;
+            return await this.pmTilesService.getTile(url);
         } catch (ex) {
-            this.loggingService.error(`[Database] Failed opening ${dbName}, ${(ex as Error).message}`);
-            throw ex;
+            this.loggingService.error(`[Database] Failed to get tile from pmtiles: ${(ex as Error).message}`);
         }
-    }
-
-    public async moveDownloadedDatabaseFile(dbFileName: string) {
-        await this.closeDatabase(dbFileName.replace(".db", ""));
-        this.loggingService.info(`[Database] Starting moving file ${dbFileName}`);
-        await this.sqlite.moveDatabasesAndAddSuffix("cache", [dbFileName]);
-        this.loggingService.info(`[Database] Finished moving file ${dbFileName}`);
+        return this.mbtilesService.getTile(url);
     }
 
     public storePois(pois: GeoJSON.Feature[]): Promise<any> {
@@ -333,11 +253,4 @@ export class DatabaseService {
         }
         return storedState;
     }
-
-    public async migrateDatabasesIfNeeded(): Promise<void> {
-        this.loggingService.info("[Database] Starting migrating old databases using sqlite plugin");
-        await this.sqlite.moveDatabasesAndAddSuffix("default", ["Contour.db", "IHM.db", "TerrainRGB.db"]);
-        const databases = await this.sqlite.getDatabaseList();
-        this.loggingService.info("[Database] Finished migrating old databases using sqlite plugin, " + JSON.stringify(databases.values));
-    }
 }