Merge pull request #2327 from botpress/ya-fix-bpfs

fix(bpfs): various fixes due to legacy logic

src/bp/bootstrap.ts

@@ -6,16 +6,29 @@ import './common/polyfills'
 
 import sdk from 'botpress/sdk'
 import chalk from 'chalk'
-import { Botpress, Config, Logger } from 'core/app'
+import { Botpress, Config, Db, Ghost, Logger } from 'core/app'
 import center from 'core/logger/center'
 import { ModuleLoader } from 'core/module-loader'
 import ModuleResolver from 'core/modules/resolver'
 import fs from 'fs'
 import os from 'os'
 
+import { DatabaseType } from 'core/database'
 import { FatalError } from './errors'
 
+async function setupEnv() {
+  const { DATABASE_URL } = process.env
+
+  const useDbDriver = process.BPFS_STORAGE === 'database'
+  Ghost.initialize(useDbDriver)
+
+  const dbType = DATABASE_URL && DATABASE_URL.toLowerCase().startsWith('postgres') ? 'postgres' : 'sqlite'
+  await Db.initialize(<DatabaseType>dbType, DATABASE_URL)
+}
+
 async function start() {
+  await setupEnv()
+
   const logger = await Logger('Launcher')
   logger.info(chalk`========================================
 {bold ${center(`Botpress Server`, 40)}}

src/bp/core/app.ts

@@ -6,17 +6,23 @@ import 'reflect-metadata'
 import { container } from './app.inversify'
 import { Botpress as Core } from './botpress'
 import { ConfigProvider } from './config/config-loader'
+import Database from './database'
 import { LoggerProvider } from './logger/logger'
+import { GhostService } from './services'
 import { TYPES } from './types'
 
 let botpress
 let logger: LoggerProvider | undefined
 let config: ConfigProvider | undefined
+let ghost: GhostService | undefined
+let database: Database | undefined
 
 try {
   botpress = container.get<Core>(TYPES.Botpress)
   logger = container.get<LoggerProvider>(TYPES.LoggerProvider)
   config = container.get<ConfigProvider>(TYPES.ConfigProvider)
+  ghost = container.get<GhostService>(TYPES.GhostService)
+  database = container.get<Database>(TYPES.Database)
 
   const licensing = container.get<LicensingService>(TYPES.LicensingService)
   licensing.installProtection()
@@ -27,3 +33,5 @@ try {
 export const Botpress = botpress
 export const Logger = logger!
 export const Config = config
+export const Ghost = ghost
+export const Db = database

src/bp/core/botpress.ts

@@ -16,7 +16,7 @@ import { setDebugScopes } from '../debug'
 import { createForGlobalHooks } from './api'
 import { BotpressConfig } from './config/botpress.config'
 import { ConfigProvider } from './config/config-loader'
-import Database, { DatabaseType } from './database'
+import Database from './database'
 import { LoggerDbPersister, LoggerFilePersister, LoggerProvider } from './logger'
 import { ModuleLoader } from './module-loader'
 import HTTPServer from './server'
@@ -106,20 +106,16 @@ export class Botpress {
   }
 
   private async initialize(options: StartOptions) {
+    this.config = await this.configProvider.getBotpressConfig()
+
     this.trackStart()
     this.trackHeartbeat()
 
     setDebugScopes(process.core_env.DEBUG || (process.IS_PRODUCTION ? '' : 'bp:dialog'))
 
-    this.config = await this.loadConfiguration()
-    await this.createDatabase()
-    await this.initializeGhost()
-    await this.restoreDebugScope()
-
-    // Invalidating the configuration to force it to load it from the ghost if enabled
-    this.config = await this.loadConfiguration(true)
-
     await AppLifecycle.setDone(AppLifecycleEvents.CONFIGURATION_LOADED)
+
+    await this.restoreDebugScope()
     await this.checkJwtSecret()
     await this.loadModules(options.modules)
     await this.migrationService.initialize()
@@ -162,7 +158,8 @@ export class Botpress {
   }
 
   async checkEditionRequirements() {
-    const databaseType = this.getDatabaseType()
+    const { DATABASE_URL } = process.env
+    const dbType = DATABASE_URL && DATABASE_URL.toLowerCase().startsWith('postgres') ? 'postgres' : 'sqlite'
 
     if (!process.IS_PRO_ENABLED && process.CLUSTER_ENABLED) {
       this.logger.warn(
@@ -203,7 +200,7 @@ export class Botpress {
         'Botpress can be run on a cluster. If you want to do so, make sure Redis is running and properly configured in your environment variables'
       )
     }
-    if (process.IS_PRO_ENABLED && databaseType !== 'postgres' && process.CLUSTER_ENABLED) {
+    if (process.IS_PRO_ENABLED && dbType !== 'postgres' && process.CLUSTER_ENABLED) {
       throw new Error(
         'Postgres is required to use Botpress in a cluster. Please migrate your database to Postgres and enable it in your Botpress configuration file.'
       )
@@ -370,26 +367,6 @@ export class Botpress {
     await AppLifecycle.setDone(AppLifecycleEvents.SERVICES_READY)
   }
 
-  private async loadConfiguration(forceInvalidate?): Promise<BotpressConfig> {
-    if (forceInvalidate) {
-      await this.configProvider.invalidateBotpressConfig()
-    }
-    return this.configProvider.getBotpressConfig()
-  }
-
-  private getDatabaseType(): DatabaseType {
-    const databaseUrl = process.env.DATABASE_URL
-    const databaseType = databaseUrl && databaseUrl.toLowerCase().startsWith('postgres') ? 'postgres' : 'sqlite'
-
-    return databaseType
-  }
-
-  @WrapErrorsWith(`Error initializing Database. Please check your configuration`)
-  private async createDatabase(): Promise<void> {
-    const databaseType = this.getDatabaseType()
-    await this.database.initialize(<DatabaseType>databaseType.toLowerCase(), process.env.DATABASE_URL)
-  }
-
   private async loadModules(modules: sdk.ModuleEntryPoint[]): Promise<void> {
     const loadedModules = await this.moduleLoader.loadModules(modules)
     this.logger.info(`Loaded ${loadedModules.length} ${plur('module', loadedModules.length)}`)

src/bp/core/config/config-loader.ts

@@ -5,7 +5,6 @@ import { GhostService } from 'core/services'
 import { TYPES } from 'core/types'
 import { FatalError } from 'errors'
 import fs from 'fs'
-import fse from 'fs-extra'
 import { inject, injectable } from 'inversify'
 import defaultJsonBuilder from 'json-schema-defaults'
 import _, { PartialDeep } from 'lodash'
@@ -51,11 +50,6 @@ export class ConfigProvider {
     await this.ghostService.global().upsertFile('/', 'botpress.config.json', stringify(config))
   }
 
-  async invalidateBotpressConfig(): Promise<void> {
-    this._botpressConfigCache = undefined
-    await this.ghostService.global().invalidateFile('/', 'botpress.config.json')
-  }
-
   async getBotConfig(botId: string): Promise<BotConfig> {
     return this.getConfig<BotConfig>('bot.config.json', botId)
   }
@@ -72,13 +66,13 @@ export class ConfigProvider {
   }
 
   public async createDefaultConfigIfMissing() {
-    const botpressConfig = path.resolve(process.PROJECT_LOCATION, 'data', 'global', 'botpress.config.json')
+    if (!(await this.ghostService.global().fileExists('/', 'botpress.config.json'))) {
+      await this._copyConfigSchemas()
 
-    if (!fse.existsSync(botpressConfig)) {
-      await this.ensureDataFolderStructure()
-
-      const botpressConfigSchema = path.resolve(process.PROJECT_LOCATION, 'data', 'botpress.config.schema.json')
-      const defaultConfig = defaultJsonBuilder(JSON.parse(fse.readFileSync(botpressConfigSchema, 'utf-8')))
+      const botpressConfigSchema = await this.ghostService
+        .root()
+        .readFileAsObject<any>('/', 'botpress.config.schema.json')
+      const defaultConfig = defaultJsonBuilder(botpressConfigSchema)
 
       const config = {
         $schema: `../botpress.config.schema.json`,
@@ -87,22 +81,16 @@ export class ConfigProvider {
         version: process.BOTPRESS_VERSION
       }
 
-      await fse.writeFileSync(botpressConfig, stringify(config))
+      await this.ghostService.global().upsertFile('/', 'botpress.config.json', stringify(config))
     }
   }
 
-  private async ensureDataFolderStructure() {
-    const requiredFolders = ['bots', 'global', 'storage']
+  private async _copyConfigSchemas() {
     const schemasToCopy = ['botpress.config.schema.json', 'bot.config.schema.json']
-    const dataFolder = path.resolve(process.PROJECT_LOCATION, 'data')
-
-    for (const folder of requiredFolders) {
-      await fse.ensureDir(path.resolve(dataFolder, folder))
-    }
 
     for (const schema of schemasToCopy) {
       const schemaContent = fs.readFileSync(path.join(__dirname, 'schemas', schema))
-      fs.writeFileSync(path.resolve(dataFolder, schema), schemaContent)
+      await this.ghostService.root().upsertFile('/', schema, schemaContent)
     }
   }
 

src/bp/core/database/index.ts

@@ -1,7 +1,7 @@
 import { Logger } from 'botpress/sdk'
 import { KnexExtension } from 'common/knex'
 import { TYPES } from 'core/types'
-import fs from 'fs'
+import { mkdirpSync } from 'fs-extra'
 import { inject, injectable, tagged } from 'inversify'
 import Knex from 'knex'
 import _ from 'lodash'
@@ -65,6 +65,7 @@ export default class Database {
       })
     } else {
       const dbLocation = databaseUrl ? databaseUrl : `${process.PROJECT_LOCATION}/data/storage/core.sqlite`
+      mkdirpSync(path.dirname(dbLocation))
 
       Object.assign(config, {
         client: 'sqlite3',

src/bp/core/server.ts

@@ -183,6 +183,7 @@ export default class HTTPServer {
 
   @postConstruct()
   async initialize() {
+    await AppLifecycle.waitFor(AppLifecycleEvents.CONFIGURATION_LOADED)
     await this.setupRootPath()
 
     const app = express()

src/bp/core/services/ghost/service.ts

@@ -63,13 +63,18 @@ export class GhostService {
 
   initialize(enabled: boolean) {
     this.enabled = enabled
+    this._scopedGhosts.clear()
+  }
+
+  // Not caching this scope since it's rarely used
+  root(): ScopedGhostService {
+    return new ScopedGhostService(`./data`, this.diskDriver, this.dbDriver, this.enabled, this.cache, this.logger)
   }
 
   global(): ScopedGhostService {
-    // Disabling temporarily
-    // if (this._scopedGhosts.has(GLOBAL_GHOST_KEY)) {
-    //   return this._scopedGhosts.get(GLOBAL_GHOST_KEY)!
-    // }
+    if (this._scopedGhosts.has(GLOBAL_GHOST_KEY)) {
+      return this._scopedGhosts.get(GLOBAL_GHOST_KEY)!
+    }
 
     const scopedGhost = new ScopedGhostService(
       `./data/global`,
@@ -80,7 +85,7 @@ export class GhostService {
       this.logger
     )
 
-    // this._scopedGhosts.set(GLOBAL_GHOST_KEY, scopedGhost)
+    this._scopedGhosts.set(GLOBAL_GHOST_KEY, scopedGhost)
     return scopedGhost
   }
 

src/bp/core/services/migration/index.ts

@@ -4,13 +4,12 @@ import { BotpressAPIProvider } from 'core/api'
 import { ConfigProvider } from 'core/config/config-loader'
 import Database from 'core/database'
 import center from 'core/logger/center'
+import { stringify } from 'core/misc/utils'
 import { TYPES } from 'core/types'
-import fs from 'fs'
 import fse from 'fs-extra'
 import glob from 'glob'
 import { Container, inject, injectable, tagged } from 'inversify'
 import _ from 'lodash'
-import mkdirp from 'mkdirp'
 import path from 'path'
 import semver from 'semver'
 
@@ -24,13 +23,18 @@ const types = {
   config: 'Config File Changes',
   content: 'Changes to Content Files (*.json)'
 }
+/**
+ * Use a combination of these environment variables to easily test migrations.
+ * TESTMIG_BP_VERSION: Change the target version of your migration
+ * TESTMIG_CONFIG_VERSION: Override the current version of the server
+ * TESTMIG_IGNORE_COMPLETED: Ignore completed migrations (so they can be run again and again)
+ */
 
 @injectable()
 export class MigrationService {
   /** This is the actual running version (package.json) */
   private currentVersion: string
   private loadedMigrations: { [filename: string]: Migration | sdk.ModuleMigration } = {}
-  private completedMigrationsDir: string
 
   constructor(
     @tagged('name', 'Migration')
@@ -40,13 +44,11 @@ export class MigrationService {
     @inject(TYPES.ConfigProvider) private configProvider: ConfigProvider,
     @inject(TYPES.GhostService) private ghostService: GhostService
   ) {
-    this.currentVersion = process.env.MIGRATION_TEST_VERSION || process.BOTPRESS_VERSION
-    this.completedMigrationsDir = path.resolve(process.PROJECT_LOCATION, `data/migrations`)
-    mkdirp.sync(this.completedMigrationsDir)
+    this.currentVersion = process.env.TESTMIG_BP_VERSION || process.BOTPRESS_VERSION
   }
 
   async initialize() {
-    const configVersion = (await this.configProvider.getBotpressConfig()).version
+    const configVersion = process.env.TESTMIG_CONFIG_VERSION || (await this.configProvider.getBotpressConfig()).version
     debug(`Migration Check: %o`, { configVersion, currentVersion: this.currentVersion })
 
     if (process.env.SKIP_MIGRATIONS) {
@@ -109,7 +111,7 @@ export class MigrationService {
 {bold ${center(`Executing ${missingMigrations.length.toString()} migrations`, 40)}}
 ========================================`)
 
-    const completed = this._getCompletedMigrations()
+    const completed = await this._getCompletedMigrations()
     let hasFailures = false
 
     // Clear the Botpress cache before executing any migrations
@@ -132,7 +134,7 @@ export class MigrationService {
 
       const result = await this.loadedMigrations[filename].up(opts)
       if (result.success) {
-        this._saveCompletedMigration(filename, result)
+        await this._saveCompletedMigration(filename, result)
         await this.logger.info(`- ${result.message || 'Success'}`)
       } else {
         hasFailures = true
@@ -205,12 +207,16 @@ export class MigrationService {
     return [...coreMigrations, ...moduleMigrations]
   }
 
-  private _getCompletedMigrations(): string[] {
-    return fs.readdirSync(this.completedMigrationsDir)
+  private async _getCompletedMigrations(): Promise<string[]> {
+    if (process.env.TESTMIG_IGNORE_COMPLETED) {
+      return []
+    }
+
+    return this.ghostService.root().directoryListing('migrations')
   }
 
-  private _saveCompletedMigration(filename: string, result: sdk.MigrationResult) {
-    fs.writeFileSync(path.resolve(`${this.completedMigrationsDir}/${filename}`), JSON.stringify(result, undefined, 2))
+  private _saveCompletedMigration(filename: string, result: sdk.MigrationResult): Promise<void> {
+    return this.ghostService.root().upsertFile('migrations', filename, stringify(result))
   }
 
   private _loadMigrations = (fileList: MigrationFile[]) =>

src/bp/core/stats.ts

@@ -1,5 +1,6 @@
 import { gaId, machineUUID } from 'common/stats'
 import { inject, injectable, postConstruct } from 'inversify'
+import { AppLifecycle, AppLifecycleEvents } from 'lifecycle'
 import ua, { Visitor } from 'universal-analytics'
 
 import { ConfigProvider } from './config/config-loader'
@@ -15,6 +16,7 @@ export class Statistics {
 
   @postConstruct()
   public async init() {
+    await AppLifecycle.waitFor(AppLifecycleEvents.CONFIGURATION_LOADED)
     const botpressConfig = await this.configProvider.getBotpressConfig()
     this._sendUsageStats = botpressConfig.sendUsageStats