fix(schema-compiler): Correct models transpilation in native in multi…

…tenant environments (#9234) * add types to ErrorReporter * update native error reporter to use RefCells instead of Arc Mutex * Revert "update native error reporter to use RefCells instead of Arc Mutex" This reverts commit 406ee85. * pass compiler Id to transpilation native * Fix warn * use cargo insta for snapshot testing * introduce LruCache for transpilation
cube-js · Feb 18, 2025 · 84f90c0 · 84f90c0
1 parent 9c6153d
commit 84f90c0
Show file tree

Hide file tree

Showing 26 changed files with 631 additions and 271 deletions.
diff --git a/packages/cubejs-backend-native/Cargo.lock b/packages/cubejs-backend-native/Cargo.lock
diff --git a/packages/cubejs-backend-native/Cargo.toml b/packages/cubejs-backend-native/Cargo.toml
@@ -48,6 +48,7 @@ tokio = { version = "1", features = ["full", "rt"] }
 uuid = { version = "1", features = ["v4"] }
 bytes = "1.5.0"
 regex = "1.10.2"
+lru = "0.13.0"
 
 [dependencies.neon]
 version = "=1"

diff --git a/packages/cubejs-backend-native/js/index.ts b/packages/cubejs-backend-native/js/index.ts
@@ -104,6 +104,7 @@ export type SQLInterfaceOptions = {
 export interface TransformConfig {
   fileName: string;
   transpilers: string[];
+  compilerId: string;
   metaData?: {
     cubeNames: string[];
     cubeSymbols: Record<string, Record<string, boolean>>;

diff --git a/packages/cubejs-backend-native/src/transpilers.rs b/packages/cubejs-backend-native/src/transpilers.rs
@@ -2,12 +2,15 @@ use crate::node_obj_deserializer::JsValueDeserializer;
 use crate::node_obj_serializer::NodeObjSerializer;
 use anyhow::anyhow;
 use cubetranspilers::{run_transpilers, TransformConfig, Transpilers};
+use lru::LruCache;
 use neon::context::{Context, FunctionContext, ModuleContext};
 use neon::prelude::{JsPromise, JsResult, JsValue, NeonResult};
 use neon::types::JsString;
 use serde::Deserialize;
 use std::collections::{HashMap, HashSet};
-use std::sync::RwLock;
+use std::env;
+use std::num::NonZeroUsize;
+use std::sync::{LazyLock, Mutex};
 
 #[derive(Deserialize, Default, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
@@ -22,10 +25,25 @@ pub struct TransformMetaData {
 pub struct TransformRequestConfig {
     pub file_name: String,
     pub transpilers: Vec<Transpilers>,
+    pub compiler_id: String,
     pub meta_data: Option<TransformMetaData>,
 }
 
-static METADATA_CACHE: RwLock<Option<TransformMetaData>> = RwLock::new(None);
+/// It should be equal or more then number of internal libuv threads used by Neon
+/// By 01.2025 it defaults to 4. But maybe changed via `UV_THREADPOOL_SIZE` env var.
+/// `CUBEJS_TRANSPILER_METADATA_CACHE_SIZE` env var is provided for fine tuning.
+/// @see https://docs.libuv.org/en/v1.x/threadpool.html
+/// @see https://nodejs.org/api/cli.html#cli_uv_threadpool_size_size
+static DEFAULT_CACHE_SIZE: usize = 16;
+
+static METADATA_CACHE: LazyLock<Mutex<LruCache<String, TransformMetaData>>> = LazyLock::new(|| {
+    let cache_size = env::var("CUBEJS_TRANSPILER_METADATA_CACHE_SIZE")
+        .ok()
+        .and_then(|s| s.parse::<usize>().ok())
+        .and_then(NonZeroUsize::new)
+        .unwrap_or(NonZeroUsize::new(DEFAULT_CACHE_SIZE).unwrap());
+    Mutex::new(LruCache::new(cache_size))
+});
 
 pub fn register_module(cx: &mut ModuleContext) -> NeonResult<()> {
     cx.export_function("transpileJs", transpile_js)?;
@@ -44,7 +62,7 @@ pub fn transpile_js(mut cx: FunctionContext) -> JsResult<JsPromise> {
             let transform_config: TransformConfig = match transform_request_config {
                 Ok(data) => match data.meta_data {
                     Some(meta_data) => {
-                        let mut config_lock = METADATA_CACHE.write().unwrap();
+                        let mut config_lock = METADATA_CACHE.lock().unwrap();
                         let cache = TransformMetaData {
                             cube_names: meta_data.cube_names,
                             cube_symbols: meta_data.cube_symbols,
@@ -57,17 +75,27 @@ pub fn transpile_js(mut cx: FunctionContext) -> JsResult<JsPromise> {
                             cube_symbols: cache.cube_symbols.clone(),
                             context_symbols: cache.context_symbols.clone(),
                         };
-                        *config_lock = Some(cache);
+                        config_lock.put(data.compiler_id.clone(), cache);
                         cfg
                     }
                     None => {
-                        let cache = METADATA_CACHE.read().unwrap().clone().unwrap_or_default();
-                        TransformConfig {
-                            file_name: data.file_name,
-                            transpilers: data.transpilers,
-                            cube_names: cache.cube_names.clone(),
-                            cube_symbols: cache.cube_symbols.clone(),
-                            context_symbols: cache.context_symbols.clone(),
+                        let mut config_lock = METADATA_CACHE.lock().unwrap();
+
+                        match config_lock.get(&data.compiler_id) {
+                            Some(cached) => TransformConfig {
+                                file_name: data.file_name,
+                                transpilers: data.transpilers,
+                                cube_names: cached.cube_names.clone(),
+                                cube_symbols: cached.cube_symbols.clone(),
+                                context_symbols: cached.context_symbols.clone(),
+                            },
+                            None => TransformConfig {
+                                file_name: data.file_name,
+                                transpilers: data.transpilers,
+                                cube_names: HashSet::new(),
+                                cube_symbols: HashMap::new(),
+                                context_symbols: HashMap::new(),
+                            },
                         }
                     }
                 },

diff --git a/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js b/packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js
@@ -46,6 +46,7 @@ export class DataSchemaCompiler {
     this.yamlCompiler.dataSchemaCompiler = this;
     this.pythonContext = null;
     this.workerPool = null;
+    this.compilerId = options.compilerId;
   }
 
   compileObjects(compileServices, objects, errorsReport) {
@@ -97,6 +98,7 @@ export class DataSchemaCompiler {
 
     const transpilationWorkerThreads = getEnv('transpilationWorkerThreads');
     const transpilationNative = getEnv('transpilationNative');
+    const { compilerId } = this;
 
     if (!transpilationNative && transpilationWorkerThreads) {
       const wc = getEnv('transpilationWorkerThreadsCount');
@@ -141,9 +143,9 @@ export class DataSchemaCompiler {
           content: ';',
         };
 
-        await this.transpileJsFile(dummyFile, errorsReport, { cubeNames, cubeSymbols, transpilerNames, contextSymbols: CONTEXT_SYMBOLS });
+        await this.transpileJsFile(dummyFile, errorsReport, { cubeNames, cubeSymbols, transpilerNames, contextSymbols: CONTEXT_SYMBOLS, compilerId });
 
-        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { transpilerNames })));
+        results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { transpilerNames, compilerId })));
       } else if (transpilationWorkerThreads) {
         results = await Promise.all(toCompile.map(f => this.transpileFile(f, errorsReport, { cubeNames, cubeSymbols, transpilerNames })));
       } else {
@@ -172,7 +174,11 @@ export class DataSchemaCompiler {
             content: ';',
           };
 
-          return this.transpileJsFile(dummyFile, errorsReport, { cubeNames: [], cubeSymbols: {}, transpilerNames: [], contextSymbols: {} });
+          return this.transpileJsFile(
+            dummyFile,
+            errorsReport,
+            { cubeNames: [], cubeSymbols: {}, transpilerNames: [], contextSymbols: {}, compilerId: this.compilerId }
+          );
         } else if (transpilationWorkerThreads && this.workerPool) {
           this.workerPool.terminate();
         }
@@ -219,12 +225,13 @@ export class DataSchemaCompiler {
     }
   }
 
-  async transpileJsFile(file, errorsReport, { cubeNames, cubeSymbols, contextSymbols, transpilerNames }) {
+  async transpileJsFile(file, errorsReport, { cubeNames, cubeSymbols, contextSymbols, transpilerNames, compilerId }) {
     try {
       if (getEnv('transpilationNative')) {
         const reqData = {
           fileName: file.fileName,
           transpilers: transpilerNames,
+          compilerId,
           ...(cubeNames && {
             metaData: {
               cubeNames,

diff --git a/packages/cubejs-schema-compiler/src/compiler/ErrorReporter.ts b/packages/cubejs-schema-compiler/src/compiler/ErrorReporter.ts
@@ -3,6 +3,12 @@ import { codeFrameColumns, SourceLocation } from '@babel/code-frame';
 import { UserError } from './UserError';
 import { CompileError } from './CompileError';
 
+export type ErrorLikeObject = {
+  message: string;
+};
+
+export type PossibleError = Error | UserError | string | ErrorLikeObject;
+
 export interface CompilerErrorInterface {
   message: string;
   plainMessage?: string
@@ -141,16 +147,16 @@ export class ErrorReporter {
     return this.rootReporter().errors;
   }
 
-  public addErrors(errors: any[]) {
+  public addErrors(errors: PossibleError[]) {
     errors.forEach((e: any) => { this.error(e); });
   }
 
   public getWarnings() {
     return this.rootReporter().warnings;
   }
 
-  public addWarnings(warnings: any[]) {
-    warnings.forEach((w: any) => { this.warning(w); });
+  public addWarnings(warnings: SyntaxErrorInterface[]) {
+    warnings.forEach(w => { this.warning(w); });
   }
 
   protected rootReporter(): ErrorReporter {

diff --git a/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts b/packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts
@@ -59,6 +59,8 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
     transpilers.push(new CubeCheckDuplicatePropTranspiler());
   }
 
+  const compilerId = uuidv4();
+
   const compiler = new DataSchemaCompiler(repo, Object.assign({}, {
     cubeNameCompilers: [cubeDictionary],
     preTranspileCubeCompilers: [cubeSymbols, cubeValidator],
@@ -80,6 +82,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
     standalone: options.standalone,
     nativeInstance,
     yamlCompiler,
+    compilerId,
   }, options));
 
   return {
@@ -90,7 +93,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
     joinGraph,
     compilerCache,
     headCommitId: options.headCommitId,
-    compilerId: uuidv4(),
+    compilerId,
   };
 };
 

diff --git a/rust/cubenativeutils/Cargo.lock b/rust/cubenativeutils/Cargo.lock
diff --git a/rust/cubesql/Cargo.lock b/rust/cubesql/Cargo.lock