diff --git a/backend/src/south/south-mssql/manifest.ts b/backend/src/south/south-mssql/manifest.ts index 2c62f68d87..0ba7f54a84 100644 --- a/backend/src/south/south-mssql/manifest.ts +++ b/backend/src/south/south-mssql/manifest.ts @@ -67,6 +67,7 @@ const manifest: SouthConnectorManifest = { type: 'OibCheckbox', label: 'Use encryption', defaultValue: false, + validators: [{ key: 'required' }], newRow: true, displayInViewMode: true }, diff --git a/backend/src/south/south-mssql/south-mssql.spec.ts b/backend/src/south/south-mssql/south-mssql.spec.ts index 3bb8ac18cb..87e642eb25 100644 --- a/backend/src/south/south-mssql/south-mssql.spec.ts +++ b/backend/src/south/south-mssql/south-mssql.spec.ts @@ -103,24 +103,7 @@ const items: Array> = [ connectorId: 'southId', settings: { query: 'SELECT * FROM table', - dateTimeFields: [ - { - fieldName: 'anotherTimestamp', - useAsReference: false, - type: 'unix-epoch-ms', - timezone: null, - format: null, - locale: null - } as unknown as SouthMSSQLItemSettingsDateTimeFields, - { - fieldName: 'timestamp', - useAsReference: true, - type: 'string', - timezone: 'Europe/Paris', - format: 'yyyy-MM-dd HH:mm:ss.SSS', - locale: 'en-US' - } - ], + dateTimeFields: [], serialization: { type: 'csv', filename: 'sql-@CurrentDate.csv', @@ -199,7 +182,7 @@ describe('SouthMSSQL with authentication', () => { username: 'username', password: 'password', domain: 'domain', - encryption: true, + encryption: null, connectionTimeout: 1000, requestTimeout: 1000, trustServerCertificate: true @@ -286,7 +269,6 @@ describe('SouthMSSQL with authentication', () => { connectionTimeout: configuration.settings.connectionTimeout, requestTimeout: configuration.settings.requestTimeout, options: { - encrypt: configuration.settings.encryption, trustServerCertificate: configuration.settings.trustServerCertificate, useUTC: true }, @@ -320,7 +302,7 @@ describe('SouthMSSQL without authentication', () => { username: null, password: null, domain: '', - encryption: false, + encryption: true, connectionTimeout: 1000, requestTimeout: 1000, trustServerCertificate: false diff --git a/backend/src/south/south-mssql/south-mssql.ts b/backend/src/south/south-mssql/south-mssql.ts index 1fa57c1d49..3e214df332 100644 --- a/backend/src/south/south-mssql/south-mssql.ts +++ b/backend/src/south/south-mssql/south-mssql.ts @@ -56,7 +56,7 @@ export default class SouthMSSQL connectionTimeout: this.connector.settings.connectionTimeout, requestTimeout: this.connector.settings.requestTimeout, options: { - encrypt: this.connector.settings.encryption == null ? undefined : this.connector.settings.encryption, + encrypt: this.connector.settings.encryption || undefined, trustServerCertificate: this.connector.settings.trustServerCertificate, useUTC: true } @@ -78,9 +78,6 @@ export default class SouthMSSQL request = pool.request(); } catch (error: any) { this.logger.error(`Unable to connect to database: ${error.message}`); - if (pool) { - await pool.close(); - } switch (error.code) { case 'ETIMEOUT': diff --git a/backend/src/south/south-odbc/manifest.ts b/backend/src/south/south-odbc/manifest.ts index 49433f08ad..c0b07ebf98 100644 --- a/backend/src/south/south-odbc/manifest.ts +++ b/backend/src/south/south-odbc/manifest.ts @@ -15,47 +15,31 @@ const manifest: SouthConnectorManifest = { }, settings: [ { - key: 'driverPath', - type: 'OibText', - label: 'ODBC Driver Path', + key: 'remoteAgent', + type: 'OibCheckbox', + label: 'Use remote agent', + defaultValue: false, newRow: true, validators: [{ key: 'required' }], displayInViewMode: true }, { - key: 'host', + key: 'agentUrl', type: 'OibText', - label: 'Host', - defaultValue: 'localhost', - newRow: true, + label: 'Use remote agent', + defaultValue: 'http://ip-adress-or-host:2224', validators: [{ key: 'required' }], - displayInViewMode: true - }, - { - key: 'port', - type: 'OibNumber', - label: 'Port', - defaultValue: 1433, - newRow: false, - class: 'col-2', - validators: [{ key: 'required' }, { key: 'min', params: { min: 1 } }, { key: 'max', params: { max: 65535 } }], - displayInViewMode: true + conditionalDisplay: { field: 'remoteAgent', values: [true] } }, { - key: 'database', + key: 'connectionString', type: 'OibText', - label: 'Database', - defaultValue: 'db', + label: 'Connection string', + defaultValue: 'localhost', newRow: true, validators: [{ key: 'required' }], displayInViewMode: true }, - { - key: 'username', - type: 'OibText', - label: 'Username', - displayInViewMode: true - }, { key: 'password', type: 'OibSecret', @@ -74,11 +58,13 @@ const manifest: SouthConnectorManifest = { displayInViewMode: false }, { - key: 'trustServerCertificate', - type: 'OibCheckbox', - label: 'Accept self-signed certificate', - defaultValue: false, - validators: [{ key: 'required' }], + key: 'requestTimeout', + type: 'OibNumber', + label: 'Request timeout', + defaultValue: 15_000, + unitLabel: 'ms', + class: 'col-4', + validators: [{ key: 'required' }, { key: 'min', params: { min: 100 } }, { key: 'max', params: { max: 30000 } }], displayInViewMode: false } ], diff --git a/backend/src/south/south-odbc/south-odbc-no-lib.spec.ts b/backend/src/south/south-odbc/south-odbc-no-lib.spec.ts new file mode 100644 index 0000000000..4da0f2922a --- /dev/null +++ b/backend/src/south/south-odbc/south-odbc-no-lib.spec.ts @@ -0,0 +1,100 @@ +import SouthODBC from './south-odbc'; +import DatabaseMock from '../../tests/__mocks__/database.mock'; +import pino from 'pino'; +// eslint-disable-next-line import/no-unresolved +import PinoLogger from '../../tests/__mocks__/logger.mock'; +import EncryptionService from '../../service/encryption.service'; +import EncryptionServiceMock from '../../tests/__mocks__/encryption-service.mock'; +import RepositoryService from '../../service/repository.service'; +import RepositoryServiceMock from '../../tests/__mocks__/repository-service.mock'; +import { SouthConnectorDTO, SouthConnectorItemDTO } from '../../../../shared/model/south-connector.model'; +import { SouthODBCItemSettings, SouthODBCSettings } from '../../../../shared/model/south-settings.model'; + +jest.mock('../../service/utils'); +jest.mock('odbc', () => { + throw new Error('bad'); +}); +jest.mock('node:fs/promises'); + +const database = new DatabaseMock(); +jest.mock( + '../../service/south-cache.service', + () => + function () { + return { + createSouthCacheScanModeTable: jest.fn(), + southCacheRepository: { + database + } + }; + } +); + +jest.mock( + '../../service/south-connector-metrics.service', + () => + function () { + return { + initMetrics: jest.fn(), + updateMetrics: jest.fn(), + get stream() { + return { stream: 'myStream' }; + }, + metrics: { + numberOfValuesRetrieved: 1, + numberOfFilesRetrieved: 1 + } + }; + } +); +const addValues = jest.fn(); +const addFile = jest.fn(); + +const logger: pino.Logger = new PinoLogger(); +const encryptionService: EncryptionService = new EncryptionServiceMock('', ''); +const repositoryService: RepositoryService = new RepositoryServiceMock(); +const items: Array> = []; + +let south: SouthODBC; + +// Spy on console info and error +jest.spyOn(global.console, 'info').mockImplementation(() => {}); +jest.spyOn(global.console, 'error').mockImplementation(() => {}); + +describe('SouthODBC without ODBC Library', () => { + const configuration: SouthConnectorDTO = { + id: 'southId', + name: 'south', + type: 'odbc', + description: 'my test connector', + enabled: true, + history: { + maxInstantPerItem: true, + maxReadInterval: 3600, + readDelay: 0 + }, + settings: { + remoteAgent: false, + connectionString: 'Driver={SQL Server};SERVER=127.0.0.1;TrustServerCertificate=yes', + password: 'password', + connectionTimeout: 1000, + requestTimeout: 1000 + } + }; + + beforeEach(() => { + jest.clearAllMocks(); + + south = new SouthODBC(configuration, items, addValues, addFile, encryptionService, repositoryService, logger, 'baseFolder'); + }); + + it('should throw error if library not loaded', async () => { + await expect(south.testOdbcConnection()).rejects.toThrow(new Error('odbc library not loaded')); + + const startTime = '2020-01-01T00:00:00.000Z'; + const endTime = '2022-01-01T00:00:00.000Z'; + await expect(south.queryOdbcData({} as SouthConnectorItemDTO, startTime, endTime)).rejects.toThrow( + new Error('odbc library not loaded') + ); + }); +}); diff --git a/backend/src/south/south-odbc/south-odbc.spec.ts b/backend/src/south/south-odbc/south-odbc.spec.ts index fe9e91ae84..d78790caa2 100644 --- a/backend/src/south/south-odbc/south-odbc.spec.ts +++ b/backend/src/south/south-odbc/south-odbc.spec.ts @@ -2,17 +2,19 @@ import path from 'node:path'; import SouthODBC from './south-odbc'; import * as utils from '../../service/utils'; +import { convertDateTimeToInstant, formatInstant, persistResults } from '../../service/utils'; import DatabaseMock from '../../tests/__mocks__/database.mock'; + import pino from 'pino'; // eslint-disable-next-line import/no-unresolved import odbc from 'odbc'; +import fetch from 'node-fetch'; import PinoLogger from '../../tests/__mocks__/logger.mock'; import EncryptionService from '../../service/encryption.service'; import EncryptionServiceMock from '../../tests/__mocks__/encryption-service.mock'; import RepositoryService from '../../service/repository.service'; import RepositoryServiceMock from '../../tests/__mocks__/repository-service.mock'; import { SouthConnectorDTO, SouthConnectorItemDTO } from '../../../../shared/model/south-connector.model'; -import { DateTime } from 'luxon'; import { SouthODBCItemSettings, SouthODBCItemSettingsDateTimeFields, @@ -22,6 +24,7 @@ import { jest.mock('../../service/utils'); jest.mock('odbc'); jest.mock('node:fs/promises'); +jest.mock('node-fetch'); const database = new DatabaseMock(); jest.mock( @@ -104,24 +107,7 @@ const items: Array> = [ connectorId: 'southId', settings: { query: 'SELECT * FROM table', - dateTimeFields: [ - { - fieldName: 'anotherTimestamp', - useAsReference: false, - type: 'unix-epoch-ms', - timezone: null, - format: null, - locale: null - } as unknown as SouthODBCItemSettingsDateTimeFields, - { - fieldName: 'timestamp', - useAsReference: true, - type: 'string', - timezone: 'Europe/Paris', - format: 'yyyy-MM-dd HH:mm:ss.SSS', - locale: 'en-US' - } - ], + dateTimeFields: [], serialization: { type: 'csv', filename: 'sql-@CurrentDate.csv', @@ -174,7 +160,11 @@ const items: Array> = [ const nowDateString = '2020-02-02T02:02:02.222Z'; let south: SouthODBC; -describe('SouthODBC with authentication', () => { +// Spy on console info and error +jest.spyOn(global.console, 'info').mockImplementation(() => {}); +jest.spyOn(global.console, 'error').mockImplementation(() => {}); + +describe('SouthODBC odbc driver with authentication', () => { const configuration: SouthConnectorDTO = { id: 'southId', name: 'south', @@ -187,20 +177,19 @@ describe('SouthODBC with authentication', () => { readDelay: 0 }, settings: { - driverPath: 'myOdbcDriver', - host: 'localhost', - port: 1433, - database: 'db', - username: 'username', + remoteAgent: false, + connectionString: 'Driver={SQL Server};SERVER=127.0.0.1;TrustServerCertificate=yes', password: 'password', connectionTimeout: 1000, - trustServerCertificate: true + requestTimeout: 1000 } }; beforeEach(async () => { jest.clearAllMocks(); jest.useFakeTimers().setSystemTime(new Date(nowDateString)); + (convertDateTimeToInstant as jest.Mock).mockImplementation(value => value); + south = new SouthODBC(configuration, items, addValues, addFile, encryptionService, repositoryService, logger, 'baseFolder'); }); @@ -209,31 +198,21 @@ describe('SouthODBC with authentication', () => { expect(utils.createFolder).toHaveBeenCalledWith(path.resolve('baseFolder', 'tmp')); }); + it('should do nothing on connect and disconnect', async () => { + await south.connect(); + await south.disconnect(); + expect(fetch).not.toHaveBeenCalled(); + }); + it('should properly run historyQuery', async () => { const startTime = '2020-01-01T00:00:00.000Z'; - south.queryData = jest - .fn() - .mockReturnValueOnce([ - { timestamp: '2020-02-01T00:00:00.000Z', anotherTimestamp: '2023-02-01T00:00:00.000Z', value: 123 }, - { timestamp: '2020-03-01T00:00:00.000Z', anotherTimestamp: '2023-02-01T00:00:00.000Z', value: 456 } - ]) - .mockReturnValue([]); - (utils.formatInstant as jest.Mock) - .mockReturnValueOnce('2020-02-01 00:00:00.000') - .mockReturnValueOnce('2020-03-01 00:00:00.000') - .mockReturnValue(startTime); - (utils.convertDateTimeToInstant as jest.Mock).mockImplementation(instant => instant); + south.queryOdbcData = jest.fn().mockReturnValueOnce('2023-02-01T00:00:00.000Z').mockReturnValue('2023-02-01T00:00:00.000Z'); await south.historyQuery(items, startTime, nowDateString); - expect(utils.persistResults).toHaveBeenCalledTimes(1); - expect(south.queryData).toHaveBeenCalledTimes(3); - expect(south.queryData).toHaveBeenCalledWith(items[0], startTime, nowDateString); - expect(south.queryData).toHaveBeenCalledWith(items[1], '2020-03-01T00:00:00.000Z', nowDateString); - expect(south.queryData).toHaveBeenCalledWith(items[2], '2020-03-01T00:00:00.000Z', nowDateString); - - expect(logger.info).toHaveBeenCalledWith(`Found 2 results for item ${items[0].name} in 0 ms`); - expect(logger.debug).toHaveBeenCalledWith(`No result found for item ${items[1].name}. Request done in 0 ms`); - expect(logger.debug).toHaveBeenCalledWith(`No result found for item ${items[2].name}. Request done in 0 ms`); + expect(south.queryOdbcData).toHaveBeenCalledTimes(3); + expect(south.queryOdbcData).toHaveBeenCalledWith(items[0], startTime, nowDateString); + expect(south.queryOdbcData).toHaveBeenCalledWith(items[1], '2023-02-01T00:00:00.000Z', nowDateString); + expect(south.queryOdbcData).toHaveBeenCalledWith(items[2], '2023-02-01T00:00:00.000Z', nowDateString); }); it('should get data from ODBC', async () => { @@ -242,33 +221,71 @@ describe('SouthODBC with authentication', () => { const odbcConnection = { close: jest.fn(), - query: jest.fn().mockReturnValue([{ timestamp: '2020-02-01T00:00:00.000Z' }, { timestamp: '2020-03-01T00:00:00.000Z' }]) + query: jest + .fn() + .mockReturnValueOnce([ + { value: 1, timestamp: '2020-02-01T00:00:00.000Z', anotherTimestamp: '2020-02-01T00:00:00.000Z' }, + { value: 2, timestamp: '2020-03-01T00:00:00.000Z', anotherTimestamp: '2020-03-01T00:00:00.000Z' } + ]) + .mockReturnValueOnce([]) }; + (formatInstant as jest.Mock).mockImplementation(value => value); + (odbc.connect as jest.Mock).mockReturnValue(odbcConnection); - (utils.formatInstant as jest.Mock) - .mockReturnValueOnce(DateTime.fromISO(startTime).toFormat('yyyy-MM-dd HH:mm:ss.SSS')) - .mockReturnValueOnce(DateTime.fromISO(endTime).toFormat('yyyy-MM-dd HH:mm:ss.SSS')); - const result = await south.queryData(items[0], startTime, endTime); + const result = await south.queryOdbcData(items[0], startTime, endTime); - expect(utils.logQuery).toHaveBeenCalledWith( - items[0].settings.query, - DateTime.fromISO(startTime).toFormat('yyyy-MM-dd HH:mm:ss.SSS'), - DateTime.fromISO(endTime).toFormat('yyyy-MM-dd HH:mm:ss.SSS'), - logger - ); + expect(utils.logQuery).toHaveBeenCalledWith(items[0].settings.query, startTime, endTime, logger); - let expectedConnectionString = `Driver=${configuration.settings.driverPath};SERVER=${configuration.settings.host};PORT=${configuration.settings.port};`; - expectedConnectionString += `TrustServerCertificate=yes;Database=${configuration.settings.database};UID=${configuration.settings.username};`; expect(odbc.connect).toHaveBeenCalledWith({ - connectionString: expectedConnectionString + 'PWD=password;', + connectionString: `${configuration.settings.connectionString};` + 'PWD=password;', connectionTimeout: configuration.settings.connectionTimeout }); - expect(logger.debug).toHaveBeenCalledWith(`Connecting with connection string ${expectedConnectionString}PWD=;`); + expect(logger.debug).toHaveBeenCalledWith(`Connecting with connection string ${configuration.settings.connectionString}PWD=;`); expect(odbcConnection.query).toHaveBeenCalledWith(items[0].settings.query); expect(odbcConnection.close).toHaveBeenCalledTimes(1); - expect(result).toEqual([{ timestamp: '2020-02-01T00:00:00.000Z' }, { timestamp: '2020-03-01T00:00:00.000Z' }]); + expect(result).toEqual('2020-03-01T00:00:00.000Z'); + expect(persistResults).toHaveBeenCalledWith( + [ + { value: 1, timestamp: '2020-02-01T00:00:00.000Z', anotherTimestamp: '2020-02-01T00:00:00.000Z' }, + { value: 2, timestamp: '2020-03-01T00:00:00.000Z', anotherTimestamp: '2020-03-01T00:00:00.000Z' } + ], + items[0].settings.serialization, + configuration.name, + path.resolve('baseFolder', 'tmp'), + expect.any(Function), + expect.any(Function), + logger + ); + + await south.queryOdbcData(items[0], startTime, endTime); + expect(logger.debug).toHaveBeenCalledWith(`No result found for item ${items[0].name}. Request done in 0 ms`); + }); + + it('should get data from ODBC without datetime reference', async () => { + const startTime = '2020-01-01T00:00:00.000Z'; + const endTime = '2022-01-01T00:00:00.000Z'; + + const odbcConnection = { + close: jest.fn(), + query: jest + .fn() + .mockReturnValueOnce([ + { value: 1, timestamp: '2020-02-01T00:00:00.000Z', anotherTimestamp: '2020-02-01T00:00:00.000Z' }, + { value: 2, timestamp: '2020-03-01T00:00:00.000Z', anotherTimestamp: '2020-03-01T00:00:00.000Z' } + ]) + .mockReturnValueOnce([]) + }; + (formatInstant as jest.Mock).mockImplementation(value => value); + + (odbc.connect as jest.Mock).mockReturnValue(odbcConnection); + + const result = await south.queryOdbcData(items[1], startTime, endTime); + + expect(utils.logQuery).toHaveBeenCalledWith(items[1].settings.query, startTime, endTime, logger); + + expect(result).toEqual(startTime); }); it('should manage query error', async () => { @@ -292,7 +309,7 @@ describe('SouthODBC with authentication', () => { let error; try { - await south.queryData(items[0], startTime, endTime); + await south.queryOdbcData(items[0], startTime, endTime); } catch (err) { error = err; } @@ -302,7 +319,7 @@ describe('SouthODBC with authentication', () => { expect(odbcConnection.close).toHaveBeenCalledTimes(1); try { - await south.queryData(items[0], startTime, endTime); + await south.queryOdbcData(items[0], startTime, endTime); } catch (err) { error = err; } @@ -315,7 +332,7 @@ describe('SouthODBC with authentication', () => { }); }); -describe('SouthODBC without authentication', () => { +describe('SouthODBC odbc driver without authentication', () => { const configuration: SouthConnectorDTO = { id: 'southId', name: 'south', @@ -328,14 +345,11 @@ describe('SouthODBC without authentication', () => { readDelay: 0 }, settings: { - driverPath: 'myOdbcDriver', - host: 'localhost', - port: 1433, - database: '', - username: null, + remoteAgent: false, + connectionString: 'Driver={SQL Server};SERVER=127.0.0.1;TrustServerCertificate=yes', password: null, connectionTimeout: 1000, - trustServerCertificate: false + requestTimeout: 1000 } }; beforeEach(async () => { @@ -349,22 +363,32 @@ describe('SouthODBC without authentication', () => { const startTime = '2020-01-01T00:00:00.000Z'; const endTime = '2022-01-01T00:00:00.000Z'; + (convertDateTimeToInstant as jest.Mock).mockImplementation(value => value); + (formatInstant as jest.Mock).mockImplementation(value => value); const odbcConnection = { close: jest.fn(), query: jest.fn().mockReturnValue([{ timestamp: '2020-02-01T00:00:00.000Z' }, { timestamp: '2020-03-01T00:00:00.000Z' }]) }; (odbc.connect as jest.Mock).mockReturnValue(odbcConnection); - const result = await south.queryData(items[0], startTime, endTime); + const result = await south.queryOdbcData(items[0], startTime, endTime); - const expectedConnectionString = `Driver=${configuration.settings.driverPath};SERVER=${configuration.settings.host};PORT=${configuration.settings.port};`; expect(odbc.connect).toHaveBeenCalledWith({ - connectionString: expectedConnectionString, + connectionString: configuration.settings.connectionString, connectionTimeout: configuration.settings.connectionTimeout }); - expect(logger.debug).toHaveBeenCalledWith(`Connecting with connection string ${expectedConnectionString}`); - - expect(result).toEqual([{ timestamp: '2020-02-01T00:00:00.000Z' }, { timestamp: '2020-03-01T00:00:00.000Z' }]); + expect(logger.debug).toHaveBeenCalledWith(`Connecting with connection string ${configuration.settings.connectionString}`); + + expect(result).toEqual('2020-03-01T00:00:00.000Z'); + expect(persistResults).toHaveBeenCalledWith( + [{ timestamp: '2020-02-01T00:00:00.000Z' }, { timestamp: '2020-03-01T00:00:00.000Z' }], + items[0].settings.serialization, + configuration.name, + path.resolve('baseFolder', 'tmp'), + expect.any(Function), + expect.any(Function), + logger + ); }); it('should manage connection error', async () => { @@ -377,21 +401,20 @@ describe('SouthODBC without authentication', () => { let error; try { - await south.queryData(items[0], startTime, endTime); + await south.queryOdbcData(items[0], startTime, endTime); } catch (err) { error = err; } - const expectedConnectionString = `Driver=${configuration.settings.driverPath};SERVER=${configuration.settings.host};PORT=${configuration.settings.port};`; expect(odbc.connect).toHaveBeenCalledWith({ - connectionString: expectedConnectionString, + connectionString: configuration.settings.connectionString, connectionTimeout: configuration.settings.connectionTimeout }); expect(error).toEqual(new Error('connection error')); }); }); -describe('SouthODBC test connection', () => { - const configuration: SouthConnectorDTO = { +describe('SouthODBC odbc driver test connection', () => { + const configuration: SouthConnectorDTO = { id: 'southId', name: 'south', type: 'odbc', @@ -403,14 +426,11 @@ describe('SouthODBC test connection', () => { readDelay: 0 }, settings: { - driverPath: 'myOdbcDriver', - host: 'localhost', - port: 1433, - database: 'db', - username: 'username', + remoteAgent: false, + connectionString: 'Driver={SQL Server};SERVER=127.0.0.1;TrustServerCertificate=yes', password: 'password', connectionTimeout: 1000, - trustServerCertificate: true + requestTimeout: 1000 } }; @@ -429,7 +449,7 @@ describe('SouthODBC test connection', () => { HOST: 'Please check host and port', PORT: 'Please check host and port', CREDENTIALS: 'Please check username and password', - DB_ACCESS: `User "${configuration.settings.username}" does not have access to database "${configuration.settings.database}"`, + DB_ACCESS: `User does not have access to database`, DEFAULT: 'Please check logs' } as const; const connectionErrorMessage = 'Error creating connection'; @@ -511,18 +531,29 @@ describe('SouthODBC test connection', () => { const tables = result.map((row: any) => `${row.table_name}: [${row.columns}]`).join(',\n'); expect((logger.info as jest.Mock).mock.calls).toEqual([ - [`Testing connection on "${configuration.settings.host}"`], + ['Testing ODBC connection with "Driver={SQL Server};SERVER=127.0.0.1;TrustServerCertificate=yes"'], ['Database is live with tables (table:[columns]):\n%s', tables] ]); }); + it('Database is reachable but reading table fails', async () => { + const odbcConnection = { + close: jest.fn(), + tables: jest.fn(() => { + throw { odbcErrors: [], message: 'table error' }; + }) + }; + (odbc.connect as jest.Mock).mockReturnValue(odbcConnection); + await expect(south.testConnection()).rejects.toThrow('Unable to read tables in database, check logs'); + }); + it.each(flattenedErrors)( 'Unable to create connection with $driver, error code $error.driverError.odbcErrors.0.code', async driverTest => { (odbc.connect as jest.Mock).mockImplementationOnce(() => { throw driverTest.error.driverError; }); - configuration.settings.driverPath = driverTest.driver; + configuration.settings.connectionString = `Driver=${driverTest.driver}`; await expect(south.testConnection()).rejects.toThrow(driverTest.error.expectedError); @@ -530,24 +561,23 @@ describe('SouthODBC test connection', () => { [`Unable to connect to database: ${connectionErrorMessage}`], [`Error from ODBC driver: ${driverTest.error.driverError.odbcErrors[0].message}`] ]); - expect((logger.info as jest.Mock).mock.calls).toEqual([[`Testing connection on "${configuration.settings.host}"`]]); } ); it('Could not load driver', async () => { - configuration.settings.driverPath = 'Unknown driver'; + configuration.settings.connectionString = `Driver=Unknown driver`; + const error = new NodeOdbcError(connectionErrorMessage, [{ code: -1, message: 'Driver not found', state: 'IM002' }]); (odbc.connect as jest.Mock).mockImplementation(() => { throw error; }); - await expect(south.testConnection()).rejects.toThrow(new Error(`Driver "${configuration.settings.driverPath}" not found`)); + await expect(south.testConnection()).rejects.toThrow(new Error(`Driver not found. Check connection string and driver`)); expect((logger.error as jest.Mock).mock.calls).toEqual([ [`Unable to connect to database: ${connectionErrorMessage}`], [`Error from ODBC driver: ${error.odbcErrors[0].message}`] ]); - expect((logger.info as jest.Mock).mock.calls).toEqual([[`Testing connection on "${configuration.settings.host}"`]]); }); it('System table unreachable', async () => { @@ -560,15 +590,10 @@ describe('SouthODBC test connection', () => { }; (odbc.connect as jest.Mock).mockReturnValue(odbcConnection); - await expect(south.testConnection()).rejects.toThrow( - new Error(`Unable to read tables in database "${configuration.settings.database}", check logs`) - ); + await expect(south.testConnection()).rejects.toThrow(new Error(`Unable to read tables in database, check logs`)); expect(odbcConnection.close).toHaveBeenCalled(); - expect((logger.error as jest.Mock).mock.calls).toEqual([ - [`Unable to read tables in database "${configuration.settings.database}": ${errorMessage}`] - ]); - expect((logger.info as jest.Mock).mock.calls).toEqual([[`Testing connection on "${configuration.settings.host}"`]]); + expect((logger.error as jest.Mock).mock.calls).toEqual([[`Unable to read tables in database: ${errorMessage}`]]); }); it('Database has no tables', async () => { @@ -581,13 +606,12 @@ describe('SouthODBC test connection', () => { await expect(south.testConnection()).rejects.toThrow(new Error('Database has no table')); expect(odbcConnection.close).toHaveBeenCalled(); - expect((logger.warn as jest.Mock).mock.calls).toEqual([[`Database "${configuration.settings.database}" has no table`]]); - expect((logger.info as jest.Mock).mock.calls).toEqual([[`Testing connection on "${configuration.settings.host}"`]]); + expect((logger.warn as jest.Mock).mock.calls).toEqual([[`Database has no table`]]); }); it('Unable to connect to database without password', async () => { const errorMessage = 'Error connecting to database'; - configuration.settings.driverPath = 'myOdbcDriver'; + configuration.settings.connectionString = 'myOdbcDriver'; configuration.settings.password = ''; (odbc.connect as jest.Mock).mockImplementationOnce(() => { throw new NodeOdbcError(errorMessage, [{ code: -1, message: errorMessage, state: '' }]); @@ -599,6 +623,260 @@ describe('SouthODBC test connection', () => { [`Unable to connect to database: ${errorMessage}`], [`Error from ODBC driver: ${errorMessage}`] ]); - expect((logger.info as jest.Mock).mock.calls).toEqual([[`Testing connection on "${configuration.settings.host}"`]]); + }); +}); + +describe('SouthODBC odbc remote with authentication', () => { + const configuration: SouthConnectorDTO = { + id: 'southId', + name: 'south', + type: 'odbc', + description: 'my test connector', + enabled: true, + history: { + maxInstantPerItem: true, + maxReadInterval: 3600, + readDelay: 0 + }, + settings: { + remoteAgent: true, + agentUrl: 'http://localhost:2224', + connectionString: 'Driver={SQL Server};SERVER=127.0.0.1;TrustServerCertificate=yes', + password: 'password', + connectionTimeout: 1000, + requestTimeout: 1000 + } + }; + beforeEach(async () => { + jest.clearAllMocks(); + jest.useFakeTimers().setSystemTime(new Date(nowDateString)); + + (convertDateTimeToInstant as jest.Mock).mockImplementation(value => value); + + south = new SouthODBC(configuration, items, addValues, addFile, encryptionService, repositoryService, logger, 'baseFolder'); + }); + + it('should properly connect to remote agent and disconnect ', async () => { + await south.connect(); + expect(fetch).toHaveBeenCalledWith(`${configuration.settings.agentUrl}/api/odbc/${configuration.id}/connect`, { + method: 'PUT', + body: JSON.stringify({ + connectionString: configuration.settings.connectionString, + connectionTimeout: configuration.settings.connectionTimeout + }), + headers: { + 'Content-Type': 'application/json' + }, + timeout: configuration.settings.connectionTimeout + }); + + await south.disconnect(); + expect(fetch).toHaveBeenCalledWith(`${configuration.settings.agentUrl}/api/odbc/${configuration.id}/disconnect`, { + method: 'DELETE', + timeout: configuration.settings.connectionTimeout + }); + }); + + it('should properly run historyQuery', async () => { + const startTime = '2020-01-01T00:00:00.000Z'; + south.queryRemoteAgentData = jest.fn().mockReturnValueOnce('2023-02-01T00:00:00.000Z').mockReturnValue('2023-02-01T00:00:00.000Z'); + + await south.historyQuery(items, startTime, nowDateString); + expect(south.queryRemoteAgentData).toHaveBeenCalledTimes(3); + expect(south.queryRemoteAgentData).toHaveBeenCalledWith(items[0], startTime, nowDateString); + expect(south.queryRemoteAgentData).toHaveBeenCalledWith(items[1], '2023-02-01T00:00:00.000Z', nowDateString); + expect(south.queryRemoteAgentData).toHaveBeenCalledWith(items[2], '2023-02-01T00:00:00.000Z', nowDateString); + }); + + it('should get data from Remote agent', async () => { + const startTime = '2020-01-01T00:00:00.000Z'; + const endTime = '2022-01-01T00:00:00.000Z'; + + (fetch as unknown as jest.Mock) + .mockReturnValueOnce( + Promise.resolve({ + status: 200, + json: () => ({ + recordCount: 2, + content: [{ timestamp: '2020-02-01T00:00:00.000Z' }, { timestamp: '2020-03-01T00:00:00.000Z' }], + maxInstantRetrieved: '2020-03-01T00:00:00.000Z' + }) + }) + ) + .mockReturnValueOnce( + Promise.resolve({ + status: 200, + json: () => ({ + recordCount: 0, + content: [], + maxInstantRetrieved: '2020-03-01T00:00:00.000Z' + }) + }) + ); + + const result = await south.queryRemoteAgentData(items[0], startTime, endTime); + + expect(utils.logQuery).toHaveBeenCalledWith(items[0].settings.query, startTime, endTime, logger); + + expect(fetch).toHaveBeenCalledWith(`${configuration.settings.agentUrl}/api/odbc/${configuration.id}/read`, { + method: 'PUT', + body: JSON.stringify({ + connectionString: configuration.settings.connectionString, + sql: items[0].settings.query, + readTimeout: configuration.settings.requestTimeout, + timeColumn: items[0].settings.dateTimeFields[1].fieldName, + datasourceTimestampFormat: items[0].settings.dateTimeFields[1].format, + datasourceTimezone: items[0].settings.dateTimeFields[1].timezone, + delimiter: items[0].settings.serialization.delimiter, + outputTimestampFormat: items[0].settings.serialization.outputTimestampFormat, + outputTimezone: items[0].settings.serialization.outputTimezone + }), + headers: { + 'Content-Type': 'application/json' + } + }); + + expect(result).toEqual('2020-03-01T00:00:00.000Z'); + expect(persistResults).toHaveBeenCalledWith( + [{ timestamp: '2020-02-01T00:00:00.000Z' }, { timestamp: '2020-03-01T00:00:00.000Z' }], + items[0].settings.serialization, + configuration.name, + path.resolve('baseFolder', 'tmp'), + expect.any(Function), + expect.any(Function), + logger + ); + + await south.queryRemoteAgentData(items[0], startTime, endTime); + expect(logger.debug).toHaveBeenCalledWith(`No result found for item ${items[0].name}. Request done in 0 ms`); + }); + + it('should get data from Remote agent without datetime reference', async () => { + const startTime = '2020-01-01T00:00:00.000Z'; + const endTime = '2022-01-01T00:00:00.000Z'; + + (fetch as unknown as jest.Mock).mockReturnValueOnce( + Promise.resolve({ + status: 200, + json: () => ({ + recordCount: 2, + content: [{ timestamp: '2020-02-01T00:00:00.000Z' }, { timestamp: '2020-03-01T00:00:00.000Z' }], + maxInstantRetrieved: startTime + }) + }) + ); + + const result = await south.queryRemoteAgentData(items[1], startTime, endTime); + + expect(utils.logQuery).toHaveBeenCalledWith(items[1].settings.query, startTime, endTime, logger); + + expect(fetch).toHaveBeenCalledWith(`${configuration.settings.agentUrl}/api/odbc/${configuration.id}/read`, { + method: 'PUT', + body: JSON.stringify({ + connectionString: configuration.settings.connectionString, + sql: items[0].settings.query, + readTimeout: configuration.settings.requestTimeout, + delimiter: items[0].settings.serialization.delimiter, + outputTimestampFormat: items[0].settings.serialization.outputTimestampFormat, + outputTimezone: items[0].settings.serialization.outputTimezone + }), + headers: { + 'Content-Type': 'application/json' + } + }); + + expect(result).toEqual(startTime); + }); + + it('should manage query error', async () => { + const startTime = '2020-01-01T00:00:00.000Z'; + const endTime = '2022-01-01T00:00:00.000Z'; + + (fetch as unknown as jest.Mock) + .mockReturnValueOnce( + Promise.resolve({ + status: 400, + text: () => 'bad request' + }) + ) + .mockReturnValueOnce( + Promise.resolve({ + status: 500 + }) + ); + + await south.queryRemoteAgentData(items[0], startTime, endTime); + expect(logger.error).toHaveBeenCalledWith(`Error occurred when querying remote agent with status 400: bad request`); + + await south.queryRemoteAgentData(items[0], startTime, endTime); + expect(logger.error).toHaveBeenCalledWith(`Error occurred when querying remote agent with status 500`); + }); +}); + +describe('SouthODBC odbc remote test connection', () => { + const configuration: SouthConnectorDTO = { + id: 'southId', + name: 'south', + type: 'odbc', + description: 'my test connector', + enabled: true, + history: { + maxInstantPerItem: true, + maxReadInterval: 3600, + readDelay: 0 + }, + settings: { + remoteAgent: true, + agentUrl: 'http://localhost:2224', + connectionString: 'Driver={SQL Server};SERVER=127.0.0.1;TrustServerCertificate=yes', + password: 'password', + connectionTimeout: 1000, + requestTimeout: 1000 + } + }; + + beforeEach(async () => { + jest.clearAllMocks(); + jest.useFakeTimers().setSystemTime(new Date(nowDateString)); + + south = new SouthODBC(configuration, items, addValues, addFile, encryptionService, repositoryService, logger, 'baseFolder'); + }); + + it('should test connection successfully', async () => { + (fetch as unknown as jest.Mock).mockReturnValueOnce( + Promise.resolve({ + status: 200 + }) + ); + await south.testConnection(); + expect(logger.info).toHaveBeenCalledWith('Connected to remote odbc. Disconnecting...'); + expect(logger.info).toHaveBeenCalledWith( + `Testing ODBC OIBus Agent connection on ${configuration.settings.agentUrl} with "${configuration.settings.connectionString}"` + ); + }); + + it('should test connection fail', async () => { + (fetch as unknown as jest.Mock) + .mockReturnValueOnce( + Promise.resolve({ + status: 400, + text: () => 'bad request' + }) + ) + .mockReturnValueOnce( + Promise.resolve({ + status: 500, + text: () => 'another error' + }) + ); + await expect(south.testConnection()).rejects.toThrow( + new Error(`Error occurred when sending connect command to remote agent with status 400: bad request`) + ); + expect(logger.error).toHaveBeenCalledWith(`Error occurred when sending connect command to remote agent with status 400: bad request`); + + await expect(south.testConnection()).rejects.toThrow( + new Error(`Error occurred when sending connect command to remote agent with status 500`) + ); + expect(logger.error).toHaveBeenCalledWith(`Error occurred when sending connect command to remote agent with status 500`); }); }); diff --git a/backend/src/south/south-odbc/south-odbc.ts b/backend/src/south/south-odbc/south-odbc.ts index 55a02184ec..976ceaff22 100644 --- a/backend/src/south/south-odbc/south-odbc.ts +++ b/backend/src/south/south-odbc/south-odbc.ts @@ -11,6 +11,7 @@ import { Instant } from '../../../../shared/model/types'; import { DateTime } from 'luxon'; import { QueriesHistory, TestsConnection } from '../south-interface'; import { SouthODBCItemSettings, SouthODBCSettings } from '../../../../shared/model/south-settings.model'; +import fetch from 'node-fetch'; let odbc: any | null = null; // @ts-ignore @@ -30,6 +31,7 @@ export default class SouthODBC extends SouthConnector, @@ -53,12 +55,49 @@ export default class SouthODBC extends SouthConnector { + if (this.connector.settings.remoteAgent) { + this.connected = false; + const headers: Record = {}; + headers['Content-Type'] = 'application/json'; + const fetchOptions = { + method: 'PUT', + body: JSON.stringify({ + connectionString: this.connector.settings.connectionString, + connectionTimeout: this.connector.settings.connectionTimeout + }), + headers, + timeout: this.connector.settings.connectionTimeout + }; + + await fetch(`${this.connector.settings.agentUrl}/api/odbc/${this.connector.id}/connect`, fetchOptions); + this.connected = true; + } + await super.connect(); + } + + async disconnect(): Promise { + this.connected = false; + if (this.connector.settings.remoteAgent) { + const fetchOptions = { method: 'DELETE', timeout: this.connector.settings.connectionTimeout }; + await fetch(`${this.connector.settings.agentUrl}/api/odbc/${this.connector.id}/disconnect`, fetchOptions); + } + await super.disconnect(); + } + override async testConnection(): Promise { + if (this.connector.settings.remoteAgent) { + await this.testAgentConnection(); + } else { + await this.testOdbcConnection(); + } + } + + async testOdbcConnection(): Promise { if (!odbc) { throw new Error('odbc library not loaded'); } - - this.logger.info(`Testing connection on "${this.connector.settings.host}"`); + this.logger.info(`Testing ODBC connection with "${this.connector.settings.connectionString}"`); let connection; try { @@ -67,18 +106,14 @@ export default class SouthODBC extends SouthConnector `${row.table_name}: [${row.columns}]`).join(',\n'); this.logger.info('Database is live with tables (table:[columns]):\n%s', tablesString); } + async testAgentConnection(): Promise { + this.logger.info( + `Testing ODBC OIBus Agent connection on ${this.connector.settings.agentUrl} with "${this.connector.settings.connectionString}"` + ); + + const headers: Record = {}; + headers['Content-Type'] = 'application/json'; + const fetchOptions = { + method: 'PUT', + body: JSON.stringify({ + connectionString: this.connector.settings.connectionString, + connectionTimeout: this.connector.settings.connectionTimeout + }), + headers + }; + const response = await fetch(`${this.connector.settings.agentUrl!}/api/odbc/${this.connector.id}/connect`, fetchOptions); + if (response.status === 200) { + this.logger.info('Connected to remote odbc. Disconnecting...'); + await fetch(`${this.connector.settings.agentUrl}/api/odbc/${this.connector.id}/disconnect`, { method: 'DELETE' }); + } else if (response.status === 400) { + const errorMessage = await response.text(); + this.logger.error(`Error occurred when sending connect command to remote agent with status ${response.status}: ${errorMessage}`); + throw new Error(`Error occurred when sending connect command to remote agent with status ${response.status}: ${errorMessage}`); + } else { + this.logger.error(`Error occurred when sending connect command to remote agent with status ${response.status}`); + throw new Error(`Error occurred when sending connect command to remote agent with status ${response.status}`); + } + } + /** * Get entries from the database between startTime and endTime (if used in the SQL query) * and write them into a CSV file and send it to the engine. @@ -141,38 +199,52 @@ export default class SouthODBC extends SouthConnector = await this.queryData(item, updatedStartTime, endTime); + if (this.connector.settings.remoteAgent) { + updatedStartTime = await this.queryRemoteAgentData(item, updatedStartTime, endTime); + } else { + updatedStartTime = await this.queryOdbcData(item, updatedStartTime, endTime); + } + } + return updatedStartTime; + } + + async queryRemoteAgentData(item: SouthConnectorItemDTO, startTime: Instant, endTime: Instant): Promise { + let updatedStartTime = startTime; + const startRequest = DateTime.now().toMillis(); + + const headers: Record = {}; + headers['Content-Type'] = 'application/json'; + + const referenceTimestampField = item.settings.dateTimeFields.find(dateTimeField => dateTimeField.useAsReference); + const odbcStartTime = referenceTimestampField ? formatInstant(startTime, referenceTimestampField) : startTime; + const odbcEndTime = referenceTimestampField ? formatInstant(endTime, referenceTimestampField) : endTime; + const adaptedQuery = item.settings.query.replace(/@StartTime/g, `${odbcStartTime}`).replace(/@EndTime/g, `${odbcEndTime}`); + logQuery(adaptedQuery, odbcStartTime, odbcEndTime, this.logger); + + const fetchOptions = { + method: 'PUT', + body: JSON.stringify({ + connectionString: this.connector.settings.connectionString, + sql: adaptedQuery, + readTimeout: this.connector.settings.requestTimeout, + timeColumn: referenceTimestampField?.fieldName, + datasourceTimestampFormat: referenceTimestampField?.format, + datasourceTimezone: referenceTimestampField?.timezone, + delimiter: item.settings.serialization.delimiter, + outputTimestampFormat: item.settings.serialization.outputTimestampFormat, + outputTimezone: item.settings.serialization.outputTimezone + }), + headers + }; + const response = await fetch(`${this.connector.settings.agentUrl}/api/odbc/${this.connector.id}/read`, fetchOptions); + if (response.status === 200) { + const result: { recordCount: number; content: Array; maxInstantRetrieved: Instant } = await response.json(); const requestDuration = DateTime.now().toMillis() - startRequest; + this.logger.info(`Found ${result.recordCount} results for item ${item.name} in ${requestDuration} ms`); - if (result.length > 0) { - this.logger.info(`Found ${result.length} results for item ${item.name} in ${requestDuration} ms`); - - const formattedResult = result.map(entry => { - const formattedEntry: Record = {}; - Object.entries(entry).forEach(([key, value]) => { - const datetimeField = item.settings.dateTimeFields.find(dateTimeField => dateTimeField.fieldName === key); - if (!datetimeField) { - formattedEntry[key] = value; - } else { - const entryDate = convertDateTimeToInstant(value, datetimeField); - if (datetimeField.useAsReference) { - if (entryDate > updatedStartTime) { - updatedStartTime = entryDate; - } - } - formattedEntry[key] = formatInstant(entryDate, { - type: 'string', - format: item.settings.serialization.outputTimestampFormat, - timezone: item.settings.serialization.outputTimezone, - locale: 'en-En' - }); - } - }); - return formattedEntry; - }); + if (result.content.length > 0) { await persistResults( - formattedResult, + result.content, item.settings.serialization, this.connector.name, this.tmpFolder, @@ -180,34 +252,42 @@ export default class SouthODBC extends SouthConnector updatedStartTime) { + updatedStartTime = result.maxInstantRetrieved; + } } else { this.logger.debug(`No result found for item ${item.name}. Request done in ${requestDuration} ms`); } + } else if (response.status === 400) { + const errorMessage = await response.text(); + this.logger.error(`Error occurred when querying remote agent with status ${response.status}: ${errorMessage}`); + } else { + this.logger.error(`Error occurred when querying remote agent with status ${response.status}`); } + return updatedStartTime; } - /** - * Apply the SQL query to the target ODBC database - */ - async queryData(item: SouthConnectorItemDTO, startTime: Instant, endTime: Instant) { + async queryOdbcData(item: SouthConnectorItemDTO, startTime: Instant, endTime: Instant): Promise { if (!odbc) { throw new Error('odbc library not loaded'); } + let updatedStartTime = startTime; + const startRequest = DateTime.now().toMillis(); + let result: Array = []; let connection; try { const connectionConfig = await this.createConnectionConfig(this.connector.settings); connection = await odbc.connect(connectionConfig); - const referenceTimestampField = item.settings.dateTimeFields.find(dateTimeField => dateTimeField.useAsReference) || null; - const odbcStartTime = referenceTimestampField == null ? startTime : formatInstant(startTime, referenceTimestampField); - const odbcEndTime = referenceTimestampField == null ? endTime : formatInstant(endTime, referenceTimestampField); + const referenceTimestampField = item.settings.dateTimeFields.find(dateTimeField => dateTimeField.useAsReference); + const odbcStartTime = referenceTimestampField ? formatInstant(startTime, referenceTimestampField) : startTime; + const odbcEndTime = referenceTimestampField ? formatInstant(endTime, referenceTimestampField) : endTime; const adaptedQuery = item.settings.query.replace(/@StartTime/g, `${odbcStartTime}`).replace(/@EndTime/g, `${odbcEndTime}`); logQuery(adaptedQuery, odbcStartTime, odbcEndTime, this.logger); - const data = await connection.query(adaptedQuery); + result = await connection.query(adaptedQuery); await connection.close(); - return data; } catch (error: any) { if (error.odbcErrors?.length > 0) { this.logOdbcErrors(error.odbcErrors); @@ -217,25 +297,60 @@ export default class SouthODBC extends SouthConnector 0) { + this.logger.info(`Found ${result.length} results for item ${item.name} in ${requestDuration} ms`); + + const formattedResult = result.map(entry => { + const formattedEntry: Record = {}; + Object.entries(entry).forEach(([key, value]) => { + const datetimeField = item.settings.dateTimeFields.find(dateTimeField => dateTimeField.fieldName === key); + if (!datetimeField) { + formattedEntry[key] = value; + } else { + const entryDate = convertDateTimeToInstant(value, datetimeField); + if (datetimeField.useAsReference) { + if (entryDate > updatedStartTime) { + updatedStartTime = entryDate; + } + } + formattedEntry[key] = formatInstant(entryDate, { + type: 'string', + format: item.settings.serialization.outputTimestampFormat, + timezone: item.settings.serialization.outputTimezone, + locale: 'en-En' + }); + } + }); + return formattedEntry; + }); + await persistResults( + formattedResult, + item.settings.serialization, + this.connector.name, + this.tmpFolder, + this.addFile.bind(this), + this.addValues.bind(this), + this.logger + ); + } else { + this.logger.debug(`No result found for item ${item.name}. Request done in ${requestDuration} ms`); + } + return updatedStartTime; } async createConnectionConfig(settings: SouthODBCSettings): Promise<{ connectionString: string; connectionTimeout?: number; }> { - let connectionString = `Driver=${settings.driverPath};SERVER=${settings.host};PORT=${settings.port};`; - if (settings.trustServerCertificate) { - connectionString += `TrustServerCertificate=yes;`; - } - if (settings.database) { - connectionString += `Database=${settings.database};`; - } - if (settings.username) { - connectionString += `UID=${settings.username};`; - } + let connectionString = settings.connectionString; - if (settings.username && settings.password) { + if (settings.password) { this.logger.debug(`Connecting with connection string ${connectionString}PWD=;`); + if (!connectionString.endsWith(';')) { + connectionString += ';'; + } connectionString += `PWD=${await this.encryptionService.decryptText(settings.password)};`; } else { this.logger.debug(`Connecting with connection string ${connectionString}`); @@ -251,7 +366,7 @@ export default class SouthODBC extends SouthConnector { + throw new Error('bad'); +}); +jest.mock('../../service/utils'); + +const database = new DatabaseMock(); +jest.mock( + '../../service/south-cache.service', + () => + function () { + return { + createSouthCacheScanModeTable: jest.fn(), + southCacheRepository: { + database + } + }; + } +); + +jest.mock( + '../../service/south-connector-metrics.service', + () => + function () { + return { + initMetrics: jest.fn(), + updateMetrics: jest.fn(), + get stream() { + return { stream: 'myStream' }; + }, + metrics: { + numberOfValuesRetrieved: 1, + numberOfFilesRetrieved: 1 + } + }; + } +); +const addValues = jest.fn(); +const addFile = jest.fn(); + +const logger: pino.Logger = new PinoLogger(); +const encryptionService: EncryptionService = new EncryptionServiceMock('', ''); +const repositoryService: RepositoryService = new RepositoryServiceMock(); +const items: Array> = []; + +// Spy on console info and error +jest.spyOn(global.console, 'info').mockImplementation(() => {}); +jest.spyOn(global.console, 'error').mockImplementation(() => {}); + +let south: SouthOracle; +describe('SouthOracle with authentication', () => { + const connector: SouthConnectorDTO = { + id: 'southId', + name: 'south', + type: 'oracle', + description: 'my test connector', + enabled: true, + history: { + maxInstantPerItem: true, + maxReadInterval: 3600, + readDelay: 0 + }, + settings: { + host: 'localhost', + port: 1521, + database: 'db', + username: 'username', + password: 'password', + connectionTimeout: 1000, + requestTimeout: 1000 + } + }; + beforeEach(async () => { + jest.clearAllMocks(); + + south = new SouthOracle(connector, items, addValues, addFile, encryptionService, repositoryService, logger, 'baseFolder'); + }); + + it('should throw error if library not loaded', async () => { + await expect(south.testConnection()).rejects.toThrow(new Error('oracledb library not loaded')); + + const startTime = '2020-01-01T00:00:00.000Z'; + const endTime = '2022-01-01T00:00:00.000Z'; + await expect(south.queryData({} as SouthConnectorItemDTO, startTime, endTime)).rejects.toThrow( + new Error('oracledb library not loaded') + ); + }); +}); diff --git a/backend/src/south/south-oracle/south-oracle.spec.ts b/backend/src/south/south-oracle/south-oracle.spec.ts index 3636d03eda..620615af09 100644 --- a/backend/src/south/south-oracle/south-oracle.spec.ts +++ b/backend/src/south/south-oracle/south-oracle.spec.ts @@ -104,24 +104,7 @@ const items: Array> = [ connectorId: 'southId', settings: { query: 'SELECT * FROM table', - dateTimeFields: [ - { - fieldName: 'anotherTimestamp', - useAsReference: false, - type: 'unix-epoch-ms', - timezone: null, - format: null, - locale: null - } as unknown as SouthOracleItemSettingsDateTimeFields, - { - fieldName: 'timestamp', - useAsReference: true, - type: 'string', - timezone: 'Europe/Paris', - format: 'yyyy-MM-dd HH:mm:ss.SSS', - locale: 'en-US' - } - ], + dateTimeFields: [], serialization: { type: 'csv', filename: 'sql-@CurrentDate.csv', @@ -171,6 +154,10 @@ const items: Array> = [ } ]; +// Spy on console info and error +jest.spyOn(global.console, 'info').mockImplementation(() => {}); +jest.spyOn(global.console, 'error').mockImplementation(() => {}); + const nowDateString = '2020-02-02T02:02:02.222Z'; let south: SouthOracle; @@ -294,6 +281,36 @@ describe('SouthOracle with authentication', () => { expect(nullResult).toEqual([]); }); + it('should get data from Oracle without datetime reference', async () => { + const startTime = '2020-01-01T00:00:00.000Z'; + const endTime = '2022-01-01T00:00:00.000Z'; + + (generateReplacementParameters as jest.Mock).mockReturnValue({ + startTime: DateTime.fromISO(startTime).toFormat('yyyy-MM-dd HH:mm:ss.SSS'), + endTime: DateTime.fromISO(endTime).toFormat('yyyy-MM-dd HH:mm:ss.SSS') + }); + const oracleConnection = { + callTimeout: connector.settings.requestTimeout, + close: jest.fn(), + execute: jest + .fn() + .mockReturnValueOnce({ + rows: [{ timestamp: '2020-02-01T00:00:00.000Z' }, { timestamp: '2020-03-01T00:00:00.000Z' }] + }) + .mockReturnValue({ rows: null }) + }; + (oracledb.getConnection as jest.Mock).mockReturnValue(oracleConnection); + + await south.queryData(items[1], startTime, endTime); + + expect(utils.logQuery).toHaveBeenCalledWith(items[1].settings.query, startTime, endTime, logger); + + expect(generateReplacementParameters).toHaveBeenCalledWith(items[1].settings.query, startTime, endTime); + + const nullResult = await south.queryData(items[1], startTime, endTime); + expect(nullResult).toEqual([]); + }); + it('should manage query error', async () => { const startTime = '2020-01-01T00:00:00.000Z'; const endTime = '2022-01-01T00:00:00.000Z'; diff --git a/backend/src/south/south-oracle/south-oracle.ts b/backend/src/south/south-oracle/south-oracle.ts index 75763be205..ccc77e537b 100644 --- a/backend/src/south/south-oracle/south-oracle.ts +++ b/backend/src/south/south-oracle/south-oracle.ts @@ -208,9 +208,9 @@ export default class SouthOracle connectString: `${this.connector.settings.host}:${this.connector.settings.port}/${this.connector.settings.database}` }; - const referenceTimestampField = item.settings.dateTimeFields.find(dateTimeField => dateTimeField.useAsReference) || null; - const oracleStartTime = referenceTimestampField == null ? startTime : formatInstant(startTime, referenceTimestampField); - const oracleEndTime = referenceTimestampField == null ? endTime : formatInstant(endTime, referenceTimestampField); + const referenceTimestampField = item.settings.dateTimeFields.find(dateTimeField => dateTimeField.useAsReference); + const oracleStartTime = referenceTimestampField ? formatInstant(startTime, referenceTimestampField) : startTime; + const oracleEndTime = referenceTimestampField ? formatInstant(endTime, referenceTimestampField) : endTime; logQuery(item.settings.query, oracleStartTime, oracleEndTime, this.logger); let connection; diff --git a/shared/model/south-settings.model.ts b/shared/model/south-settings.model.ts index 22adbe4c8a..91531f6dde 100644 --- a/shared/model/south-settings.model.ts +++ b/shared/model/south-settings.model.ts @@ -207,7 +207,7 @@ export interface SouthMSSQLSettings extends BaseSouthSettings { username: string | null; password: string | null; domain: string | null; - encryption: boolean | null; + encryption: boolean; trustServerCertificate: boolean; connectionTimeout: number; requestTimeout: number; @@ -224,14 +224,12 @@ export interface SouthMySQLSettings extends BaseSouthSettings { } export interface SouthODBCSettings extends BaseSouthSettings { - driverPath: string; - host: string; - port: number; - database: string; - username: string | null; + remoteAgent: boolean; + agentUrl?: string; + connectionString: string; password: string | null; connectionTimeout: number; - trustServerCertificate: boolean; + requestTimeout: number; } export interface SouthOIAnalyticsSettings extends BaseSouthSettings {