diff --git a/frontend/package.json b/frontend/package.json index 49018a21306..b336edae8b7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -16,7 +16,8 @@ "dev-workorder": "pnpm -r --filter ./providers/workorder run dev", "gen:theme-typings": "pnpm chakra-cli tokens packages/ui/src/theme/theme.ts --out node_modules/.pnpm/node_modules/@chakra-ui/styled-system/dist/theming.types.d.ts", "postinstall": "pnpm run gen:theme-typings", - "prepare": "cd .. && husky frontend/.husky" + "prepare": "cd .. && husky frontend/.husky", + "build-packages": "pnpm -r --filter ./packages/client-sdk run build" }, "workspaces": [ "./packages/*", diff --git a/frontend/packages/ui/src/components/Menu/index.tsx b/frontend/packages/ui/src/components/Menu/index.tsx index eaad5283d09..d607f1ff834 100644 --- a/frontend/packages/ui/src/components/Menu/index.tsx +++ b/frontend/packages/ui/src/components/Menu/index.tsx @@ -28,6 +28,8 @@ export const SealosMenu = ({ width, Button, menuList }: Props) => { {Button} =12'} dev: false + /@tanstack/virtual-core@3.10.8: + resolution: {integrity: sha512-PBu00mtt95jbKFi6Llk9aik8bnR3tR/oQP1o3TSi+iG//+Q2RTIzCEgKkHG8BB86kxMNW6O8wku+Lmi+QFR6jA==} + dev: false + /@testing-library/dom@9.3.3: resolution: {integrity: sha512-fB0R+fa3AUqbLHWyxXa2kGVtf1Fe1ZZFr0Zp6AIbIAzXb2mKbEXl+PCQNUOaq5lbTab5tfctfXRNsWXxa2f7Aw==} engines: {node: '>=14'} @@ -10263,6 +10290,12 @@ packages: resolution: {integrity: sha512-k7kRA033QNtC+gLc4VPlfnue58CM1iQLgn1IMAU8VPHGOj7oIHPp9UlhedEnD/Gl8evoCjwkZjlBORtZ3JByUA==} dev: true + /@types/papaparse@5.3.15: + resolution: {integrity: sha512-JHe6vF6x/8Z85nCX4yFdDslN11d+1pr12E526X8WAfhadOeaOTx5AuIkvDKIBopfvlzpzkdMx4YyvSKCM9oqtw==} + dependencies: + '@types/node': 20.10.0 + dev: true + /@types/parse-json@4.0.2: resolution: {integrity: sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==} @@ -18952,6 +18985,10 @@ packages: resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==} dev: false + /papaparse@5.4.1: + resolution: {integrity: sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw==} + dev: false + /parent-module@1.0.1: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} engines: {node: '>=6'} diff --git a/frontend/providers/dbprovider/package.json b/frontend/providers/dbprovider/package.json index 49a886939f7..4f8ec263d64 100644 --- a/frontend/providers/dbprovider/package.json +++ b/frontend/providers/dbprovider/package.json @@ -11,7 +11,6 @@ "unlink-sdk": "yalc remove --all && pnpm install sealos-desktop-sdk" }, "dependencies": { - "@sealos/driver": "workspace:^", "@chakra-ui/anatomy": "^2.2.1", "@chakra-ui/icons": "^2.1.1", "@chakra-ui/next-js": "^2.1.5", @@ -21,8 +20,11 @@ "@emotion/styled": "^11.11.0", "@kubernetes/client-node": "^0.18.1", "@next/font": "13.1.6", + "@sealos/driver": "workspace:^", "@sealos/ui": "workspace:^", "@tanstack/react-query": "^4.35.3", + "@tanstack/react-table": "^8.10.7", + "@tanstack/react-virtual": "^3.10.8", "ansi_up": "^5.2.1", "axios": "^1.5.1", "date-fns": "^2.30.0", @@ -43,6 +45,7 @@ "next": "13.1.6", "next-i18next": "^15.3.0", "nprogress": "^0.2.0", + "papaparse": "^5.4.1", "prettier": "^2.8.8", "react": "18.2.0", "react-day-picker": "^8.8.2", @@ -65,6 +68,7 @@ "@types/multer": "^1.4.10", "@types/node": "18.13.0", "@types/nprogress": "^0.2.1", + "@types/papaparse": "^5.3.15", "@types/react": "18.2.37", "@types/react-dom": "18.0.10", "@types/react-syntax-highlighter": "^15.5.7", diff --git a/frontend/providers/dbprovider/public/locales/en/common.json b/frontend/providers/dbprovider/public/locales/en/common.json index 944825846ae..9f051b919e5 100644 --- a/frontend/providers/dbprovider/public/locales/en/common.json +++ b/frontend/providers/dbprovider/public/locales/en/common.json @@ -23,6 +23,7 @@ "Migrating": "Migrating", "Monday": "Monday", "Option": "Optional", + "Page": "Page", "Password": "Password", "Pause": "Pause", "Paused": "Paused", @@ -45,6 +46,7 @@ "Success": "succeeded", "Sunday": "Sun", "Thursday": "Thu", + "Total": "total", "Tuesday": "Tue", "Type": "Type", "Unknown": "Unknown", @@ -173,6 +175,15 @@ "duration_of_transaction": "Transaction Duration", "enable_external_network_access": "Allow public network access", "enter_save": "Press Enter to save. 'All' exports the entire database.", + "error_log": { + "analysis": "Log Analysis", + "collection_time": "Collection Time", + "content": "Information", + "error_log": "Error Log", + "runtime_log": "Run Log", + "search_content": "Search Log Content", + "slow_query": "Slow Log" + }, "event_analyze": "Intelligent Analytics", "event_analyze_error": "Intelligent analytics error", "external_address": "Public Domain", @@ -306,4 +317,4 @@ "within_5_minutes": "Within 5 minutes", "yaml_file": "YAML", "you_have_successfully_deployed_database": "You have successfully deployed and created a database!" -} +} \ No newline at end of file diff --git a/frontend/providers/dbprovider/public/locales/zh/common.json b/frontend/providers/dbprovider/public/locales/zh/common.json index 38c656d93aa..78f4a432032 100644 --- a/frontend/providers/dbprovider/public/locales/zh/common.json +++ b/frontend/providers/dbprovider/public/locales/zh/common.json @@ -23,6 +23,7 @@ "Migrating": "正在迁移", "Monday": "周一", "Option": "选填", + "Page": "页", "Password": "密码", "Pause": "暂停", "Paused": "已暂停", @@ -45,6 +46,7 @@ "Success": "成功", "Sunday": "周日", "Thursday": "周四", + "Total": "总数", "Tuesday": "周二", "Type": "类型", "Unknown": "未知", @@ -173,6 +175,15 @@ "duration_of_transaction": "事务持续时间", "enable_external_network_access": "开启外网访问", "enter_save": "回车保存 && All 代表导出整个库", + "error_log": { + "analysis": "日志分析", + "collection_time": "采集时间", + "content": "信息", + "error_log": "错误日志", + "runtime_log": "运行日志", + "search_content": "搜索日志内容", + "slow_query": "慢日志" + }, "event_analyze": "智能分析", "event_analyze_error": "智能分析出错了~", "external_address": "外网地址", diff --git a/frontend/providers/dbprovider/src/api/db.ts b/frontend/providers/dbprovider/src/api/db.ts index b771bed0ec0..58d416b18ea 100644 --- a/frontend/providers/dbprovider/src/api/db.ts +++ b/frontend/providers/dbprovider/src/api/db.ts @@ -1,18 +1,22 @@ -import { GET, POST, DELETE } from '@/services/request'; -import { adaptDBListItem, adaptDBDetail, adaptPod, adaptEvents } from '@/utils/adapt'; +import type { SecretResponse } from '@/pages/api/getSecretByName'; +import { DELETE, GET, POST } from '@/services/request'; +import { KbPgClusterType } from '@/types/cluster'; import type { BackupItemType, DBEditType, DBType, OpsRequestItemType, - PodDetailType + PodDetailType, + SupportReconfigureDBType } from '@/types/db'; +import { LogTypeEnum } from '@/constants/log'; +import { MonitorChartDataResult } from '@/types/monitor'; +import { adaptDBDetail, adaptDBListItem, adaptEvents, adaptPod } from '@/utils/adapt'; import { json2Restart } from '@/utils/json2Yaml'; -import { json2StartOrStop } from '../utils/json2Yaml'; -import type { SecretResponse } from '@/pages/api/getSecretByName'; +import { TFile } from '@/utils/kubeFileSystem'; +import { LogResult } from '@/utils/logParsers/LogParser'; import { V1Service, V1StatefulSet } from '@kubernetes/client-node'; -import { KbPgClusterType } from '@/types/cluster'; -import { MonitorChartDataResult } from '@/types/monitor'; +import { json2StartOrStop } from '../utils/json2Yaml'; export const getMyDBList = () => GET('/api/getDBList').then((data) => data.map(adaptDBListItem)); @@ -111,3 +115,42 @@ export const getOpsRequest = ({ label, dbType }); + +export const getLogFiles = ({ + podName, + dbType, + logType +}: { + podName: string; + dbType: SupportReconfigureDBType; + logType: LogTypeEnum; +}) => + POST(`/api/logs/getFiles`, { + podName, + dbType, + logType + }); + +export const getLogContent = ({ + logPath, + page, + pageSize, + dbType, + logType, + podName +}: { + logPath: string; + page: number; + pageSize: number; + dbType: SupportReconfigureDBType; + logType: LogTypeEnum; + podName: string; +}) => + POST(`/api/logs/get`, { + logPath, + page, + pageSize, + dbType, + logType, + podName + }); diff --git a/frontend/providers/dbprovider/src/components/BaseTable/SwitchPage.tsx b/frontend/providers/dbprovider/src/components/BaseTable/SwitchPage.tsx new file mode 100644 index 00000000000..fe1c44dd4e2 --- /dev/null +++ b/frontend/providers/dbprovider/src/components/BaseTable/SwitchPage.tsx @@ -0,0 +1,158 @@ +import { Button, ButtonProps, Flex, FlexProps, Text } from '@chakra-ui/react'; +import { useTranslation } from 'next-i18next'; + +import { Icon, IconProps } from '@chakra-ui/react'; + +export function ToLeftIcon(props: IconProps) { + return ( + + + + ); +} + +export function RightFirstIcon(props: IconProps) { + return ( + + + + ); +} + +export function SwitchPage({ + totalPage, + totalItem, + pageSize, + currentPage, + setCurrentPage, + isPreviousData, + ...props +}: { + currentPage: number; + totalPage: number; + totalItem: number; + pageSize: number; + isPreviousData?: boolean; + setCurrentPage: (idx: number) => void; +} & FlexProps) { + const { t } = useTranslation(); + const switchStyle: ButtonProps = { + width: '24px', + height: '24px', + minW: '0', + background: 'grayModern.250', + flexGrow: '0', + borderRadius: 'full', + // variant:'unstyled', + _hover: { + background: 'grayModern.150', + minW: '0' + }, + _disabled: { + borderRadius: 'full', + background: 'grayModern.150', + cursor: 'not-allowed', + minW: '0' + } + }; + return ( + + + {t('Total')}: + + + {totalItem} + + + + + {currentPage} + / + {totalPage} + + + + + {pageSize} + + + /{t('Page')} + + + ); +} diff --git a/frontend/providers/dbprovider/src/components/BaseTable/baseTable.tsx b/frontend/providers/dbprovider/src/components/BaseTable/baseTable.tsx new file mode 100644 index 00000000000..fc3832a73f4 --- /dev/null +++ b/frontend/providers/dbprovider/src/components/BaseTable/baseTable.tsx @@ -0,0 +1,76 @@ +import { + Spinner, + Table, + TableContainer, + TableContainerProps, + Tbody, + Td, + Th, + Thead, + Tr +} from '@chakra-ui/react'; +import { Table as ReactTable, flexRender } from '@tanstack/react-table'; + +export function BaseTable({ + table, + isLoading, + ...props +}: { table: ReactTable; isLoading: boolean } & TableContainerProps) { + return ( + + + + {table.getHeaderGroups().map((headers) => { + return ( + + {headers.headers.map((header, i) => { + return ( + + ); + })} + + ); + })} + + + {isLoading ? ( + + + + ) : ( + table.getRowModel().rows.map((item) => { + return ( + + {item.getAllCells().map((cell, i) => { + return ( + + ); + })} + + ); + }) + )} + +
+ {flexRender(header.column.columnDef.header, header.getContext())} +
+ +
+ {flexRender(cell.column.columnDef.cell, cell.getContext())} +
+ + ); +} diff --git a/frontend/providers/dbprovider/src/components/Icon/icons/backup.svg b/frontend/providers/dbprovider/src/components/Icon/icons/backup.svg new file mode 100644 index 00000000000..6bc2be8a53e --- /dev/null +++ b/frontend/providers/dbprovider/src/components/Icon/icons/backup.svg @@ -0,0 +1,4 @@ + + + + diff --git a/frontend/providers/dbprovider/src/components/Icon/icons/config.svg b/frontend/providers/dbprovider/src/components/Icon/icons/config.svg new file mode 100644 index 00000000000..aff15fbd251 --- /dev/null +++ b/frontend/providers/dbprovider/src/components/Icon/icons/config.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/frontend/providers/dbprovider/src/components/Icon/icons/file.svg b/frontend/providers/dbprovider/src/components/Icon/icons/file.svg new file mode 100644 index 00000000000..e24054c2dfc --- /dev/null +++ b/frontend/providers/dbprovider/src/components/Icon/icons/file.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/frontend/providers/dbprovider/src/components/Icon/icons/import.svg b/frontend/providers/dbprovider/src/components/Icon/icons/import.svg new file mode 100644 index 00000000000..d57c59dccd2 --- /dev/null +++ b/frontend/providers/dbprovider/src/components/Icon/icons/import.svg @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/frontend/providers/dbprovider/src/components/Icon/icons/instance.svg b/frontend/providers/dbprovider/src/components/Icon/icons/instance.svg new file mode 100644 index 00000000000..864e27c4cc4 --- /dev/null +++ b/frontend/providers/dbprovider/src/components/Icon/icons/instance.svg @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/frontend/providers/dbprovider/src/components/Icon/icons/monitor.svg b/frontend/providers/dbprovider/src/components/Icon/icons/monitor.svg new file mode 100644 index 00000000000..185a68a8b10 --- /dev/null +++ b/frontend/providers/dbprovider/src/components/Icon/icons/monitor.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/frontend/providers/dbprovider/src/components/Icon/icons/time.svg b/frontend/providers/dbprovider/src/components/Icon/icons/time.svg new file mode 100644 index 00000000000..061580378a0 --- /dev/null +++ b/frontend/providers/dbprovider/src/components/Icon/icons/time.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/frontend/providers/dbprovider/src/components/Icon/index.tsx b/frontend/providers/dbprovider/src/components/Icon/index.tsx index 1ceaa31a029..3e03b06897d 100644 --- a/frontend/providers/dbprovider/src/components/Icon/index.tsx +++ b/frontend/providers/dbprovider/src/components/Icon/index.tsx @@ -46,7 +46,14 @@ const map = { upload: require('./icons/upload.svg').default, target: require('./icons/target.svg').default, gift: require('./icons/gift.svg').default, + time: require('./icons/time.svg').default, help: require('./icons/help.svg').default, + backup: require('./icons/backup.svg').default, + instance: require('./icons/instance.svg').default, + import: require('./icons/import.svg').default, + file: require('./icons/file.svg').default, + config: require('./icons/config.svg').default, + monitor: require('./icons/monitor.svg').default, arrowDown: require('./icons/arrowDown.svg').default, docs: require('./icons/docs.svg').default }; diff --git a/frontend/providers/dbprovider/src/constants/log.ts b/frontend/providers/dbprovider/src/constants/log.ts new file mode 100644 index 00000000000..1a8f694d855 --- /dev/null +++ b/frontend/providers/dbprovider/src/constants/log.ts @@ -0,0 +1,72 @@ +import { SupportReconfigureDBType } from '@/types/db'; +import { TFile } from '@/utils/kubeFileSystem'; + +export enum LogTypeEnum { + RuntimeLog = 'runtimeLog', + SlowQuery = 'slowQuery', + ErrorLog = 'errorLog' +} + +export type LogConfig = { + path: string; + containerNames: string[]; + filter: (files: TFile[]) => TFile[]; +}; + +export type LoggingConfiguration = { + [LogTypeEnum.RuntimeLog]?: LogConfig; + [LogTypeEnum.SlowQuery]?: LogConfig; + [LogTypeEnum.ErrorLog]?: LogConfig; +}; + +export const ServiceLogConfigs: Record = { + redis: { + [LogTypeEnum.RuntimeLog]: { + path: '/data/running.log', + containerNames: ['redis', 'lorry'], + filter: (files: TFile[]) => + files + .filter((f) => f.size > 0 && f.name.toLowerCase().endsWith('.log')) + .sort((a, b) => b.updateTime.getTime() - a.updateTime.getTime()) + } + }, + postgresql: { + [LogTypeEnum.RuntimeLog]: { + path: '/home/postgres/pgdata/pgroot/pg_log', + containerNames: ['postgresql', 'lorry'], + filter: (files: TFile[]) => + files + .filter((f) => f.size > 0 && f.name.toLowerCase().endsWith('.csv')) + .sort((a, b) => b.updateTime.getTime() - a.updateTime.getTime()) + } + }, + mongodb: { + [LogTypeEnum.RuntimeLog]: { + path: '/data/mongodb/mongodb.log', + containerNames: ['mongodb'], + filter: (files: TFile[]) => { + return files; + } + } + }, + 'apecloud-mysql': { + [LogTypeEnum.ErrorLog]: { + path: '/data/mysql/log', + containerNames: ['mysql', 'lorry'], + filter: (files: TFile[]) => + files + .filter((f) => f.size > 0 && f.name.toLowerCase().includes('mysqld-error')) + .sort((a, b) => b.updateTime.getTime() - a.updateTime.getTime()) + }, + [LogTypeEnum.SlowQuery]: { + path: '/data/mysql/log', + containerNames: ['mysql', 'lorry'], + filter: (files: TFile[]) => { + console.log('slow query files:', files); + return files + .filter((f) => f.size > 1024 && f.name.toLowerCase().includes('slow-query')) + .sort((a, b) => b.updateTime.getTime() - a.updateTime.getTime()); + } + } + } +}; diff --git a/frontend/providers/dbprovider/src/pages/api/getEnv.ts b/frontend/providers/dbprovider/src/pages/api/getEnv.ts index f7efd0c6ec5..02701531213 100644 --- a/frontend/providers/dbprovider/src/pages/api/getEnv.ts +++ b/frontend/providers/dbprovider/src/pages/api/getEnv.ts @@ -12,6 +12,14 @@ export type SystemEnvResponse = { SHOW_DOCUMENT: boolean; }; +process.on('unhandledRejection', (reason, promise) => { + console.error(`Caught unhandledRejection:`, reason, promise); +}); + +process.on('uncaughtException', (err) => { + console.error(`Caught uncaughtException:`, err); +}); + export default async function handler(req: NextApiRequest, res: NextApiResponse) { jsonRes(res, { data: { diff --git a/frontend/providers/dbprovider/src/pages/api/guide/getBonus.ts b/frontend/providers/dbprovider/src/pages/api/guide/getBonus.ts index c52a23b5700..bc75fd355e6 100644 --- a/frontend/providers/dbprovider/src/pages/api/guide/getBonus.ts +++ b/frontend/providers/dbprovider/src/pages/api/guide/getBonus.ts @@ -32,7 +32,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< }; } = await response.json(); - const rechargeOptions = Object.entries(result.discount.firstRechargeDiscount).map( + const rechargeOptions = Object.entries(result?.discount?.firstRechargeDiscount ?? {}).map( ([amount, rate]) => ({ amount: Number(amount), gift: Math.floor((Number(amount) * Number(rate)) / 100) @@ -44,10 +44,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< data: rechargeOptions }); } catch (err: any) { - console.log(err); jsonRes(res, { code: 500, - error: err + error: '/api/guide/getBonus error' }); } } diff --git a/frontend/providers/dbprovider/src/pages/api/logs/get.ts b/frontend/providers/dbprovider/src/pages/api/logs/get.ts new file mode 100644 index 00000000000..3e557396c51 --- /dev/null +++ b/frontend/providers/dbprovider/src/pages/api/logs/get.ts @@ -0,0 +1,65 @@ +import { DBTypeEnum } from '@/constants/db'; +import { ServiceLogConfigs } from '@/constants/log'; +import { authSession } from '@/services/backend/auth'; +import { getK8s } from '@/services/backend/kubernetes'; +import { jsonRes } from '@/services/backend/response'; +import { ApiResp } from '@/services/kubernet'; +import { SupportReconfigureDBType } from '@/types/db'; +import { LogTypeEnum } from '@/constants/log'; +import { DatabaseLogService } from '@/utils/logParsers/LogParser'; +import type { NextApiRequest, NextApiResponse } from 'next'; + +export default async function handsler(req: NextApiRequest, res: NextApiResponse) { + try { + const { namespace, k8sExec, k8sCore } = await getK8s({ + kubeconfig: await authSession(req) + }); + + const { + podName, + dbType, + logType, + logPath, + page = 1, + pageSize = 100 + } = req.body as { + podName: string; + dbType: SupportReconfigureDBType; + logType: LogTypeEnum; + logPath: string; + page?: number; + pageSize?: number; + }; + + if (!podName || !dbType || !logType || !logPath) { + throw new Error('Missing required parameters: podName, dbType, logType or logPath'); + } + + const logConfig = ServiceLogConfigs[dbType][logType]; + + if (!logConfig) { + throw new Error('Invalid log type'); + } + + const logService = new DatabaseLogService(k8sExec, k8sCore, namespace); + + const result = await logService.readLogs({ + podName, + containerNames: logConfig.containerNames, + logPath, + page, + pageSize, + dbType: dbType as DBTypeEnum, + logType + }); + + console.log(result.metadata, 'result'); + + jsonRes(res, { data: result }); + } catch (err: any) { + jsonRes(res, { + code: 500, + error: err + }); + } +} diff --git a/frontend/providers/dbprovider/src/pages/api/logs/getFiles.ts b/frontend/providers/dbprovider/src/pages/api/logs/getFiles.ts new file mode 100644 index 00000000000..b480ab6ae73 --- /dev/null +++ b/frontend/providers/dbprovider/src/pages/api/logs/getFiles.ts @@ -0,0 +1,75 @@ +import { ServiceLogConfigs } from '@/constants/log'; +import { authSession } from '@/services/backend/auth'; +import { getK8s } from '@/services/backend/kubernetes'; +import { jsonRes } from '@/services/backend/response'; +import { ApiResp } from '@/services/kubernet'; +import { SupportReconfigureDBType } from '@/types/db'; +import { LogTypeEnum } from '@/constants/log'; +import { KubeFileSystem } from '@/utils/kubeFileSystem'; +import type { NextApiRequest, NextApiResponse } from 'next'; + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + const { namespace, k8sExec } = await getK8s({ + kubeconfig: await authSession(req) + }); + + const { podName, dbType, logType } = req.body as { + podName: string; + dbType: SupportReconfigureDBType; + logType: LogTypeEnum; + }; + + if (!podName || !dbType) { + throw new Error('Missing required parameters: podName, containerName or logPath'); + } + + const kubefs = new KubeFileSystem(k8sExec); + + const logConfig = ServiceLogConfigs[dbType][logType]; + + console.log('/api/logs/getFiles', { podName, dbType, logType, logConfig }); + + if (!logConfig) { + throw new Error('Invalid log type'); + } + + let files, directories; + let lastError: any; + for (const container of logConfig.containerNames) { + try { + const result = await kubefs.ls({ + namespace, + podName, + containerName: container, + path: logConfig.path, + showHidden: false + }); + files = result.files; + directories = result.directories; + break; // 成功后退出循环 + } catch (error) { + lastError = error; + console.error('/api/logs/getFiles error', error); + continue; + } + } + + if (!files) { + throw new Error(lastError?.message || 'No valid log files found in any container'); + } + + const validFiles = logConfig.filter(files); + + if (!validFiles || validFiles.length === 0) { + throw new Error('No valid log files found'); + } + + jsonRes(res, { data: validFiles }); + } catch (err: any) { + jsonRes(res, { + code: 500, + error: err + }); + } +} diff --git a/frontend/providers/dbprovider/src/pages/db/detail/components/AppBaseInfo.tsx b/frontend/providers/dbprovider/src/pages/db/detail/components/AppBaseInfo.tsx index 3dd01fb4f25..5061fd41595 100644 --- a/frontend/providers/dbprovider/src/pages/db/detail/components/AppBaseInfo.tsx +++ b/frontend/providers/dbprovider/src/pages/db/detail/components/AppBaseInfo.tsx @@ -54,6 +54,7 @@ const AppBaseInfo = ({ db = defaultDBDetail }: { db: DBDetailType }) => { ['getDBStatefulSetByName', db.dbName, db.dbType], () => getDBStatefulSetByName(db.dbName, db.dbType), { + retry: 2, enabled: !!db.dbName && !!db.dbType } ); @@ -73,6 +74,7 @@ const AppBaseInfo = ({ db = defaultDBDetail }: { db: DBDetailType }) => { enabled: supportConnectDB, retry: 3, onSuccess(data) { + console.log(data, !!data, 'service'); setIsChecked(!!data); }, onError(error) { @@ -147,7 +149,7 @@ const AppBaseInfo = ({ db = defaultDBDetail }: { db: DBDetailType }) => { [DBTypeEnum.postgresql]: `psql '${secret.connection}'`, [DBTypeEnum.mongodb]: `mongosh '${secret.connection}'`, [DBTypeEnum.mysql]: `mysql -h ${secret.host} -P ${secret.port} -u ${secret.username} -p${secret.password}`, - [DBTypeEnum.redis]: `redis-cli -h ${secret.host} -p ${secret.port}`, + [DBTypeEnum.redis]: `redis-cli -u redis://${secret.username}:${secret.password}@${secret.host}:${secret.port}`, [DBTypeEnum.kafka]: ``, [DBTypeEnum.qdrant]: ``, [DBTypeEnum.nebula]: ``, @@ -168,6 +170,7 @@ const AppBaseInfo = ({ db = defaultDBDetail }: { db: DBDetailType }) => { const openNetWorkService = async () => { try { + console.log('openNetWorkService', dbStatefulSet, db); if (!dbStatefulSet || !db) { return toast({ title: 'Missing Parameters', @@ -175,7 +178,6 @@ const AppBaseInfo = ({ db = defaultDBDetail }: { db: DBDetailType }) => { }); } const yaml = json2NetworkService({ dbDetail: db, dbStatefulSet: dbStatefulSet }); - console.log(yaml); await applyYamlList([yaml], 'create'); onClose(); setIsChecked(true); diff --git a/frontend/providers/dbprovider/src/pages/db/detail/components/ErrorLog/RunTimeLog.tsx b/frontend/providers/dbprovider/src/pages/db/detail/components/ErrorLog/RunTimeLog.tsx new file mode 100644 index 00000000000..ae191a09c69 --- /dev/null +++ b/frontend/providers/dbprovider/src/pages/db/detail/components/ErrorLog/RunTimeLog.tsx @@ -0,0 +1,295 @@ +import { getLogContent, getLogFiles } from '@/api/db'; +import { BaseTable } from '@/components/BaseTable/baseTable'; +import { SwitchPage } from '@/components/BaseTable/SwitchPage'; +import MyIcon from '@/components/Icon'; +import { useDBStore } from '@/store/db'; +import { DBDetailType, SupportReconfigureDBType } from '@/types/db'; +import { LogTypeEnum } from '@/constants/log'; +import { TFile } from '@/utils/kubeFileSystem'; +import { formatTime } from '@/utils/tools'; +import { ChevronDownIcon } from '@chakra-ui/icons'; +import { + Box, + Button, + Flex, + MenuButton, + Input, + InputGroup, + InputLeftElement +} from '@chakra-ui/react'; +import { SealosMenu } from '@sealos/ui'; +import { useQuery } from '@tanstack/react-query'; +import { + ColumnDef, + getCoreRowModel, + getFilteredRowModel, + useReactTable +} from '@tanstack/react-table'; +import { useTranslation } from 'next-i18next'; +import { useMemo, useState } from 'react'; +import { I18nCommonKey } from '@/types/i18next'; + +type LogContent = { + timestamp: string; + content: string; +}; + +const getEmptyLogResult = (page = 0, pageSize = 0) => ({ + logs: [] as LogContent[], + metadata: { + total: 0, + page, + pageSize, + processingTime: '', + hasMore: false + } +}); + +export default function RunTimeLog({ + db, + logType, + filteredSubNavList, + updateSubMenu +}: { + db: DBDetailType; + logType: LogTypeEnum; + updateSubMenu: (value: LogTypeEnum) => void; + filteredSubNavList?: { + label: string; + value: LogTypeEnum; + }[]; +}) { + const { t } = useTranslation(); + const { intervalLoadPods, dbPods } = useDBStore(); + const [podName, setPodName] = useState(''); + const [logFile, setLogFile] = useState(); + const [data, setData] = useState([]); + + const [page, setPage] = useState(1); + const [pageSize, setPageSize] = useState(100); + + const [globalFilter, setGlobalFilter] = useState(''); + + useQuery(['intervalLoadPods', db?.dbName], () => db?.dbName && intervalLoadPods(db?.dbName), { + onSuccess: () => { + !podName && setPodName(dbPods[0]?.podName); + } + }); + + const { data: logFiles = [] } = useQuery( + ['getLogFiles', podName, db?.dbType], + async () => { + if (!podName || !db?.dbType) return []; + return await getLogFiles({ + podName, + dbType: db.dbType as SupportReconfigureDBType, + logType + }); + }, + { + enabled: !!podName && db?.dbType !== 'mongodb', + onSuccess: (data) => { + !logFile && setLogFile(data[0]); + } + } + ); + + const { data: logData, isLoading } = useQuery( + ['getLogContent', logFile?.path, podName, db?.dbType, page, pageSize], + async () => { + if (!podName || !db?.dbType) return getEmptyLogResult(); + + const params = { + page, + pageSize, + podName, + dbType: db.dbType as SupportReconfigureDBType, + logType, + logPath: 'default' + } as const; + + if (db.dbType === 'mongodb') { + return await getLogContent(params); + } + + if (!logFile?.path) { + return getEmptyLogResult(); + } + + return await getLogContent({ ...params, logPath: logFile.path }); + }, + { + onSuccess(data) { + setData(data.logs); + } + } + ); + + const columns = useMemo>>( + () => [ + { + accessorKey: 'timestamp', + cell: ({ row }) => { + return ( + + {formatTime(row.original.timestamp, 'YYYY-MM-DD HH:mm:ss.SSS')} + + ); + }, + header: () => { + return ( + + {t('error_log.collection_time')} + + ); + } + }, + { + accessorKey: 'content', + header: () => { + return ( + + {t('error_log.content')} + + ); + }, + cell: ({ row }) => { + return ( + + {row.original.content} + + ); + } + } + ], + [] + ); + + const table = useReactTable({ + data, + columns, + getCoreRowModel: getCoreRowModel(), + getFilteredRowModel: getFilteredRowModel(), + state: { + globalFilter + }, + onGlobalFilterChange: setGlobalFilter, + globalFilterFn: (row, columnId, filterValue) => { + const timestamp = formatTime(row.original.timestamp, 'YYYY-MM-DD HH:mm:ss.SSS') + .toLowerCase() + .includes(filterValue.toLowerCase()); + const content = row.original.content.toLowerCase().includes(filterValue.toLowerCase()); + return timestamp || content; + } + }); + + return ( + + + {filteredSubNavList?.map((item) => ( + item.value !== logType && updateSubMenu(item.value)} + > + {t(item.label as I18nCommonKey)} + + ))} + + } + w={'200px'} + h={'32px'} + textAlign={'start'} + bg={'grayModern.100'} + borderRadius={'md'} + border={'1px solid #E8EBF0'} + > + + + {podName} + + + + + } + menuList={dbPods.map((item) => ({ + isActive: item.podName === podName, + child: {item.podName}, + onClick: () => setPodName(item.podName) + }))} + /> + + {db?.dbType !== 'mongodb' && ( + } + w={'200px'} + h={'32px'} + textAlign={'start'} + bg={'grayModern.100'} + borderRadius={'md'} + border={'1px solid #E8EBF0'} + > + + + {logFile?.name} + + + + + } + menuList={logFiles.map((item) => ({ + isActive: item.name === logFile?.name, + child: {item.name}, + onClick: () => setLogFile(item) + }))} + /> + )} + + + + + + table.setGlobalFilter(e.target.value)} + /> + + + + setPage(idx)} + /> + + ); +} diff --git a/frontend/providers/dbprovider/src/pages/db/detail/components/ErrorLog/index.tsx b/frontend/providers/dbprovider/src/pages/db/detail/components/ErrorLog/index.tsx new file mode 100644 index 00000000000..6e397d46174 --- /dev/null +++ b/frontend/providers/dbprovider/src/pages/db/detail/components/ErrorLog/index.tsx @@ -0,0 +1,90 @@ +import { LogTypeEnum } from '@/constants/log'; +import { DBDetailType, SupportReconfigureDBType } from '@/types/db'; +import { I18nCommonKey } from '@/types/i18next'; +import { Box, Flex } from '@chakra-ui/react'; +import { useTranslation } from 'next-i18next'; +import { useRouter } from 'next/router'; +import React, { ForwardedRef, forwardRef, useEffect, useMemo, useState } from 'react'; +import RunTimeLog from './RunTimeLog'; + +export type ComponentRef = { + openBackup: () => void; +}; + +const DB_LOG_TYPES: Record = { + postgresql: [LogTypeEnum.RuntimeLog], + mongodb: [LogTypeEnum.RuntimeLog], + 'apecloud-mysql': [LogTypeEnum.ErrorLog, LogTypeEnum.SlowQuery], + redis: [LogTypeEnum.RuntimeLog] +}; + +const ErrorLog = ({ db }: { db?: DBDetailType }, ref: ForwardedRef) => { + if (!db) return <>; + + const { t } = useTranslation(); + + const router = useRouter(); + const [subMenu, setSubMenu] = useState(LogTypeEnum.RuntimeLog); + + const parsedSubMenu = useMemo(() => { + const parseSubMenu = (subMenu: string): LogTypeEnum => { + if (Object.values(LogTypeEnum).includes(subMenu as LogTypeEnum)) { + return subMenu as LogTypeEnum; + } + + const dbType = db?.dbType as SupportReconfigureDBType; + const availableMenus = DB_LOG_TYPES[dbType] || []; + + if (availableMenus.includes(LogTypeEnum.ErrorLog)) { + return LogTypeEnum.ErrorLog; + } + + return LogTypeEnum.RuntimeLog; + }; + + return parseSubMenu(router.query.subMenu as string); + }, [router.query.subMenu, db?.dbType]); + + useEffect(() => { + setSubMenu(parsedSubMenu); + }, [parsedSubMenu]); + + const updateSubMenu = (newSubMenu: LogTypeEnum) => { + setSubMenu(newSubMenu); + router.push({ + query: { ...router.query, subMenu: newSubMenu } + }); + }; + + const { filteredSubNavList } = useMemo(() => { + const SubNavList = [ + { label: t('error_log.runtime_log'), value: LogTypeEnum.RuntimeLog }, + { label: t('error_log.error_log'), value: LogTypeEnum.ErrorLog }, + { label: t('error_log.slow_query'), value: LogTypeEnum.SlowQuery } + ]; + + const availableSubMenus = DB_LOG_TYPES[db.dbType as SupportReconfigureDBType] || []; + const filteredSubNavList = SubNavList.filter((item) => availableSubMenus.includes(item.value)); + + return { + availableSubMenus, + filteredSubNavList + }; + }, [t, db.dbType]); + + return ( + + {db && ( + + )} + + ); +}; + +export default React.memo(forwardRef(ErrorLog)); diff --git a/frontend/providers/dbprovider/src/pages/db/detail/index.tsx b/frontend/providers/dbprovider/src/pages/db/detail/index.tsx index d04336682ea..ff545e2f854 100644 --- a/frontend/providers/dbprovider/src/pages/db/detail/index.tsx +++ b/frontend/providers/dbprovider/src/pages/db/detail/index.tsx @@ -20,6 +20,8 @@ import Pods from './components/Pods'; import { I18nCommonKey } from '@/types/i18next'; import ReconfigureTable from './components/Reconfigure/index'; import useDetailDriver from '@/hooks/useDetailDriver'; +import ErrorLog from '@/pages/db/detail/components/ErrorLog'; +import MyIcon from '@/components/Icon'; enum TabEnum { pod = 'pod', @@ -27,7 +29,8 @@ enum TabEnum { monitor = 'monitor', InternetMigration = 'InternetMigration', DumpImport = 'DumpImport', - Reconfigure = 'reconfigure' + Reconfigure = 'reconfigure', + ErrorLog = 'errorLog' } const AppDetail = ({ @@ -54,13 +57,60 @@ const AppDetail = ({ SystemEnv.BACKUP_ENABLED; const listNavValue = [ - { label: 'monitor_list', value: TabEnum.monitor }, - { label: 'replicas_list', value: TabEnum.pod }, - ...(PublicNetMigration ? [{ label: 'dbconfig.parameter', value: TabEnum.Reconfigure }] : []), - ...(BackupSupported ? [{ label: 'backup_list', value: TabEnum.backup }] : []), - ...(PublicNetMigration ? [{ label: 'online_import', value: TabEnum.InternetMigration }] : []), + { + label: 'monitor_list', + value: TabEnum.monitor, + icon: + }, + { + label: 'replicas_list', + value: TabEnum.pod, + icon: + }, + ...(PublicNetMigration + ? [ + { + label: 'dbconfig.parameter', + value: TabEnum.Reconfigure, + icon: + } + ] + : []), + ...(BackupSupported + ? [ + { + label: 'backup_list', + value: TabEnum.backup, + icon: + } + ] + : []), + ...(PublicNetMigration + ? [ + { + label: 'online_import', + value: TabEnum.InternetMigration, + icon: + } + ] + : []), ...(PublicNetMigration && !!SystemEnv.minio_url - ? [{ label: 'import_through_file', value: TabEnum.DumpImport }] + ? [ + { + label: 'import_through_file', + value: TabEnum.DumpImport, + icon: + } + ] + : []), + ...(BackupSupported + ? [ + { + label: 'error_log.analysis', + value: TabEnum.ErrorLog, + icon: + } + ] : []) ]; @@ -80,7 +130,7 @@ const AppDetail = ({ const { dbDetail, loadDBDetail, dbPods } = useDBStore(); const [showSlider, setShowSlider] = useState(false); - useQuery([dbName, 'loadDBDetail', 'intervalLoadPods'], () => loadDBDetail(dbName), { + useQuery(['loadDBDetail', 'intervalLoadPods', dbName], () => loadDBDetail(dbName), { refetchInterval: 3000, onError(err) { router.replace('/dbs'); @@ -128,9 +178,11 @@ const AppDetail = ({ border={theme.borders.base} borderRadius={'lg'} > - + {listNav.map((item) => ( - + {item.icon} + {t(item.label as I18nCommonKey)} - + ))} {listType === TabEnum.pod && {dbPods.length} Items} @@ -196,6 +250,7 @@ const AppDetail = ({ {listType === TabEnum.Reconfigure && ( )} + {listType === TabEnum.ErrorLog && } diff --git a/frontend/providers/dbprovider/src/services/backend/kubernetes.ts b/frontend/providers/dbprovider/src/services/backend/kubernetes.ts index f7d54fbe142..9d055e34d56 100644 --- a/frontend/providers/dbprovider/src/services/backend/kubernetes.ts +++ b/frontend/providers/dbprovider/src/services/backend/kubernetes.ts @@ -331,6 +331,7 @@ export async function getK8s({ kubeconfig }: { kubeconfig: string }) { applyYamlList, delYamlList, getUserQuota: () => getUserQuota(kc, namespace), - getUserBalance: () => getUserBalance(kc) + getUserBalance: () => getUserBalance(kc), + k8sExec: new k8s.Exec(kc) }); } diff --git a/frontend/providers/dbprovider/src/types/log.ts b/frontend/providers/dbprovider/src/types/log.ts new file mode 100644 index 00000000000..d2bb696fcd2 --- /dev/null +++ b/frontend/providers/dbprovider/src/types/log.ts @@ -0,0 +1,49 @@ +import { DBTypeEnum } from '@/constants/db'; +import { LogTypeEnum } from '@/constants/log'; + +export interface BaseLogEntry { + timestamp: string; + level: string; + content: string; +} + +export interface PgLogEntry extends BaseLogEntry {} + +export interface MysqlLogEntry extends BaseLogEntry {} + +export interface RedisLogEntry extends BaseLogEntry { + processId: string; + role: string; +} + +export interface MongoLogEntry extends BaseLogEntry { + component: string; + context: string; + connectionId?: string; +} + +export interface LogResult { + logs: BaseLogEntry[]; + metadata: { + total: number; + page: number; + pageSize: number; + processingTime: string; + hasMore: boolean; + }; +} + +export type LogParserParams = { + podName: string; + containerNames: string[]; + logPath: string; + page: number; + pageSize: number; + dbType: DBTypeEnum; + keyword?: string; + logType?: LogTypeEnum; +}; + +export interface ILogParser { + readLogs(params: LogParserParams): Promise; +} diff --git a/frontend/providers/dbprovider/src/utils/kubeFileSystem.ts b/frontend/providers/dbprovider/src/utils/kubeFileSystem.ts new file mode 100644 index 00000000000..fa4fddd3213 --- /dev/null +++ b/frontend/providers/dbprovider/src/utils/kubeFileSystem.ts @@ -0,0 +1,424 @@ +import { PassThrough, Readable, Writable } from 'stream'; +import * as k8s from '@kubernetes/client-node'; +import dayjs from 'dayjs'; +import utc from 'dayjs/plugin/utc'; +import timezone from 'dayjs/plugin/timezone'; + +dayjs.extend(utc); +dayjs.extend(timezone); + +export type TFile = { + name: string; + path: string; + dir: string; + kind: string; + attr: string; + hardLinks: number; + owner: string; + group: string; + size: number; + updateTime: Date; + linkTo?: string; + processed?: boolean; +}; + +export class KubeFileSystem { + private readonly k8sExec: k8s.Exec; + + constructor(k8sExec: k8s.Exec) { + this.k8sExec = k8sExec; + } + + async execCommand( + namespace: string, + podName: string, + containerName: string, + command: string[], + stdin: Readable | null = null, + stdout: Writable | null = null + ): Promise { + const stderr = new PassThrough(); + let chunks = Buffer.alloc(0); + + if (!stdout) { + stdout = new PassThrough(); + stdout.on('data', (chunk) => { + chunks = Buffer.concat([chunks, chunk]); + }); + } + + const free = () => { + stderr.removeAllListeners(); + stdout?.removeAllListeners(); + stdin?.removeAllListeners(); + }; + + try { + const ws = await this.k8sExec.exec( + namespace, + podName, + containerName, + command, + stdout, + stderr, + stdin, + false + ); + + return await new Promise((resolve, reject) => { + // Add WebSocket error handling + ws?.on('error', (error: any) => { + free(); + const errorMessage = error?.message || error?.toString() || 'Unknown error'; + reject(new Error(`WebSocket error: ${errorMessage}`)); + }); + + stderr?.on('data', (error) => { + free(); + reject(new Error(`Command execution error: ${error.toString()}`)); + }); + + stdout?.on('end', () => { + free(); + resolve(chunks.toString()); + }); + + stdout?.on('error', (error) => { + free(); + reject(new Error(`Output stream error: ${error.message}`)); + }); + + if (stdin) { + stdin.on('end', () => { + free(); + }); + } + }).catch((error) => { + // Ensure all Promise-related errors are caught + free(); + throw error; + }); + } catch (error: any) { + free(); + if (error?.type === 'error' && error?.target instanceof WebSocket) { + throw new Error(`WebSocket error: ${error.message || 'Unknown error'}`); + } + throw new Error(`Command execution failed: ${error.message || 'Unknown error'}`); + } + } + + async getPodTimezone(namespace: string, podName: string, containerName: string): Promise { + try { + const dateOutput = await this.execCommand(namespace, podName, containerName, ['date', '+%z']); + const offset = dateOutput.trim(); + if (offset) { + return this.getTimezoneFromOffset(offset); + } + } catch (error) { + console.log('Failed to get timezone offset using date command:', error); + } + + try { + const timezoneFile = await this.execCommand(namespace, podName, containerName, [ + 'cat', + '/etc/timezone' + ]); + if (timezoneFile.trim()) { + return timezoneFile.trim(); + } + } catch (error) {} + + try { + const localtimeLink = await this.execCommand(namespace, podName, containerName, [ + 'readlink', + '-f', + '/etc/localtime' + ]); + const match = localtimeLink.match(/zoneinfo\/(.+)$/); + if (match) { + return match[1]; + } + } catch (error) {} + + return 'Etc/UTC'; + } + + async ls({ + namespace, + podName, + containerName, + path, + showHidden = false + }: { + namespace: string; + podName: string; + containerName: string; + path: string; + showHidden: boolean; + }) { + let output: string; + let isCompatibleMode = false; + try { + output = await this.execCommand(namespace, podName, containerName, [ + 'ls', + showHidden ? '-laQ' : '-lQ', + '--color=never', + '--full-time', + path + ]); + } catch (error) { + if (typeof error === 'string' && error.includes('ls: unrecognized option: full-time')) { + isCompatibleMode = true; + output = await this.execCommand(namespace, podName, containerName, [ + 'ls', + showHidden ? '-laQ' : '-lQ', + '--color=never', + '-e', + path + ]); + } else { + throw error; + } + } + const lines: string[] = output.split('\n').filter((v) => v.length > 3); + + const directories: TFile[] = []; + const files: TFile[] = []; + const symlinks: TFile[] = []; + + const podTimezone = await this.getPodTimezone(namespace, podName, containerName); + + lines.forEach((line) => { + const parts = line.split('"'); + const name = parts[1]; + + if (!name || name === '.' || name === '..') return; + + const attrs = parts[0].split(' ').filter((v) => !!v); + + const file: TFile = { + name: name, + path: (name.startsWith('/') ? '' : path + '/') + name, + dir: path, + kind: line[0], + attr: attrs[0], + hardLinks: parseInt(attrs[1]), + owner: attrs[2], + group: attrs[3], + size: parseInt(attrs[4]), + updateTime: this.convertToUTC(attrs.slice(5, 7).join(' '), podTimezone) + }; + + if (isCompatibleMode) { + file.updateTime = this.convertToUTC(attrs.slice(5, 10).join(' '), podTimezone); + } + + if (file.kind === 'c') { + if (isCompatibleMode) { + file.updateTime = this.convertToUTC(attrs.slice(7, 11).join(' '), podTimezone); + } else { + file.updateTime = this.convertToUTC(attrs.slice(6, 8).join(' '), podTimezone); + } + file.size = parseInt(attrs[5]); + } + if (file.kind === 'l') { + file.linkTo = parts[3]; + symlinks.push(file); + } + if (file.kind === 'd') { + directories.push(file); + } else { + if (file.kind !== 't' && file.kind !== '') { + files.push(file); + } + } + }); + + if (symlinks.length > 0) { + const command = ['ls', '-ldQ', '--color=never']; + + try { + symlinks.forEach((symlink) => { + let linkTo = symlink.linkTo!; + if (linkTo[0] !== '/') { + linkTo = (symlink.dir === '/' ? '' : symlink.dir) + '/' + linkTo; + } + symlink.linkTo = linkTo; + command.push(linkTo); + }); + } catch (error) { + } finally { + const output = await this.execCommand(namespace, podName, containerName, command); + const lines = output.split('\n').filter((v) => !!v); + + try { + for (const line of lines) { + if (line && line.includes('command terminated with non-zero exit code')) { + const parts = line.split('"'); + try { + symlinks.map((symlink) => { + if (symlink.linkTo === parts[1] && !symlink.processed) { + symlink.processed = true; + symlink.kind = line[0]; + if (symlink.kind === 'd') { + directories.push(symlink); + } + } + }); + } catch (error) {} + } + } + } catch (error) {} + } + } + + directories.sort((a, b) => (a.name > b.name ? 1 : -1)); + + return { + directories: directories, + files: files + }; + } + + async mv({ + namespace, + podName, + containerName, + from, + to + }: { + namespace: string; + podName: string; + containerName: string; + from: string; + to: string; + }) { + return await this.execCommand(namespace, podName, containerName, ['mv', from, to]); + } + + async rm({ + namespace, + podName, + containerName, + path + }: { + namespace: string; + podName: string; + containerName: string; + path: string; + }) { + return await this.execCommand(namespace, podName, containerName, ['rm', '-rf', path]); + } + + async download({ + namespace, + podName, + containerName, + path, + stdout + }: { + namespace: string; + podName: string; + containerName: string; + path: string; + stdout: Writable; + }) { + return await this.execCommand( + namespace, + podName, + containerName, + ['dd', `if=${path}`, 'status=none'], + null, + stdout + ); + } + + async mkdir({ + namespace, + podName, + containerName, + path + }: { + namespace: string; + podName: string; + containerName: string; + path: string; + }) { + return await this.execCommand(namespace, podName, containerName, ['mkdir', path]); + } + + async touch({ + namespace, + podName, + containerName, + path + }: { + namespace: string; + podName: string; + containerName: string; + path: string; + }) { + return await this.execCommand(namespace, podName, containerName, ['touch', path]); + } + + async upload({ + namespace, + podName, + containerName, + path, + file + }: { + namespace: string; + podName: string; + containerName: string; + path: string; + file: PassThrough; + }): Promise { + const result = await this.execCommand( + namespace, + podName, + containerName, + ['sh', '-c', `dd of=${path} status=none bs=32767`], + file + ); + return result; + } + + async md5sum({ + namespace, + podName, + containerName, + path + }: { + namespace: string; + podName: string; + containerName: string; + path: string; + }) { + return await this.execCommand(namespace, podName, containerName, ['md5sum', path]); + } + + getTimezoneFromOffset(offset: string): string { + const hours = parseInt(offset.slice(1, 3)); + const sign = offset.startsWith('-') ? '+' : '-'; + return `Etc/GMT${sign}${hours}`; + } + + convertToUTC(dateString: string, timezone: string): Date { + dateString = dateString.trim(); + timezone = timezone.trim(); + + if (timezone === 'Etc/UTC') { + timezone = 'UTC'; + } + + const dt = dayjs.tz(dateString, timezone).utc(); + + if (!dt.isValid()) { + console.error(`Failed to parse date: "${dateString}" with timezone "${timezone}"`); + return new Date(); + } + + return dt.toDate(); + } +} diff --git a/frontend/providers/dbprovider/src/utils/logParsers/LogParser.ts b/frontend/providers/dbprovider/src/utils/logParsers/LogParser.ts new file mode 100644 index 00000000000..859fb1f7309 --- /dev/null +++ b/frontend/providers/dbprovider/src/utils/logParsers/LogParser.ts @@ -0,0 +1,58 @@ +import { DBTypeEnum } from '@/constants/db'; +import { LogTypeEnum } from '@/constants/log'; +import { + BaseLogEntry, + ILogParser, + LogResult, + MongoLogEntry, + PgLogEntry, + RedisLogEntry +} from '@/types/log'; +import * as k8s from '@kubernetes/client-node'; +import { MongoLogParser } from './MongoLogParser'; +import { PostgresLogParser } from './PostgresLogParser'; +import { RedisLogParser } from './RedisLogParser'; +import { MysqlLogParser } from './MysqlLogParser'; + +class DatabaseLogService { + private parsers: Map; + + constructor( + private k8sExec: k8s.Exec, + private k8sCore: k8s.CoreV1Api, + private namespace: string + ) { + this.parsers = new Map([ + [DBTypeEnum.postgresql, new PostgresLogParser(k8sExec, namespace)], + [DBTypeEnum.redis, new RedisLogParser(k8sExec, namespace)], + [DBTypeEnum.mongodb, new MongoLogParser(k8sExec, k8sCore, namespace)], + [DBTypeEnum.mysql, new MysqlLogParser(k8sExec, namespace)] + ]); + } + + async readLogs(params: { + podName: string; + containerNames: string[]; + logPath: string; + page: number; + pageSize: number; + keyword?: string; + dbType: DBTypeEnum; + logType?: LogTypeEnum; + }): Promise { + const parser = this.parsers.get(params.dbType); + if (!parser) { + throw new Error(`Unsupported database type: ${params.dbType}`); + } + return parser.readLogs(params); + } +} + +export { + DatabaseLogService, + type BaseLogEntry, + type LogResult, + type MongoLogEntry, + type PgLogEntry, + type RedisLogEntry +}; diff --git a/frontend/providers/dbprovider/src/utils/logParsers/MongoLogParser.ts b/frontend/providers/dbprovider/src/utils/logParsers/MongoLogParser.ts new file mode 100644 index 00000000000..f11546c11ff --- /dev/null +++ b/frontend/providers/dbprovider/src/utils/logParsers/MongoLogParser.ts @@ -0,0 +1,101 @@ +import * as k8s from '@kubernetes/client-node'; +import { ILogParser, LogParserParams, LogResult, MongoLogEntry } from '@/types/log'; + +export class MongoLogParser implements ILogParser { + private static readonly MONGO_LOG_PATTERN = + /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}[+-]\d{4})\s+(\w+)\s+(\w+)\s+\[(\w+)]\s+(.+?)(?:\s+(?:connection:\s*(\d+)))?$/; + + constructor( + private k8sExec: k8s.Exec, + private k8sCore: k8s.CoreV1Api, + private namespace: string + ) {} + + async readLogs(params: LogParserParams): Promise { + const { podName, containerNames, page, pageSize } = params; + const start = performance.now(); + + try { + let logs: MongoLogEntry[] = []; + let totalCount = 0; + + const oneDayInSeconds = 1 * 60 * 60; + + for (const containerName of containerNames) { + try { + const { body: logData } = await this.k8sCore.readNamespacedPodLog( + podName, + this.namespace, + containerName, + false, + undefined, + undefined, + undefined, + false, + oneDayInSeconds, + undefined, + false + ); + + if (!logData) continue; + + const allLogs = this.parseMongoLogs(logData); + totalCount = allLogs.length; + + const startIndex = (page - 1) * pageSize; + const endIndex = startIndex + pageSize; + logs = allLogs.slice(startIndex, endIndex); + + break; + } catch (error) { + continue; + } + } + + const end = performance.now(); + + return { + logs, + metadata: { + total: totalCount, + page, + pageSize, + processingTime: `${(end - start).toFixed(2)}ms`, + hasMore: page * pageSize < totalCount + } + }; + } catch (error) { + console.error('Error reading MongoDB logs:', error); + throw error; + } + } + + private parseMongoLogs(logString: string): MongoLogEntry[] { + return logString + .split('\n') + .filter((line) => line.trim()) + .map((line) => { + const match = line.match(MongoLogParser.MONGO_LOG_PATTERN); + if (!match) { + return { + timestamp: new Date().toISOString(), + level: 'INFO', + component: 'unknown', + context: 'unknown', + content: line.trim() + }; + } + + const [, timestamp, level, component, context, content, connectionId] = match; + return { + timestamp, + level, + component, + context, + content: content.trim(), + connectionId + }; + }) + .filter((entry): entry is MongoLogEntry => entry !== null); + } +} diff --git a/frontend/providers/dbprovider/src/utils/logParsers/MysqlLogParser.ts b/frontend/providers/dbprovider/src/utils/logParsers/MysqlLogParser.ts new file mode 100644 index 00000000000..47c6f50b2c8 --- /dev/null +++ b/frontend/providers/dbprovider/src/utils/logParsers/MysqlLogParser.ts @@ -0,0 +1,293 @@ +import { LogTypeEnum } from '@/constants/log'; +import { ILogParser, LogParserParams, LogResult, MysqlLogEntry } from '@/types/log'; +import { KubeFileSystem } from '@/utils/kubeFileSystem'; +import * as k8s from '@kubernetes/client-node'; + +export class MysqlLogParser implements ILogParser { + constructor(private k8sExec: k8s.Exec, private namespace: string) {} + + async readLogs(params: LogParserParams): Promise { + const { + podName, + containerNames, + logPath, + page, + pageSize, + logType = LogTypeEnum.SlowQuery + } = params; + + const start = performance.now(); + + try { + if (logType === LogTypeEnum.SlowQuery) { + const totalCount = await this.getKeywordCount(podName, containerNames, logPath, 'Time'); + if (totalCount === 0) { + return this.emptyResult(page, pageSize); + } + + const startIndex = (page - 1) * pageSize + 1; + const endIndex = Math.min(page * pageSize, totalCount); + + const { startLine, endLine } = await this.getLineNumbersForRange( + podName, + containerNames, + logPath, + 'Time', + startIndex, + endIndex + ); + + if (!startLine || !endLine) { + return this.emptyResult(page, pageSize); + } + + const data = await this.readLogsFromContainers( + podName, + containerNames, + logPath, + startLine, + endLine + ); + + const logs = this.parseSlowQueryLogs(data); + const end = performance.now(); + + return { + logs, + metadata: { + total: totalCount, + page, + pageSize, + processingTime: `${(end - start).toFixed(2)}ms`, + hasMore: endIndex < totalCount + } + }; + } else { + const totalCount = await this.getMysqlLogCount(podName, containerNames, logPath); + if (totalCount === 0) { + return this.emptyResult(page, pageSize); + } + + const startLine = (page - 1) * pageSize + 1; + const endLine = Math.min(page * pageSize, totalCount); + + const data = await this.readLogsFromContainers( + podName, + containerNames, + logPath, + startLine, + endLine + ); + + const logs = this.parseErrorLogs(data); + const end = performance.now(); + + return { + logs, + metadata: { + total: totalCount, + page, + pageSize, + processingTime: `${(end - start).toFixed(2)}ms`, + hasMore: endLine < totalCount + } + }; + } + } catch (error) { + console.error('Error reading MySQL logs:', error); + throw error; + } + } + + private async getKeywordCount( + podName: string, + containerNames: string[], + logPath: string, + keyword: string + ): Promise { + const kubefs = new KubeFileSystem(this.k8sExec); + + for (const containerName of containerNames) { + try { + const result = await kubefs.execCommand(this.namespace, podName, containerName, [ + 'grep', + '-c', + keyword, + logPath + ]); + + if (result) { + return parseInt(result.trim(), 10); + } + } catch (error) { + continue; + } + } + return 0; + } + + private async getLineNumbersForRange( + podName: string, + containerNames: string[], + logPath: string, + keyword: string, + startIndex: number, + endIndex: number + ): Promise<{ startLine?: number; endLine?: number }> { + const kubefs = new KubeFileSystem(this.k8sExec); + + for (const containerName of containerNames) { + try { + const awkCommand = ` + /${keyword}/ { + count++; + if (count == ${startIndex}) print NR; + if (count == ${endIndex}) { print NR; exit; } + } + `; + + const result = await kubefs.execCommand(this.namespace, podName, containerName, [ + 'awk', + awkCommand, + logPath + ]); + + if (result) { + const lines = result.trim().split('\n'); + return { + startLine: parseInt(lines[0], 10), + endLine: lines[1] ? parseInt(lines[1], 10) : parseInt(lines[0], 10) + }; + } + } catch (error) { + continue; + } + } + return {}; + } + + private async readLogsFromContainers( + podName: string, + containerNames: string[], + logPath: string, + startLine: number, + endLine: number + ): Promise { + const kubefs = new KubeFileSystem(this.k8sExec); + + for (const containerName of containerNames) { + try { + const data = await kubefs.execCommand(this.namespace, podName, containerName, [ + 'sed', + '-n', + `${startLine},${endLine}p`, + logPath + ]); + if (data) return data; + } catch (error) { + continue; + } + } + return ''; + } + + private async getMysqlLogCount( + podName: string, + containerNames: string[], + logPath: string + ): Promise { + const kubefs = new KubeFileSystem(this.k8sExec); + + for (const containerName of containerNames) { + try { + const result = await kubefs.execCommand(this.namespace, podName, containerName, [ + 'wc', + '-l', + logPath + ]); + + if (result) { + return parseInt(result.trim().split(' ')[0], 10); + } + } catch (error) { + continue; + } + } + return 0; + } + + private parseSlowQueryLogs(logString: string): MysqlLogEntry[] { + if (!logString.trim()) { + return []; + } + + const entries: MysqlLogEntry[] = []; + const logs = logString.split('# Time:').filter(Boolean); + + for (const log of logs) { + const lines = log.trim().split('\n'); + + const timestampMatch = lines[0].trim().match(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}/); + + if (timestampMatch) { + const timestamp = lines[0].trim(); + const content = lines.slice(1).join('\n'); + + entries.push({ + timestamp, + level: 'INFO', + content: content.trim() + }); + } + } + + return entries; + } + + private parseErrorLogs(logString: string): MysqlLogEntry[] { + return logString + .split('\n') + .filter((line) => line.trim()) + .map((line) => { + const match = line.match( + /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)\s+(\d+)\s+\[([^\]]+)\]/ + ); + if (!match) { + return { + timestamp: new Date().toISOString(), + level: 'INFO', + content: line.trim() + }; + } + + const [, timestamp, processId, level] = match; + let logLevel = 'INFO'; + if (level.includes('Warning')) logLevel = 'WARNING'; + if (level.includes('ERROR')) logLevel = 'ERROR'; + + const content = line + .substring(line.indexOf(']', line.indexOf(']', line.indexOf(']') + 1) + 1) + 1) + .trim(); + + return { + timestamp, + level: logLevel, + content + }; + }) + .filter((entry): entry is MysqlLogEntry => entry !== null); + } + + private emptyResult(page: number, pageSize: number): LogResult { + return { + logs: [], + metadata: { + total: 0, + page, + pageSize, + processingTime: '0ms', + hasMore: false + } + }; + } +} diff --git a/frontend/providers/dbprovider/src/utils/logParsers/PostgresLogParser.ts b/frontend/providers/dbprovider/src/utils/logParsers/PostgresLogParser.ts new file mode 100644 index 00000000000..0689fae0d34 --- /dev/null +++ b/frontend/providers/dbprovider/src/utils/logParsers/PostgresLogParser.ts @@ -0,0 +1,185 @@ +import * as k8s from '@kubernetes/client-node'; +import { ILogParser, LogParserParams, LogResult, PgLogEntry } from '@/types/log'; +import { KubeFileSystem } from '@/utils/kubeFileSystem'; +import Papa from 'papaparse'; + +export class PostgresLogParser implements ILogParser { + constructor(private k8sExec: k8s.Exec, private namespace: string) {} + + async readLogs(params: LogParserParams): Promise { + const { podName, containerNames, logPath, page, pageSize, keyword = 'CST' } = params; + const start = performance.now(); + + const totalCount = await this.getKeywordCount(podName, containerNames, logPath, keyword); + + if (totalCount === 0) { + return this.emptyResult(page, pageSize); + } + + const startIndex = (page - 1) * pageSize + 1; + const endIndex = Math.min(page * pageSize, totalCount); + + const { startLine, endLine } = await this.getLineNumbersForRange( + podName, + containerNames, + logPath, + keyword, + startIndex, + endIndex + ); + + if (!startLine || !endLine) { + return this.emptyResult(page, pageSize); + } + + const data = await this.readLogsFromContainers( + podName, + containerNames, + logPath, + startLine, + endLine + ); + + const logs = this.parsePostgresLogs(data); + const end = performance.now(); + + return { + logs, + metadata: { + total: totalCount, + page, + pageSize, + processingTime: `${(end - start).toFixed(2)}ms`, + hasMore: endIndex < totalCount + } + }; + } + + private async getKeywordCount( + podName: string, + containerNames: string[], + logPath: string, + keyword: string + ): Promise { + const kubefs = new KubeFileSystem(this.k8sExec); + + for (const containerName of containerNames) { + try { + const result = await kubefs.execCommand(this.namespace, podName, containerName, [ + 'grep', + '-c', + keyword, + logPath + ]); + + if (result) { + return parseInt(result.trim(), 10); + } + } catch (error) { + continue; + } + } + return 0; + } + + private async getLineNumbersForRange( + podName: string, + containerNames: string[], + logPath: string, + keyword: string, + startIndex: number, + endIndex: number + ): Promise<{ startLine?: number; endLine?: number }> { + const kubefs = new KubeFileSystem(this.k8sExec); + + for (const containerName of containerNames) { + try { + const awkCommand = ` + /${keyword}/ { + count++; + if (count == ${startIndex}) print NR; + if (count == ${endIndex}) { print NR; exit; } + } + `; + + const result = await kubefs.execCommand(this.namespace, podName, containerName, [ + 'awk', + awkCommand, + logPath + ]); + + if (result) { + const lines = result.trim().split('\n'); + return { + startLine: parseInt(lines[0], 10), + endLine: lines[1] ? parseInt(lines[1], 10) : parseInt(lines[0], 10) + }; + } + } catch (error) { + continue; + } + } + return {}; + } + + private async readLogsFromContainers( + podName: string, + containerNames: string[], + logPath: string, + startLine: number, + endLine: number + ): Promise { + const kubefs = new KubeFileSystem(this.k8sExec); + + for (const containerName of containerNames) { + try { + const data = await kubefs.execCommand(this.namespace, podName, containerName, [ + 'sed', + '-n', + `${startLine},${endLine}p`, + logPath + ]); + if (data) return data; + } catch (error) { + continue; + } + } + return ''; + } + + private parsePostgresLogs(logString: string): PgLogEntry[] { + if (!logString.trim()) { + return []; + } + + const parsed = Papa.parse(logString, { + skipEmptyLines: true, + header: false + }); + + // parsed.data.forEach((row) => { + // console.log(row, 'row'); + // }); + + return parsed.data + .filter((row) => row[0]) + .map((row) => ({ + timestamp: row[0].replace(/^"|"$/g, ''), + level: 'INFO', + content: row.slice(1).join(',') + })); + } + + private emptyResult(page: number, pageSize: number): LogResult { + return { + logs: [], + metadata: { + total: 0, + page, + pageSize, + processingTime: '0ms', + hasMore: false + } + }; + } +} diff --git a/frontend/providers/dbprovider/src/utils/logParsers/RedisLogParser.ts b/frontend/providers/dbprovider/src/utils/logParsers/RedisLogParser.ts new file mode 100644 index 00000000000..d6e39301bd6 --- /dev/null +++ b/frontend/providers/dbprovider/src/utils/logParsers/RedisLogParser.ts @@ -0,0 +1,145 @@ +import * as k8s from '@kubernetes/client-node'; +import { ILogParser, LogParserParams, LogResult, RedisLogEntry } from '@/types/log'; +import { KubeFileSystem } from '@/utils/kubeFileSystem'; +import dayjs from 'dayjs'; + +export class RedisLogParser implements ILogParser { + private static readonly REDIS_LOG_PATTERN = + /^(\d+):([A-Z])\s+(\d{1,2}\s+[A-Za-z]+\s+\d{4}\s+\d{2}:\d{2}:\d{2}\.\d{3})\s+([*#])\s+(.+)$/; + + constructor(private k8sExec: k8s.Exec, private namespace: string) {} + + async readLogs(params: LogParserParams): Promise { + const { podName, containerNames, logPath, page, pageSize } = params; + const start = performance.now(); + + try { + const totalCount = await this.getRedisLogCount(podName, containerNames, logPath); + + if (totalCount === 0) { + return this.emptyResult(page, pageSize); + } + + const startLine = (page - 1) * pageSize + 1; + const endLine = Math.min(page * pageSize, totalCount); + + const data = await this.readLogsFromContainers( + podName, + containerNames, + logPath, + startLine, + endLine + ); + + const logs = this.parseRedisLogs(data); + const end = performance.now(); + + return { + logs, + metadata: { + total: totalCount, + page, + pageSize, + processingTime: `${(end - start).toFixed(2)}ms`, + hasMore: endLine < totalCount + } + }; + } catch (error) { + console.error('Error reading Redis logs:', error); + throw error; + } + } + + private async getRedisLogCount( + podName: string, + containerNames: string[], + logPath: string + ): Promise { + const kubefs = new KubeFileSystem(this.k8sExec); + + for (const containerName of containerNames) { + try { + const result = await kubefs.execCommand(this.namespace, podName, containerName, [ + 'wc', + '-l', + logPath + ]); + + if (result) { + const count = parseInt(result.trim().split(' ')[0], 10); + return count; + } + } catch (error) { + continue; + } + } + return 0; + } + + private async readLogsFromContainers( + podName: string, + containerNames: string[], + logPath: string, + startLine: number, + endLine: number + ): Promise { + const kubefs = new KubeFileSystem(this.k8sExec); + + for (const containerName of containerNames) { + try { + const data = await kubefs.execCommand(this.namespace, podName, containerName, [ + 'sed', + '-n', + `${startLine},${endLine}p`, + logPath + ]); + if (data) return data; + } catch (error) { + continue; + } + } + return ''; + } + + private parseRedisLogs(logString: string): RedisLogEntry[] { + return logString + .split('\n') + .filter((line) => line.trim()) + .map((line) => { + // console.log(line, 'line'); + const match = line.match(RedisLogParser.REDIS_LOG_PATTERN); + if (!match) { + return { + timestamp: new Date().toISOString(), + level: 'INFO', + content: line.trim(), + processId: '', + role: '' + }; + } + + const [, processId, role, timestamp, level, content] = match; + return { + processId, + role, + timestamp: dayjs(timestamp).add(8, 'hour').format('DD MMM YYYY HH:mm:ss.SSS'), + level: level === '#' ? 'WARNING' : 'INFO', + content: content.trim() + }; + }) + .filter((entry): entry is RedisLogEntry => entry !== null); + } + + private emptyResult(page: number, pageSize: number): LogResult { + return { + logs: [], + metadata: { + total: 0, + page, + pageSize, + processingTime: '0ms', + hasMore: false + } + }; + } +}