feat: add database log analysis (labring#5189)

* support db logs * Support pgsql mysql redis mongo log viewing * fix build * support search * add catch promise * update * update mysql log * update pgsql key * update mysql * update pagesize * update ci
cuisongliu · Nov 8, 2024 · d754ebe · d754ebe
1 parent 73a0093
commit d754ebe
Show file tree

Hide file tree

Showing 34 changed files with 2,328 additions and 27 deletions.
diff --git a/frontend/package.json b/frontend/package.json
@@ -16,7 +16,8 @@
     "dev-workorder": "pnpm -r --filter ./providers/workorder  run dev",
     "gen:theme-typings": "pnpm chakra-cli tokens packages/ui/src/theme/theme.ts --out node_modules/.pnpm/node_modules/@chakra-ui/styled-system/dist/theming.types.d.ts",
     "postinstall": "pnpm run gen:theme-typings",
-    "prepare": "cd .. && husky frontend/.husky"
+    "prepare": "cd .. && husky frontend/.husky",
+    "build-packages": "pnpm -r --filter ./packages/client-sdk run build"
   },
   "workspaces": [
     "./packages/*",

diff --git a/frontend/packages/ui/src/components/Menu/index.tsx b/frontend/packages/ui/src/components/Menu/index.tsx
@@ -28,6 +28,8 @@ export const SealosMenu = ({ width, Button, menuList }: Props) => {
     <Menu offset={[0, 10]} autoSelect={false} isLazy>
       {Button}
       <MenuList
+        maxH={'300px'}
+        overflowY={'auto'}
         borderRadius={'md'}
         minW={`${width}px !important`}
         p={'6px'}

diff --git a/frontend/pnpm-lock.yaml b/frontend/pnpm-lock.yaml
diff --git a/frontend/providers/dbprovider/package.json b/frontend/providers/dbprovider/package.json
@@ -11,7 +11,6 @@
     "unlink-sdk": "yalc remove --all && pnpm install sealos-desktop-sdk"
   },
   "dependencies": {
-    "@sealos/driver": "workspace:^",
     "@chakra-ui/anatomy": "^2.2.1",
     "@chakra-ui/icons": "^2.1.1",
     "@chakra-ui/next-js": "^2.1.5",
@@ -21,8 +20,11 @@
     "@emotion/styled": "^11.11.0",
     "@kubernetes/client-node": "^0.18.1",
     "@next/font": "13.1.6",
+    "@sealos/driver": "workspace:^",
     "@sealos/ui": "workspace:^",
     "@tanstack/react-query": "^4.35.3",
+    "@tanstack/react-table": "^8.10.7",
+    "@tanstack/react-virtual": "^3.10.8",
     "ansi_up": "^5.2.1",
     "axios": "^1.5.1",
     "date-fns": "^2.30.0",
@@ -43,6 +45,7 @@
     "next": "13.1.6",
     "next-i18next": "^15.3.0",
     "nprogress": "^0.2.0",
+    "papaparse": "^5.4.1",
     "prettier": "^2.8.8",
     "react": "18.2.0",
     "react-day-picker": "^8.8.2",
@@ -65,6 +68,7 @@
     "@types/multer": "^1.4.10",
     "@types/node": "18.13.0",
     "@types/nprogress": "^0.2.1",
+    "@types/papaparse": "^5.3.15",
     "@types/react": "18.2.37",
     "@types/react-dom": "18.0.10",
     "@types/react-syntax-highlighter": "^15.5.7",

diff --git a/frontend/providers/dbprovider/public/locales/en/common.json b/frontend/providers/dbprovider/public/locales/en/common.json
@@ -23,6 +23,7 @@
   "Migrating": "Migrating",
   "Monday": "Monday",
   "Option": "Optional",
+  "Page": "Page",
   "Password": "Password",
   "Pause": "Pause",
   "Paused": "Paused",
@@ -45,6 +46,7 @@
   "Success": "succeeded",
   "Sunday": "Sun",
   "Thursday": "Thu",
+  "Total": "total",
   "Tuesday": "Tue",
   "Type": "Type",
   "Unknown": "Unknown",
@@ -173,6 +175,15 @@
   "duration_of_transaction": "Transaction Duration",
   "enable_external_network_access": "Allow public network access",
   "enter_save": "Press Enter to save. 'All' exports the entire database.",
+  "error_log": {
+    "analysis": "Log Analysis",
+    "collection_time": "Collection Time",
+    "content": "Information",
+    "error_log": "Error Log",
+    "runtime_log": "Run Log",
+    "search_content": "Search Log Content",
+    "slow_query": "Slow Log"
+  },
   "event_analyze": "Intelligent Analytics",
   "event_analyze_error": "Intelligent analytics error",
   "external_address": "Public Domain",
@@ -306,4 +317,4 @@
   "within_5_minutes": "Within 5 minutes",
   "yaml_file": "YAML",
   "you_have_successfully_deployed_database": "You have successfully deployed and created a database!"
-}
+}
diff --git a/frontend/providers/dbprovider/public/locales/zh/common.json b/frontend/providers/dbprovider/public/locales/zh/common.json
@@ -23,6 +23,7 @@
   "Migrating": "正在迁移",
   "Monday": "周一",
   "Option": "选填",
+  "Page": "页",
   "Password": "密码",
   "Pause": "暂停",
   "Paused": "已暂停",
@@ -45,6 +46,7 @@
   "Success": "成功",
   "Sunday": "周日",
   "Thursday": "周四",
+  "Total": "总数",
   "Tuesday": "周二",
   "Type": "类型",
   "Unknown": "未知",
@@ -173,6 +175,15 @@
   "duration_of_transaction": "事务持续时间",
   "enable_external_network_access": "开启外网访问",
   "enter_save": "回车保存 && All 代表导出整个库",
+  "error_log": {
+    "analysis": "日志分析",
+    "collection_time": "采集时间",
+    "content": "信息",
+    "error_log": "错误日志",
+    "runtime_log": "运行日志",
+    "search_content": "搜索日志内容",
+    "slow_query": "慢日志"
+  },
   "event_analyze": "智能分析",
   "event_analyze_error": "智能分析出错了~",
   "external_address": "外网地址",

diff --git a/frontend/providers/dbprovider/src/api/db.ts b/frontend/providers/dbprovider/src/api/db.ts
@@ -1,18 +1,22 @@
-import { GET, POST, DELETE } from '@/services/request';
-import { adaptDBListItem, adaptDBDetail, adaptPod, adaptEvents } from '@/utils/adapt';
+import type { SecretResponse } from '@/pages/api/getSecretByName';
+import { DELETE, GET, POST } from '@/services/request';
+import { KbPgClusterType } from '@/types/cluster';
 import type {
   BackupItemType,
   DBEditType,
   DBType,
   OpsRequestItemType,
-  PodDetailType
+  PodDetailType,
+  SupportReconfigureDBType
 } from '@/types/db';
+import { LogTypeEnum } from '@/constants/log';
+import { MonitorChartDataResult } from '@/types/monitor';
+import { adaptDBDetail, adaptDBListItem, adaptEvents, adaptPod } from '@/utils/adapt';
 import { json2Restart } from '@/utils/json2Yaml';
-import { json2StartOrStop } from '../utils/json2Yaml';
-import type { SecretResponse } from '@/pages/api/getSecretByName';
+import { TFile } from '@/utils/kubeFileSystem';
+import { LogResult } from '@/utils/logParsers/LogParser';
 import { V1Service, V1StatefulSet } from '@kubernetes/client-node';
-import { KbPgClusterType } from '@/types/cluster';
-import { MonitorChartDataResult } from '@/types/monitor';
+import { json2StartOrStop } from '../utils/json2Yaml';
 
 export const getMyDBList = () =>
   GET<KbPgClusterType[]>('/api/getDBList').then((data) => data.map(adaptDBListItem));
@@ -111,3 +115,42 @@ export const getOpsRequest = ({
     label,
     dbType
   });
+
+export const getLogFiles = ({
+  podName,
+  dbType,
+  logType
+}: {
+  podName: string;
+  dbType: SupportReconfigureDBType;
+  logType: LogTypeEnum;
+}) =>
+  POST<TFile[]>(`/api/logs/getFiles`, {
+    podName,
+    dbType,
+    logType
+  });
+
+export const getLogContent = ({
+  logPath,
+  page,
+  pageSize,
+  dbType,
+  logType,
+  podName
+}: {
+  logPath: string;
+  page: number;
+  pageSize: number;
+  dbType: SupportReconfigureDBType;
+  logType: LogTypeEnum;
+  podName: string;
+}) =>
+  POST<LogResult>(`/api/logs/get`, {
+    logPath,
+    page,
+    pageSize,
+    dbType,
+    logType,
+    podName
+  });