feat: create an add integrations wizard for the UI

Closes #1546

feat: create an integrations page

Closes #1545

feat: add integration form options

Closes #1546

fix: fix failing builds

fix: fix build issues

chore: rename integration view

fix: add method to save catalog scrapper

fix: add helm and flux tabs

fix: organize imports

feat: improve integrated page

fix: convert failed

fix: fix table not showing some columns

fix: remove duplicate back button footer

feat: add icons

fix: fix failing builds

fix: fix padding issue

fix: make improvements to add integration

fix: fix issues

feat: add edit integration UI

refactor: remove editor for topology and catalog scrapper

fix: fix issue to re-mapping of key-value map

fix: fix a few issues

fix: fix issue with delete redirects

fix: add back button to topology template options

fix: fix issue for catalog scrapper not saving

fix: fix issue with wizard

fix: fix padding issue

fix: fix undefined within yaml template

chore: refactor and improve form

fix: fix data mapping in flux and kubernetes

fix: source should default to UI

chore: remove topology name placeholder

chore: restore source and make readonly

feat: group by integration type

fix: remove duplicate namespace field

fix: restore topology, config scrappers and log backends

fix: issues related to the integrations pages

fix: add add integration to all sub menus and hide integrations page

fix: make improvements to the add integration wizard

fix: fix build

fix: fix a few minor issues

src/App.tsx

@@ -26,6 +26,8 @@ import { ErrorBoundary } from "./components/ErrorBoundary";
 import { Head } from "./components/Head/Head";
 import { LogsIcon } from "./components/Icons/LogsIcon";
 import { TopologyIcon } from "./components/Icons/TopologyIcon";
+import EditIntegrationPage from "./components/Integrations/EditIntegrationPage";
+import IntegrationsPage from "./components/Integrations/IntegrationsPage";
 import BootIntercom from "./components/Intercom/BootIntercom";
 import JobsHistorySettingsPage from "./components/JobsHistory/JobsHistorySettingsPage";
 import { SidebarLayout } from "./components/Layout";
@@ -59,7 +61,6 @@ import {
 import { ConnectionsPage } from "./pages/Settings/ConnectionsPage";
 import { EventQueueStatusPage } from "./pages/Settings/EventQueueStatus";
 import { FeatureFlagsPage } from "./pages/Settings/FeatureFlagsPage";
-import { LogBackendsPage } from "./pages/Settings/LogBackendsPage";
 import { TopologyCardPage } from "./pages/TopologyCard";
 import { UsersPage } from "./pages/UsersPage";
 import { ConfigInsightsPage } from "./pages/config/ConfigInsightsList";
@@ -211,6 +212,13 @@ const settingsNav: SettingsNavigationItems = {
       featureName: features.agents,
       resourceName: tables.database
     }
+    // {
+    //   name: "Integrations",
+    //   href: "/settings/integrations",
+    //   icon: MdOutlineIntegrationInstructions,
+    //   featureName: features["settings.integrations"],
+    //   resourceName: tables.database
+    // }
   ].sort((v1, v2) => stringSortHelper(v1.name, v2.name))
 };
 
@@ -353,14 +361,6 @@ export function IncidentManagerRoutes({ sidebar }: { sidebar: ReactNode }) {
             "read"
           )}
         />
-        <Route
-          path="log-backends"
-          element={withAccessCheck(
-            <LogBackendsPage />,
-            tables.database,
-            "read"
-          )}
-        />
 
         <Route
           path="event-queue-status"
@@ -378,6 +378,14 @@ export function IncidentManagerRoutes({ sidebar }: { sidebar: ReactNode }) {
           />
         </Route>
 
+        <Route path="integrations">
+          <Route index element={<IntegrationsPage />} />
+
+          <Route path=":type" caseSensitive>
+            <Route path=":id" element={<EditIntegrationPage />} />
+          </Route>
+        </Route>
+
         {settingsNav.submenu
           .filter((v) => (v as SchemaResourceType).table)
           .map((x) => {

src/api/query-hooks/mutations/useSettingsResourcesMutations.tsx

@@ -1,9 +1,4 @@
-import {
-  UseMutationOptions,
-  useMutation,
-  useQueryClient
-} from "@tanstack/react-query";
-import { useNavigate } from "react-router-dom";
+import { UseMutationOptions, useMutation } from "@tanstack/react-query";
 import {
   SchemaApi,
   SchemaResourceType
@@ -19,29 +14,20 @@ import {
 
 export const useSettingsDeleteResource = (
   resourceInfo: Pick<SchemaResourceType, "name" | "table" | "api">,
-  isModal = false
+  options: Omit<UseMutationOptions<void, Error, any>, "mutationFn"> = {}
 ) => {
-  const navigate = useNavigate();
-  const queryClient = useQueryClient();
-
-  return useMutation(
-    async (id: string) => {
+  return useMutation({
+    mutationFn: async (id: string) => {
       await deleteResource(resourceInfo, id);
     },
-    {
-      onSuccess: () => {
-        // force refetch of all resources
-        queryClient.refetchQueries(["settings", "all", resourceInfo]);
-        toastSuccess(`${resourceInfo.name} deleted successfully`);
-        if (!isModal) {
-          navigate(`/settings/${resourceInfo.table}`);
-        }
-      },
-      onError: (ex: any) => {
-        toastError(ex);
-      }
-    }
-  );
+    onSuccess: () => {
+      toastSuccess(`${resourceInfo.name} deleted successfully`);
+    },
+    onError: (ex: any) => {
+      toastError(ex);
+    },
+    ...options
+  });
 };
 
 export const useSettingsUpdateResource = (

src/api/schemaResources.ts

@@ -9,6 +9,7 @@ import {
 } from "../components/SchemaResourcePage/resourceTypes";
 import { AVATAR_INFO } from "../constants";
 import { CanaryCheckerDB, ConfigDB, IncidentCommander } from "./axios";
+import { resolve } from "./resolve";
 import { AgentItem } from "./types/common";
 import { ConfigItem } from "./types/configs";
 
@@ -33,6 +34,7 @@ export interface SchemaResourceI {
     avatar: string;
     name: string;
   };
+  integration_type: "scrapers" | "topologies" | "logging_backends";
 }
 
 export interface SchemaResourceWithJobStatus extends SchemaResourceI {
@@ -111,8 +113,10 @@ export const updateResource = (
   data: Record<string, any>
 ) => getBackend(api)?.patch(`/${table}?id=eq.${data?.id}`, data);
 
-export const getResource = ({ api, table }: SchemaApi, id: string) =>
-  getBackend(api)?.get<Record<string, any>[]>(`/${table}?id=eq.${id}`);
+export const getResource = (
+  { api, table }: Omit<SchemaApi, "name">,
+  id: string
+) => getBackend(api)?.get<SchemaResourceI[]>(`/${table}?id=eq.${id}`);
 
 export const deleteResource = ({ api, table }: SchemaApi, id: string) =>
   getBackend(api)?.patch(`/${table}?id=eq.${id}`, {
@@ -161,3 +165,25 @@ export async function getEventQueueStatus() {
   );
   return res.data ?? [];
 }
+
+export async function getIntegrationsWithJobStatus(
+  pageIndex: number,
+  pageSize: number
+) {
+  const pagingParams = `&limit=${pageSize}&offset=${pageIndex * pageSize}`;
+
+  const res = await resolve(
+    CanaryCheckerDB.get<SchemaResourceWithJobStatus[] | null>(
+      // todo: add back created_by
+      `integrations_with_status?order=created_at.desc&select=*&deleted_at=is.null${pagingParams}`
+    )
+  );
+  return res;
+}
+
+export async function getIntegrationWithJobStatus(id: string) {
+  const res = await CanaryCheckerDB.get<SchemaResourceWithJobStatus[] | null>(
+    `integrations_with_status?order=created_at.desc&select=*&deleted_at=is.null&id=eq.${id}`
+  );
+  return res.data?.[0];
+}

src/components/DataTable/Pagination/Pagination.tsx

@@ -28,7 +28,7 @@ export const Pagination = ({
   className
 }: PaginationProps) => {
   return (
-    <nav className={clsx("isolate rounded-md", className)}>
+    <nav className={clsx("isolate rounded-md px-4", className)}>
       <div className="inline-block pr-2">
         <div className="text-gray-700">
           <div className="inline-block pr-2 font-bold">

src/components/Forms/Formik/FormikConfigFormFieldsArray.tsx

@@ -50,7 +50,7 @@ export default function FormikConfigFormFieldsArray({
             {fieldValue &&
               fieldValue.length > 0 &&
               fieldValue.map((_: any, index: number) => (
-                <div className={className}>
+                <div className={className} key={index}>
                   {fields.length > 0 &&
                     fields.map(
                       ({
@@ -60,6 +60,7 @@ export default function FormikConfigFormFieldsArray({
                         className
                       }) => (
                         <Field
+                          key={fieldName}
                           name={
                             fields.length > 1
                               ? `${name}.${index}.${fieldName}`

src/components/Forms/Formik/FormikKeyValueMapField.tsx

@@ -1,8 +1,8 @@
+import { Button } from "@flanksource-ui/ui/Button";
 import { useFormikContext } from "formik";
 import { get, set } from "lodash";
-import { useEffect, useState } from "react";
+import { useCallback, useEffect, useState } from "react";
 import { FaPlus, FaTrash } from "react-icons/fa";
-import { Button } from "../../../ui/Button";
 import { TextInput } from "../../TextInput";
 
 type LocalStateValue = {
@@ -14,41 +14,65 @@ type Props = {
   name: string;
   label: string;
   hint?: string;
+  outputJson?: boolean;
 };
 
-export default function FormikKeyValueMapField({ name, label, hint }: Props) {
-  const [localValues, setLocalValue] = useState<LocalStateValue[]>([
-    { key: "", value: "" }
-  ]);
-
+export default function FormikKeyValueMapField({
+  name,
+  label,
+  hint,
+  outputJson = false
+}: Props) {
   const { values, setFieldValue } =
     useFormikContext<Record<string, Record<string, string>>>();
 
+  const [localValues, setLocalValues] = useState<LocalStateValue[]>([]);
+
   // on mount, set the local values to the formik values and set the formik
   // values to the local values on change
   useEffect(() => {
-    const localValues = Object.entries(get(values, name, {})).map(
-      ([key, value]) => ({ key, value })
-    );
-    setLocalValue(localValues);
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, []);
+    if (localValues.length === 0) {
+      const value = get(values, name, {});
+      // we need to account for the fact that the value could be a string or an
+      // object
+      const input: Record<string, string> =
+        typeof value === "string" ? JSON.parse(value) : value;
+      const localValues = Object.entries(input ?? {}).map(([key, value]) => ({
+        key,
+        value
+      }));
+      setLocalValues(localValues);
+    }
+  }, [localValues.length, name, values]);
 
-  useEffect(() => {
-    const formValue = {};
-    localValues.forEach(({ key, value }) => {
-      set(formValue, key, value);
-    });
-    setFieldValue(name, formValue);
-  }, [localValues, name, setFieldValue]);
+  const onValueChange = useCallback(
+    (value: LocalStateValue[]) => {
+      const formValue = {};
+      value.forEach(({ key, value }) => {
+        set(formValue, key, value);
+      });
+      setFieldValue(
+        name,
+        outputJson ? JSON.stringify(formValue ?? {}) : formValue
+      );
+    },
+    [name, outputJson, setFieldValue]
+  );
 
-  const handleRemove = (index: number) => {
-    setLocalValue((prev) => prev.filter((_, i) => i !== index));
-  };
+  const handleRemove = useCallback(
+    (index: number) => {
+      const newState = localValues.filter((_, i) => i !== index);
+      setLocalValues(newState);
+      onValueChange(newState);
+    },
+    [localValues, onValueChange]
+  );
 
-  const handleAdd = () => {
-    setLocalValue((prev) => [...prev, { key: "", value: "" }]);
-  };
+  const handleAdd = useCallback(() => {
+    const newState = [...localValues, { key: "", value: "" }];
+    setLocalValues(newState);
+    onValueChange(newState);
+  }, [localValues, onValueChange]);
 
   return (
     <div className="flex flex-col gap-2">
@@ -66,7 +90,8 @@ export default function FormikKeyValueMapField({ name, label, hint }: Props) {
                   }
                   return localValue;
                 });
-                setLocalValue(newLocalValues);
+                setLocalValues(newLocalValues);
+                onValueChange(newLocalValues);
               }}
               id={""}
             />
@@ -81,7 +106,8 @@ export default function FormikKeyValueMapField({ name, label, hint }: Props) {
                   }
                   return localValue;
                 });
-                setLocalValue(newLocalValues);
+                setLocalValues(newLocalValues);
+                onValueChange(newLocalValues);
               }}
               id={""}
             />

src/components/Forms/Formik/__tests__/FormikKeyValueMapField.unit.test.tsx

@@ -1,6 +1,6 @@
 import { fireEvent, render, screen, waitFor } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
-import { Formik, Form } from "formik";
+import { Form, Formik } from "formik";
 import FormikKeyValueMapField from "./../FormikKeyValueMapField";
 
 describe("FormikKeyValueMapField", () => {
@@ -123,4 +123,48 @@ describe("FormikKeyValueMapField", () => {
       );
     });
   });
+
+  // support for JSON
+  it("updates the formik values when a key or value is changed and json is true", async () => {
+    const onSubmit = jest.fn();
+    render(
+      <Formik initialValues={initialValues} onSubmit={onSubmit}>
+        <Form>
+          <FormikKeyValueMapField name="myMap" label="My Map" outputJson />
+          <button type="submit">Submit</button>
+        </Form>
+      </Formik>
+    );
+
+    await waitFor(() => {
+      expect(screen.getByDisplayValue("key1")).toBeInTheDocument();
+    });
+
+    const keyInput = screen.getByDisplayValue("key1");
+    fireEvent.change(keyInput, {
+      target: {
+        value: "newKey"
+      }
+    });
+    const valueInput = screen.getByDisplayValue("value1");
+    fireEvent.change(valueInput, {
+      target: {
+        value: "newValue"
+      }
+    });
+    const submitButton = screen.getByText("Submit");
+    userEvent.click(submitButton);
+
+    await waitFor(() => {
+      expect(onSubmit).toHaveBeenCalledWith(
+        {
+          myMap: JSON.stringify({
+            newKey: "newValue",
+            key2: "value2"
+          })
+        },
+        expect.anything()
+      );
+    });
+  });
 });