From e4e80a762d1635fd1941feefdc54d49bc82687a6 Mon Sep 17 00:00:00 2001 From: Nick Z <2420177+nickzelei@users.noreply.github.com> Date: Tue, 11 Mar 2025 15:02:24 -0700 Subject: [PATCH 01/19] adds new pii detect radio --- .../[account]/new/job/job-form-validations.ts | 6 ++++- .../web/app/(mgmt)/[account]/new/job/page.tsx | 22 +++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/job-form-validations.ts b/frontend/apps/web/app/(mgmt)/[account]/new/job/job-form-validations.ts index cc75ebf3f7..9c12c0d5c6 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/job/job-form-validations.ts +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/job-form-validations.ts @@ -17,7 +17,11 @@ import cron from 'cron-validate'; import * as Yup from 'yup'; import { isValidConnectionPair } from '../../connections/util'; -export type NewJobType = 'data-sync' | 'generate-table' | 'ai-generate-table'; +export type NewJobType = + | 'data-sync' + | 'generate-table' + | 'ai-generate-table' + | 'pii-detection'; // Schema for a job's workflow settings export const WorkflowSettingsSchema = Yup.object({ diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/page.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/job/page.tsx index 4073b2a554..e06259af53 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/job/page.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/page.tsx @@ -52,6 +52,12 @@ export default function NewJob(props: PageProps): ReactElement { aiDataGenParams.set('sessionId', sessionToken); } + const piiDetectionParams = new URLSearchParams(searchParams); + piiDetectionParams.set('jobType', 'pii-detection'); + if (!piiDetectionParams.has('sessionId')) { + piiDetectionParams.set('sessionId', sessionToken); + } + const jobData = [ { name: 'Data Synchronization', @@ -90,6 +96,18 @@ export default function NewJob(props: PageProps): ReactElement { darkModeImage: 'https://assets.nucleuscloud.com/neosync/app/aigen-dark.svg', }, + { + name: 'PII Detection', + description: + 'Scan your database for PII and sensitive data to identify security risks.', + href: `/${account?.name}/new/job/define?${piiDetectionParams.toString()}`, + icon: , + type: 'pii-detection', + experimental: true, + lightModeimage: 'https://assets.nucleuscloud.com/neosync/app/aigen.svg', + darkModeImage: + 'https://assets.nucleuscloud.com/neosync/app/aigen-dark.svg', + }, ] as const; const [selectedJobType, setSelectedJobType] = @@ -115,7 +133,7 @@ export default function NewJob(props: PageProps): ReactElement { }> {jobData.map((jd) => ( handleJobSelection(jd.type, jd.href)} > - +
Date: Tue, 11 Mar 2025 15:13:50 -0700 Subject: [PATCH 02/19] wires up define,connect pages --- .../[account]/new/job/JobsProgressSteps.tsx | 6 +- .../(mgmt)/[account]/new/job/define/page.tsx | 5 + .../[account]/new/job/job-form-validations.ts | 7 + .../new/job/piidetect/connect/page.tsx | 204 ++++++++++++++++++ 4 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/connect/page.tsx diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/JobsProgressSteps.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/job/JobsProgressSteps.tsx index 95a3475f79..6796ef5bbf 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/job/JobsProgressSteps.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/JobsProgressSteps.tsx @@ -16,6 +16,8 @@ const DATA_SYNC_STEPS_WITH_SUBSET: JobProgressStep[] = [ const DATA_GEN_STEPS: JobProgressStep[] = ['define', 'connect', 'schema']; +const PII_DETECTION_STEPS: JobProgressStep[] = ['define', 'connect', 'schema']; + type JobProgressStep = 'define' | 'connect' | 'schema' | 'subset'; interface Props { @@ -54,7 +56,7 @@ export function getJobProgressSteps( includeSubsetting: boolean ): JobProgressStep[]; export function getJobProgressSteps( - jobtype: 'ai-generate-table' | 'generate-table' + jobtype: 'ai-generate-table' | 'generate-table' | 'pii-detection' ): JobProgressStep[]; export function getJobProgressSteps(jobtype: NewJobType): JobProgressStep[]; export function getJobProgressSteps( @@ -70,6 +72,8 @@ export function getJobProgressSteps( return DATA_GEN_STEPS; case 'ai-generate-table': return DATA_GEN_STEPS; + case 'pii-detection': + return PII_DETECTION_STEPS; default: return []; } diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/define/page.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/job/define/page.tsx index ace508bf30..34987e527f 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/job/define/page.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/define/page.tsx @@ -107,6 +107,10 @@ export default function Page(props: PageProps): ReactElement { router.push( `/${account?.name}/new/job/aigenerate/single/connect?sessionId=${sessionPrefix}` ); + } else if (newJobType === 'pii-detection') { + router.push( + `/${account?.name}/new/job/piidetect/connect?sessionId=${sessionPrefix}` + ); } else { router.push( `/${account?.name}/new/job/connect?sessionId=${sessionPrefix}` @@ -364,6 +368,7 @@ function getNewJobType(jobtype?: string): NewJobType { case 'generate-table': case 'ai-generate-table': case 'data-sync': + case 'pii-detection': return jobtype as NewJobType; default: return 'data-sync'; diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/job-form-validations.ts b/frontend/apps/web/app/(mgmt)/[account]/new/job/job-form-validations.ts index 9c12c0d5c6..d181d50d9b 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/job/job-form-validations.ts +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/job-form-validations.ts @@ -312,6 +312,13 @@ export type SingleTableAiConnectFormValues = Yup.InferType< typeof SingleTableAiConnectFormValues >; +export const PiiDetectionConnectFormValues = Yup.object().shape({ + sourceId: Yup.string().required('Connection is required').uuid(), +}); +export type PiiDetectionConnectFormValues = Yup.InferType< + typeof PiiDetectionConnectFormValues +>; + export const SingleTableAiSchemaFormValues = Yup.object({ numRows: Yup.number() .required('Must provide a number of rows to generate') diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/connect/page.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/connect/page.tsx new file mode 100644 index 0000000000..89a3c7edba --- /dev/null +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/connect/page.tsx @@ -0,0 +1,204 @@ +'use client'; +import FormPersist from '@/app/(mgmt)/FormPersist'; +import { getNewJobSessionKeys } from '@/app/(mgmt)/[account]/jobs/util'; +import OverviewContainer from '@/components/containers/OverviewContainer'; +import PageHeader from '@/components/headers/PageHeader'; +import { useAccount } from '@/components/providers/account-provider'; +import { PageProps } from '@/components/types'; +import { Button } from '@/components/ui/button'; +import { + Form, + FormControl, + FormField, + FormItem, + FormMessage, +} from '@/components/ui/form'; +import { + Select, + SelectContent, + SelectTrigger, + SelectValue, +} from '@/components/ui/select'; +import { Separator } from '@/components/ui/separator'; +import { Skeleton } from '@/components/ui/skeleton'; +import { getSingleOrUndefined, splitConnections } from '@/libs/utils'; +import { useQuery } from '@connectrpc/connect-query'; +import { yupResolver } from '@hookform/resolvers/yup'; +import { ConnectionService } from '@neosync/sdk'; +import { useRouter } from 'next/navigation'; +import { usePostHog } from 'posthog-js/react'; +import { ReactElement, use, useEffect } from 'react'; +import { useForm } from 'react-hook-form'; +import { useSessionStorage } from 'usehooks-ts'; +import JobsProgressSteps, { + getJobProgressSteps, +} from '../../JobsProgressSteps'; +import ConnectionSelectContent from '../../connect/ConnectionSelectContent'; +import { PiiDetectionConnectFormValues } from '../../job-form-validations'; + +const NEW_CONNECTION_VALUE = 'new-connection'; + +export default function Page(props: PageProps): ReactElement { + const searchParams = use(props.searchParams); + const { account } = useAccount(); + const router = useRouter(); + useEffect(() => { + if (!searchParams?.sessionId) { + router.push(`/${account?.name}/new/job`); + } + }, [searchParams?.sessionId]); + const posthog = usePostHog(); + + const sessionPrefix = getSingleOrUndefined(searchParams?.sessionId) ?? ''; + const formKey = getNewJobSessionKeys(sessionPrefix).aigenerate.connect; + const [defaultValues] = useSessionStorage( + formKey, + { + sourceId: '', + } + ); + + const form = useForm({ + resolver: yupResolver( + PiiDetectionConnectFormValues + ), + defaultValues, + }); + + const { isLoading: isConnectionsLoading, data: connectionsData } = useQuery( + ConnectionService.method.getConnections, + { accountId: account?.id }, + { enabled: !!account?.id } + ); + const connections = connectionsData?.connections ?? []; + + function onSubmit(_values: PiiDetectionConnectFormValues) { + router.push( + `/${account?.name}/new/job/piidetect/schema?sessionId=${sessionPrefix}` + ); + posthog.capture('New Job Flow Connect Complete', { + jobType: 'pii-detection', + }); + } + + const { mysql, postgres, mssql } = splitConnections(connections); + + return ( +
+ + + } + /> + } + containerClassName="connect-page" + > +
+ +
+ +
+
+
+
+

+ Source +

+

+ Choose a connection that will be scanned for PII. +

+
+
+
+
+ ( + + + {isConnectionsLoading ? ( + + ) : ( + + )} + + + + )} + /> +
+
+ + +
+ + +
+ + +
+ ); +} From 687dc0733624ebe40e74482f420d25d37bd40500 Mon Sep 17 00:00:00 2001 From: Nick Z <2420177+nickzelei@users.noreply.github.com> Date: Wed, 12 Mar 2025 10:02:07 -0700 Subject: [PATCH 03/19] Adds schema selection box --- .../web/app/(mgmt)/[account]/jobs/util.ts | 5 + .../[account]/new/job/job-form-validations.ts | 45 ++++ .../new/job/piidetect/connect/page.tsx | 2 +- .../new/job/piidetect/schema/FormInputs.tsx | 150 ++++++++++++ .../new/job/piidetect/schema/page.tsx | 221 ++++++++++++++++++ .../new/job/piidetect/schema/stores.ts | 40 ++++ .../components/DualListBox/DualListBox.tsx | 50 +++- 7 files changed, 502 insertions(+), 11 deletions(-) create mode 100644 frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/FormInputs.tsx create mode 100644 frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/page.tsx create mode 100644 frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/stores.ts diff --git a/frontend/apps/web/app/(mgmt)/[account]/jobs/util.ts b/frontend/apps/web/app/(mgmt)/[account]/jobs/util.ts index 5765ae618a..4a52fa870b 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/jobs/util.ts +++ b/frontend/apps/web/app/(mgmt)/[account]/jobs/util.ts @@ -1715,6 +1715,7 @@ interface NewJobSessionKeys { dataSync: { connect: string; schema: string; subset: string }; generate: { connect: string; schema: string }; aigenerate: { connect: string; schema: string }; + piidetect: { connect: string; schema: string }; } export function getNewJobSessionKeys(sessionId: string): NewJobSessionKeys { @@ -1735,6 +1736,10 @@ export function getNewJobSessionKeys(sessionId: string): NewJobSessionKeys { connect: `${sessionId}-new-job-single-table-ai-connect`, schema: `${sessionId}-new-job-single-table-ai-schema`, }, + piidetect: { + connect: `${sessionId}-new-job-pii-detect-connect`, + schema: `${sessionId}-new-job-pii-detect-schema`, + }, }; } diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/job-form-validations.ts b/frontend/apps/web/app/(mgmt)/[account]/new/job/job-form-validations.ts index d181d50d9b..3d64d3f7c6 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/job/job-form-validations.ts +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/job-form-validations.ts @@ -319,6 +319,51 @@ export type PiiDetectionConnectFormValues = Yup.InferType< typeof PiiDetectionConnectFormValues >; +const TableScanFilterModeFormValue = Yup.string() + .required() + .oneOf(['include_all', 'include', 'exclude']); +export type TableScanFilterModeFormValue = Yup.InferType< + typeof TableScanFilterModeFormValue +>; + +const TableScanFilterPatternsFormValue = Yup.object().shape({ + schemas: Yup.array().of(Yup.string().required()).required().default([]), + tables: Yup.array() + .of( + Yup.object() + .shape({ + schema: Yup.string().required(), + table: Yup.string().required(), + }) + .required() + ) + .required() + .default([]), +}); +export type TableScanFilterPatternsFormValue = Yup.InferType< + typeof TableScanFilterPatternsFormValue +>; + +const TableScanFilterFormValue = Yup.object().shape({ + mode: TableScanFilterModeFormValue, + patterns: TableScanFilterPatternsFormValue, +}); +export type TableScanFilterFormValue = Yup.InferType< + typeof TableScanFilterFormValue +>; + +export const PiiDetectionSchemaFormValues = Yup.object().shape({ + dataSampling: Yup.object().shape({ + isEnabled: Yup.boolean().required().default(true), + }), + tableScanFilter: TableScanFilterFormValue, + userPrompt: Yup.string(), +}); + +export type PiiDetectionSchemaFormValues = Yup.InferType< + typeof PiiDetectionSchemaFormValues +>; + export const SingleTableAiSchemaFormValues = Yup.object({ numRows: Yup.number() .required('Must provide a number of rows to generate') diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/connect/page.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/connect/page.tsx index 89a3c7edba..c4f9d68f4b 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/connect/page.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/connect/page.tsx @@ -50,7 +50,7 @@ export default function Page(props: PageProps): ReactElement { const posthog = usePostHog(); const sessionPrefix = getSingleOrUndefined(searchParams?.sessionId) ?? ''; - const formKey = getNewJobSessionKeys(sessionPrefix).aigenerate.connect; + const formKey = getNewJobSessionKeys(sessionPrefix).piidetect.connect; const [defaultValues] = useSessionStorage( formKey, { diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/FormInputs.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/FormInputs.tsx new file mode 100644 index 0000000000..c620cb5a16 --- /dev/null +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/FormInputs.tsx @@ -0,0 +1,150 @@ +import { ToggleGroupItem } from '@/components/ui/toggle-group'; + +import DualListBox, { Option } from '@/components/DualListBox/DualListBox'; +import FormErrorMessage from '@/components/FormErrorMessage'; +import FormHeader from '@/components/forms/FormHeader'; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from '@/components/ui/card'; +import { ToggleGroup } from '@/components/ui/toggle-group'; +import { TableIcon } from '@radix-ui/react-icons'; +import { ReactElement, useCallback, useMemo } from 'react'; +import { + TableScanFilterModeFormValue, + TableScanFilterPatternsFormValue, +} from '../../job-form-validations'; + +interface TableScanFilterModeProps { + error?: string; + value: TableScanFilterModeFormValue; + onChange(value: TableScanFilterModeFormValue): void; +} + +export function TableScanFilterMode( + props: TableScanFilterModeProps +): ReactElement { + const { error, value, onChange } = props; + + return ( +
+ + { + if (value) { + onChange(value as TableScanFilterModeFormValue); + } + }} + value={value} + > + Include All + Include + Exclude + + +
+ ); +} + +interface TableScanFilterPatternsProps { + errors?: Record; + value: TableScanFilterPatternsFormValue; + onChange(value: TableScanFilterPatternsFormValue): void; + availableSchemas: string[]; +} + +export function TableScanFilterPatterns( + props: TableScanFilterPatternsProps +): ReactElement { + const { errors, value, onChange, availableSchemas } = props; + + return ( +
+ + { + onChange({ ...value, schemas: newSchemas }); + }} + availableSchemas={availableSchemas} + /> +
+ ); +} + +interface TableScanFilterPatternSchemasProps { + error?: string; + value: string[]; + onChange(value: string[]): void; + availableSchemas: string[]; +} + +export function TableScanFilterPatternSchemas( + props: TableScanFilterPatternSchemasProps +): ReactElement { + const { error, value, onChange, availableSchemas } = props; + + const dualListBoxOpts = useMemo((): Option[] => { + return availableSchemas.map((schema) => ({ + value: schema, + })); + }, [availableSchemas]); + + const selectedSchemas = useMemo((): Set => { + return new Set(value); + }, [value]); + + const onSelectedChange = useCallback( + (value: Set) => { + onChange(Array.from(value)); + }, + [onChange] + ); + + return ( +
+ + +
+
+ +
+ Schema Selection +
+ Select schemas to scan for PII. +
+ + + +
+ +
+ ); +} diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/page.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/page.tsx new file mode 100644 index 0000000000..21c552c4d4 --- /dev/null +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/page.tsx @@ -0,0 +1,221 @@ +'use client'; +import { getNewJobSessionKeys } from '@/app/(mgmt)/[account]/jobs/util'; +import ButtonText from '@/components/ButtonText'; +import OverviewContainer from '@/components/containers/OverviewContainer'; +import PageHeader from '@/components/headers/PageHeader'; +import { useAccount } from '@/components/providers/account-provider'; +import Spinner from '@/components/Spinner'; +import { PageProps } from '@/components/types'; +import { Button } from '@/components/ui/button'; +import { getSingleOrUndefined } from '@/libs/utils'; +import { useQuery } from '@connectrpc/connect-query'; +import { ConnectionDataService } from '@neosync/sdk'; +import { useRouter } from 'next/navigation'; +import { FormEvent, ReactElement, use, useEffect, useMemo } from 'react'; +import { useSessionStorage } from 'usehooks-ts'; +import { ValidationError } from 'yup'; +import { + PiiDetectionConnectFormValues, + PiiDetectionSchemaFormValues, +} from '../../job-form-validations'; +import JobsProgressSteps, { + getJobProgressSteps, +} from '../../JobsProgressSteps'; +import { TableScanFilterMode, TableScanFilterPatterns } from './FormInputs'; +import { usePiiDetectionSchemaStore } from './stores'; + +export default function Page(props: PageProps): ReactElement { + const searchParams = use(props.searchParams); + const { account } = useAccount(); + const router = useRouter(); + // const posthog = usePostHog(); + + useEffect(() => { + if (!searchParams?.sessionId) { + router.push(`/${account?.name}/new/job`); + } + }, [searchParams?.sessionId]); + + const sessionPrefix = getSingleOrUndefined(searchParams?.sessionId) ?? ''; + const sessionKeys = getNewJobSessionKeys(sessionPrefix); + + // Used to complete the whole form + // const defineFormKey = sessionKeys.global.define; + // const [defineFormValues] = useSessionStorage( + // defineFormKey, + // { jobName: '' } + // ); + + const connectFormKey = sessionKeys.piidetect.connect; + const [connectFormValues] = useSessionStorage( + connectFormKey, + { + sourceId: '', + } + ); + + // const schemaFormKey = sessionKeys.piidetect.schema; + // const [schemaFormData] = useSessionStorage( + // schemaFormKey, + // { + // dataSampling: { + // isEnabled: true, + // }, + // tableScanFilter: { + // mode: 'include_all', + // patterns: { + // schemas: [], + // tables: [], + // }, + // }, + // userPrompt: '', + // } + // ); + + // const { data: connectionData, isLoading: isConnectionLoading } = useQuery( + // ConnectionService.method.getConnection, + // { id: connectFormValues.sourceId }, + // { enabled: !!connectFormValues.sourceId } + // ); + + const { + data: connectionSchemaDataResp, + isPending, + isFetching, + } = useQuery( + ConnectionDataService.method.getConnectionSchema, + { connectionId: connectFormValues.sourceId }, + { enabled: !!connectFormValues.sourceId } + ); + // const { data: connectionsData } = useQuery( + // ConnectionService.method.getConnections, + // { accountId: account?.id }, + // { enabled: !!account?.id } + // ); + // const connections = connectionsData?.connections ?? []; + // const connectionsRecord = connections.reduce( + // (record, conn) => { + // record[conn.id] = conn; + // return record; + // }, + // {} as Record + // ); + + // const { mutateAsync: createNewSyncJob } = useMutation( + // JobService.method.createJob + // ); + + const availableSchemas = useMemo(() => { + if (isPending || !connectionSchemaDataResp) { + return []; + } + const uniqueSchemas = new Set(); + connectionSchemaDataResp?.schemas?.forEach((schema) => { + uniqueSchemas.add(schema.schema); + }); + return Array.from(uniqueSchemas); + }, [connectionSchemaDataResp, isPending, isFetching]); + + const { + formData, + setFormData, + errors, + setErrors, + isSubmitting, + setSubmitting, + } = usePiiDetectionSchemaStore(); + + async function onSubmit(e: FormEvent) { + e.preventDefault(); + if (isSubmitting) { + return; + } + + try { + setSubmitting(true); + setErrors({}); + + const validatedData = await PiiDetectionSchemaFormValues.validate( + formData, + { + abortEarly: false, + } + ); + console.log('TODO', validatedData); + } catch (err) { + if (err instanceof ValidationError) { + const validationErrors: Record = {}; + err.inner.forEach((error) => { + if (error.path) { + validationErrors[error.path] = error.message; + } + }); + setErrors(validationErrors); + } + } finally { + setSubmitting(false); + } + } + + return ( +
+ + } + /> + } + containerClassName="connect-page" + > +
+ + setFormData({ + ...formData, + tableScanFilter: { ...formData.tableScanFilter, mode: value }, + }) + } + error={errors['tableScanFilter.mode']} + /> + + + setFormData({ + ...formData, + tableScanFilter: { + ...formData.tableScanFilter, + patterns: value, + }, + }) + } + availableSchemas={availableSchemas} + errors={errors} + /> +
+ + +
+ +
+
+ ); +} diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/stores.ts b/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/stores.ts new file mode 100644 index 0000000000..4f17555bdf --- /dev/null +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/stores.ts @@ -0,0 +1,40 @@ +import { BaseHookStore } from '@/util/zustand.stores.util'; +import { create } from 'zustand'; +import { PiiDetectionSchemaFormValues } from '../../job-form-validations'; + +function getInitialFormState(): PiiDetectionSchemaFormValues { + return { + dataSampling: { + isEnabled: true, + }, + tableScanFilter: { + mode: 'include_all', + patterns: { + schemas: [], + tables: [], + }, + }, + userPrompt: '', + }; +} + +interface PiiDetectionSchemaStore + extends BaseHookStore {} + +export const usePiiDetectionSchemaStore = create( + (set) => ({ + formData: getInitialFormState(), + errors: {}, + isSubmitting: false, + setFormData: (data) => + set((state) => ({ formData: { ...state.formData, ...data } })), + setErrors: (errors) => set({ errors }), + setSubmitting: (isSubmitting) => set({ isSubmitting }), + resetForm: () => + set({ + formData: getInitialFormState(), + errors: {}, + isSubmitting: false, + }), + }) +); diff --git a/frontend/apps/web/components/DualListBox/DualListBox.tsx b/frontend/apps/web/components/DualListBox/DualListBox.tsx index 4f973cc680..3efd7fb1b2 100644 --- a/frontend/apps/web/components/DualListBox/DualListBox.tsx +++ b/frontend/apps/web/components/DualListBox/DualListBox.tsx @@ -31,10 +31,24 @@ interface Props { selected: Set; onChange(value: Set, action: Action): void; mode?: Mode; + leftEmptyState?: EmptyStateMessage; + rightEmptyState?: EmptyStateMessage; +} + +interface EmptyStateMessage { + noOptions?: string; + noSelected?: string; } export default function DualListBox(props: Props): ReactElement { - const { options, selected, onChange, mode = 'many' } = props; + const { + options, + selected, + onChange, + mode = 'many', + leftEmptyState, + rightEmptyState, + } = props; const [leftSelected, setLeftSelected] = useState({}); const [rightSelected, setRightSelected] = useState({}); @@ -99,7 +113,12 @@ export default function DualListBox(props: Props): ReactElement {
@@ -187,7 +206,12 @@ export default function DualListBox(props: Props): ReactElement {
@@ -197,14 +221,17 @@ export default function DualListBox(props: Props): ReactElement { function getLeftBoxNoMessage( options: Option[], leftData: Row[], - _mode: Mode + _mode: Mode, + leftEmptyState?: EmptyStateMessage ): string { // this isnt super useful right now because the options are always a combination of schema+jobmappings if (options.length === 0) { - return 'Unable to load schema or found no tables'; + return ( + leftEmptyState?.noOptions ?? 'Unable to load schema or found no tables' + ); } if (leftData.length === 0) { - return 'All tables have been added!'; + return leftEmptyState?.noSelected ?? 'All tables have been added!'; } return ''; } @@ -212,16 +239,19 @@ function getLeftBoxNoMessage( function getRightBoxNoMessage( options: Option[], rightData: Row[], - mode: Mode + mode: Mode, + rightEmptyState?: EmptyStateMessage ): string { if (options.length === 0) { - return 'Unable to load schema or found no tables'; + return ( + rightEmptyState?.noOptions ?? 'Unable to load schema or found no tables' + ); } if (rightData.length === 0) { if (mode === 'many') { - return 'Add tables to get started!'; + return rightEmptyState?.noSelected ?? 'Add tables to get started!'; } else { - return 'Add a table to get started!'; + return rightEmptyState?.noSelected ?? 'Add a table to get started!'; } } return ''; From 6b415e61714e8629a1dce25ae10c53d81ab69a3b Mon Sep 17 00:00:00 2001 From: Nick Z <2420177+nickzelei@users.noreply.github.com> Date: Wed, 12 Mar 2025 10:18:28 -0700 Subject: [PATCH 04/19] Adds table selection box --- .../[account]/new/job/job-form-validations.ts | 17 ++-- .../new/job/piidetect/schema/FormInputs.tsx | 99 ++++++++++++++++++- .../new/job/piidetect/schema/page.tsx | 19 ++++ .../components/DualListBox/DualListBox.tsx | 2 +- .../web/components/DualListBox/columns.tsx | 25 +---- 5 files changed, 125 insertions(+), 37 deletions(-) diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/job-form-validations.ts b/frontend/apps/web/app/(mgmt)/[account]/new/job/job-form-validations.ts index 3d64d3f7c6..33ca0d199f 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/job/job-form-validations.ts +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/job-form-validations.ts @@ -326,17 +326,18 @@ export type TableScanFilterModeFormValue = Yup.InferType< typeof TableScanFilterModeFormValue >; +const FilterPatternTableIdentifier = Yup.object().shape({ + schema: Yup.string().required(), + table: Yup.string().required(), +}); +export type FilterPatternTableIdentifier = Yup.InferType< + typeof FilterPatternTableIdentifier +>; + const TableScanFilterPatternsFormValue = Yup.object().shape({ schemas: Yup.array().of(Yup.string().required()).required().default([]), tables: Yup.array() - .of( - Yup.object() - .shape({ - schema: Yup.string().required(), - table: Yup.string().required(), - }) - .required() - ) + .of(FilterPatternTableIdentifier.required()) .required() .default([]), }); diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/FormInputs.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/FormInputs.tsx index c620cb5a16..a6b817250b 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/FormInputs.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/FormInputs.tsx @@ -14,6 +14,7 @@ import { ToggleGroup } from '@/components/ui/toggle-group'; import { TableIcon } from '@radix-ui/react-icons'; import { ReactElement, useCallback, useMemo } from 'react'; import { + FilterPatternTableIdentifier, TableScanFilterModeFormValue, TableScanFilterPatternsFormValue, } from '../../job-form-validations'; @@ -60,28 +61,44 @@ interface TableScanFilterPatternsProps { value: TableScanFilterPatternsFormValue; onChange(value: TableScanFilterPatternsFormValue): void; availableSchemas: string[]; + availableTableIdentifiers: FilterPatternTableIdentifier[]; } export function TableScanFilterPatterns( props: TableScanFilterPatternsProps ): ReactElement { - const { errors, value, onChange, availableSchemas } = props; - + const { + errors, + value, + onChange, + availableSchemas, + availableTableIdentifiers, + } = props; return (
{ onChange({ ...value, schemas: newSchemas }); }} availableSchemas={availableSchemas} /> + { + onChange({ ...value, tables: newTables }); + }} + availableTableIdentifiers={availableTableIdentifiers} + />
); } @@ -148,3 +165,77 @@ export function TableScanFilterPatternSchemas(
); } + +interface TableScanFilterPatternTablesProps { + error?: string; + value: FilterPatternTableIdentifier[]; + onChange(value: FilterPatternTableIdentifier[]): void; + + availableTableIdentifiers: FilterPatternTableIdentifier[]; +} + +export function TableScanFilterPatternTables( + props: TableScanFilterPatternTablesProps +): ReactElement { + const { error, value, onChange, availableTableIdentifiers } = props; + + const dualListBoxOpts = useMemo((): Option[] => { + return availableTableIdentifiers.map((tableIdentifier) => ({ + value: `${tableIdentifier.schema}.${tableIdentifier.table}`, + })); + }, [availableTableIdentifiers]); + + const selectedSchemas = useMemo((): Set => { + return new Set( + value.map( + (tableIdentifier) => + `${tableIdentifier.schema}.${tableIdentifier.table}` + ) + ); + }, [value]); + + const onSelectedChange = useCallback( + (value: Set) => { + onChange( + Array.from(value).map((tableIdentifier) => { + const [schema, table] = tableIdentifier.split('.'); + return { schema, table }; + }) + ); + }, + [onChange] + ); + + return ( +
+ + +
+
+ +
+ Table Selection +
+ Select tables to scan for PII. +
+ + + +
+ +
+ ); +} diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/page.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/page.tsx index 21c552c4d4..93b75f2a36 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/page.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/page.tsx @@ -15,6 +15,7 @@ import { FormEvent, ReactElement, use, useEffect, useMemo } from 'react'; import { useSessionStorage } from 'usehooks-ts'; import { ValidationError } from 'yup'; import { + FilterPatternTableIdentifier, PiiDetectionConnectFormValues, PiiDetectionSchemaFormValues, } from '../../job-form-validations'; @@ -116,6 +117,23 @@ export default function Page(props: PageProps): ReactElement { return Array.from(uniqueSchemas); }, [connectionSchemaDataResp, isPending, isFetching]); + const availableTableIdentifiers = useMemo(() => { + if (isPending || !connectionSchemaDataResp) { + return []; + } + const uniqueTableIdentifiers = new Map< + string, + FilterPatternTableIdentifier + >(); + connectionSchemaDataResp?.schemas?.forEach((schema) => { + uniqueTableIdentifiers.set(`${schema.schema}.${schema.table}`, { + schema: schema.schema, + table: schema.table, + }); + }); + return Array.from(uniqueTableIdentifiers.values()); + }, [connectionSchemaDataResp, isPending, isFetching]); + const { formData, setFormData, @@ -197,6 +215,7 @@ export default function Page(props: PageProps): ReactElement { }) } availableSchemas={availableSchemas} + availableTableIdentifiers={availableTableIdentifiers} errors={errors} />
diff --git a/frontend/apps/web/components/DualListBox/DualListBox.tsx b/frontend/apps/web/components/DualListBox/DualListBox.tsx index 3efd7fb1b2..d2c7362e2b 100644 --- a/frontend/apps/web/components/DualListBox/DualListBox.tsx +++ b/frontend/apps/web/components/DualListBox/DualListBox.tsx @@ -23,7 +23,7 @@ import { Mode, Row, getListBoxColumns } from './columns'; export interface Option { value: string; - // label: string; + // label?: string; } export type Action = 'add' | 'remove'; interface Props { diff --git a/frontend/apps/web/components/DualListBox/columns.tsx b/frontend/apps/web/components/DualListBox/columns.tsx index 383685fd14..5834c77993 100644 --- a/frontend/apps/web/components/DualListBox/columns.tsx +++ b/frontend/apps/web/components/DualListBox/columns.tsx @@ -1,5 +1,5 @@ import { ColumnDef } from '@tanstack/react-table'; -import { HTMLProps, useEffect, useRef } from 'react'; +import IndeterminateCheckbox from '../jobs/JobMappingTable/IndeterminateCheckbox'; import ColumnHeader from './ColumnHeader'; export type Mode = 'single' | 'many'; @@ -55,26 +55,3 @@ export function getListBoxColumns(props: ListBoxColumnProps): ColumnDef[] { }, ]; } - -function IndeterminateCheckbox({ - indeterminate, - className = 'flex w-4 h-4', - ...rest -}: { indeterminate?: boolean } & HTMLProps) { - const ref = useRef(null!); - - useEffect(() => { - if (typeof indeterminate === 'boolean') { - ref.current.indeterminate = !rest.checked && indeterminate; - } - }, [ref, indeterminate, rest.checked]); - - return ( - - ); -} From 75b8128a2d1643c6f9c4ef2b2ee4141fe43840c6 Mon Sep 17 00:00:00 2001 From: Nick Z <2420177+nickzelei@users.noreply.github.com> Date: Wed, 12 Mar 2025 10:47:56 -0700 Subject: [PATCH 05/19] wires up create --- .cursor/rules/frontend.mdc | 11 +++ .../web/app/(mgmt)/[account]/jobs/util.ts | 88 ++++++++++++++++++ .../[account]/new/job/job-form-validations.ts | 10 +++ .../new/job/piidetect/schema/page.tsx | 90 ++++++++++++------- 4 files changed, 168 insertions(+), 31 deletions(-) create mode 100644 .cursor/rules/frontend.mdc diff --git a/.cursor/rules/frontend.mdc b/.cursor/rules/frontend.mdc new file mode 100644 index 0000000000..057d05d927 --- /dev/null +++ b/.cursor/rules/frontend.mdc @@ -0,0 +1,11 @@ +--- +description: Best practices for react and typescript development +globs: frontend/**/*.tsx, frontend/**/*.ts +alwaysApply: false +--- + +# useQuery and useMutation Imports + +- When importing `useQuery` and `useMutation`, always import from `@connectrpc/connect-query`, _not_ `@tanstack/react-query`. + - This is because we use the connect wrapper library that coincides with our API's types for a better user experience. + - This library uses `@tanstack/react-query` under the hood. diff --git a/frontend/apps/web/app/(mgmt)/[account]/jobs/util.ts b/frontend/apps/web/app/(mgmt)/[account]/jobs/util.ts index 4a52fa870b..213475bb82 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/jobs/util.ts +++ b/frontend/apps/web/app/(mgmt)/[account]/jobs/util.ts @@ -71,6 +71,12 @@ import { JobSourceSchema, JobSourceSqlSubetSchemas, JobSourceSqlSubetSchemasSchema, + JobTypeConfig_JobTypePiiDetect, + JobTypeConfig_JobTypePiiDetect_IncludeAllSchema, + JobTypeConfig_JobTypePiiDetect_TablePatternsSchema, + JobTypeConfig_JobTypePiiDetect_TableScanFilter, + JobTypeConfig_JobTypePiiDetect_TableScanFilterSchema, + JobTypeConfig_JobTypePiiDetectSchema, MongoDBDestinationConnectionOptionsSchema, MongoDBSourceConnectionOptionsSchema, MssqlDestinationConnectionOptionsSchema, @@ -118,9 +124,11 @@ import { ActivityOptionsFormValues, ConnectFormValues, CreateJobFormValues, + CreatePiiDetectionJobFormValues, CreateSingleTableAiGenerateJobFormValues, CreateSingleTableGenerateJobFormValues, DefineFormValues, + PiiDetectionSchemaFormValues, SingleTableAiConnectFormValues, SingleTableAiSchemaFormValues, SingleTableConnectFormValues, @@ -128,6 +136,7 @@ import { SingleTableEditSourceFormValues, SingleTableSchemaFormValues, SubsetFormValues, + TableScanFilterFormValue, WorkflowSettingsSchema, } from '../new/job/job-form-validations'; import { getConnectionIdFromSource } from './[id]/source/components/util'; @@ -380,6 +389,85 @@ export function getCreateNewSyncJobRequest( }); } +export function getCreateNewPiiDetectJobRequest( + values: CreatePiiDetectionJobFormValues, + accountId: string, + getConnectionById: GetConnectionById +): CreateJobRequest { + return create(CreateJobRequestSchema, { + accountId, + jobName: values.define.jobName, + cronSchedule: values.define.cronSchedule, + initiateJobRun: values.define.initiateJobRun, + mappings: [], + source: toJobSource( + { + connect: { + sourceId: values.connect.sourceId, + destinations: [], + sourceOptions: {}, + }, + subset: undefined, + }, + getConnectionById + ), + destinations: [], + syncOptions: toSyncOptions(values), + workflowOptions: toWorkflowOptions(values.define.workflowSettings), + jobType: { + jobType: { + case: 'piiDetect', + value: toPiiDetectJobTypeConfig(values.schema), + }, + }, + }); +} + +function toPiiDetectJobTypeConfig( + values: PiiDetectionSchemaFormValues +): JobTypeConfig_JobTypePiiDetect { + return create(JobTypeConfig_JobTypePiiDetectSchema, { + dataSampling: values.dataSampling, + userPrompt: values.userPrompt, + tableScanFilter: toPiiDetectTableScanFilter(values.tableScanFilter), + }); +} + +function toPiiDetectTableScanFilter( + values: TableScanFilterFormValue +): JobTypeConfig_JobTypePiiDetect_TableScanFilter | undefined { + if (values.mode === 'include_all') { + return create(JobTypeConfig_JobTypePiiDetect_TableScanFilterSchema, { + mode: { + case: 'includeAll', + value: create(JobTypeConfig_JobTypePiiDetect_IncludeAllSchema, {}), + }, + }); + } + if (values.mode === 'include') { + return create(JobTypeConfig_JobTypePiiDetect_TableScanFilterSchema, { + mode: { + case: 'include', + value: create(JobTypeConfig_JobTypePiiDetect_TablePatternsSchema, { + schemas: values.patterns.schemas, + tables: values.patterns.tables, + }), + }, + }); + } + if (values.mode === 'exclude') { + return create(JobTypeConfig_JobTypePiiDetect_TableScanFilterSchema, { + mode: { + case: 'exclude', + value: create(JobTypeConfig_JobTypePiiDetect_TablePatternsSchema, { + schemas: values.patterns.schemas, + tables: values.patterns.tables, + }), + }, + }); + } + return undefined; +} export function toWorkflowOptions( values?: WorkflowSettingsSchema ): WorkflowOptions | undefined { diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/job-form-validations.ts b/frontend/apps/web/app/(mgmt)/[account]/new/job/job-form-validations.ts index 33ca0d199f..2a772a96d1 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/job/job-form-validations.ts +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/job-form-validations.ts @@ -503,6 +503,16 @@ export type CreateSingleTableAiGenerateJobFormValues = Yup.InferType< typeof CreateSingleTableAiGenerateJobFormValues >; +export const CreatePiiDetectionJobFormValues = Yup.object() + .shape({ + define: DefineFormValues, + connect: PiiDetectionConnectFormValues, + schema: PiiDetectionSchemaFormValues, + }) + .required('PII Detection form values are required.'); +export type CreatePiiDetectionJobFormValues = Yup.InferType< + typeof CreatePiiDetectionJobFormValues +>; export interface DefineFormValuesContext { accountId: string; isJobNameAvailable: UseMutateAsyncFunction< diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/page.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/page.tsx index 93b75f2a36..aa0055499a 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/page.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/page.tsx @@ -1,5 +1,9 @@ 'use client'; -import { getNewJobSessionKeys } from '@/app/(mgmt)/[account]/jobs/util'; +import { + clearNewJobSession, + getCreateNewPiiDetectJobRequest, + getNewJobSessionKeys, +} from '@/app/(mgmt)/[account]/jobs/util'; import ButtonText from '@/components/ButtonText'; import OverviewContainer from '@/components/containers/OverviewContainer'; import PageHeader from '@/components/headers/PageHeader'; @@ -8,13 +12,21 @@ import Spinner from '@/components/Spinner'; import { PageProps } from '@/components/types'; import { Button } from '@/components/ui/button'; import { getSingleOrUndefined } from '@/libs/utils'; -import { useQuery } from '@connectrpc/connect-query'; -import { ConnectionDataService } from '@neosync/sdk'; +import { useMutation, useQuery } from '@connectrpc/connect-query'; +import { + Connection, + ConnectionDataService, + ConnectionService, + JobService, +} from '@neosync/sdk'; import { useRouter } from 'next/navigation'; +import { usePostHog } from 'posthog-js/react'; import { FormEvent, ReactElement, use, useEffect, useMemo } from 'react'; +import { toast } from 'sonner'; import { useSessionStorage } from 'usehooks-ts'; import { ValidationError } from 'yup'; import { + DefineFormValues, FilterPatternTableIdentifier, PiiDetectionConnectFormValues, PiiDetectionSchemaFormValues, @@ -29,7 +41,7 @@ export default function Page(props: PageProps): ReactElement { const searchParams = use(props.searchParams); const { account } = useAccount(); const router = useRouter(); - // const posthog = usePostHog(); + const posthog = usePostHog(); useEffect(() => { if (!searchParams?.sessionId) { @@ -41,11 +53,11 @@ export default function Page(props: PageProps): ReactElement { const sessionKeys = getNewJobSessionKeys(sessionPrefix); // Used to complete the whole form - // const defineFormKey = sessionKeys.global.define; - // const [defineFormValues] = useSessionStorage( - // defineFormKey, - // { jobName: '' } - // ); + const defineFormKey = sessionKeys.global.define; + const [defineFormValues] = useSessionStorage( + defineFormKey, + { jobName: '' } + ); const connectFormKey = sessionKeys.piidetect.connect; const [connectFormValues] = useSessionStorage( @@ -73,11 +85,19 @@ export default function Page(props: PageProps): ReactElement { // } // ); - // const { data: connectionData, isLoading: isConnectionLoading } = useQuery( - // ConnectionService.method.getConnection, - // { id: connectFormValues.sourceId }, - // { enabled: !!connectFormValues.sourceId } - // ); + const { data: connectionsData } = useQuery( + ConnectionService.method.getConnections, + { accountId: account?.id }, + { enabled: !!account?.id } + ); + const connections = connectionsData?.connections ?? []; + const connectionsRecord = connections.reduce( + (record, conn) => { + record[conn.id] = conn; + return record; + }, + {} as Record + ); const { data: connectionSchemaDataResp, @@ -88,23 +108,8 @@ export default function Page(props: PageProps): ReactElement { { connectionId: connectFormValues.sourceId }, { enabled: !!connectFormValues.sourceId } ); - // const { data: connectionsData } = useQuery( - // ConnectionService.method.getConnections, - // { accountId: account?.id }, - // { enabled: !!account?.id } - // ); - // const connections = connectionsData?.connections ?? []; - // const connectionsRecord = connections.reduce( - // (record, conn) => { - // record[conn.id] = conn; - // return record; - // }, - // {} as Record - // ); - // const { mutateAsync: createNewSyncJob } = useMutation( - // JobService.method.createJob - // ); + const { mutateAsync: createJob } = useMutation(JobService.method.createJob); const availableSchemas = useMemo(() => { if (isPending || !connectionSchemaDataResp) { @@ -159,7 +164,30 @@ export default function Page(props: PageProps): ReactElement { abortEarly: false, } ); - console.log('TODO', validatedData); + + const job = await createJob( + getCreateNewPiiDetectJobRequest( + { + define: defineFormValues, + connect: connectFormValues, + schema: validatedData, + }, + account?.id ?? '', + (id) => connectionsRecord[id] + ) + ); + posthog.capture('New Job Flow Complete', { + jobType: 'pii-detection', + }); + toast.success('Successfully created job!'); + + clearNewJobSession(window.sessionStorage, sessionPrefix); + + if (job.job?.id) { + router.push(`/${account?.name}/jobs/${job.job.id}`); + } else { + router.push(`/${account?.name}/jobs`); + } } catch (err) { if (err instanceof ValidationError) { const validationErrors: Record = {}; From bc45482736d48fa27795f5d1893b99688597f4b2 Mon Sep 17 00:00:00 2001 From: Nick Z <2420177+nickzelei@users.noreply.github.com> Date: Wed, 12 Mar 2025 11:16:46 -0700 Subject: [PATCH 06/19] updates job layout to be job type specific --- .../app/(mgmt)/[account]/jobs/[id]/layout.tsx | 181 ++++++++++++------ 1 file changed, 124 insertions(+), 57 deletions(-) diff --git a/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/layout.tsx b/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/layout.tsx index cdd232b985..ce7d942f07 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/layout.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/layout.tsx @@ -1,5 +1,4 @@ 'use client'; -import { use } from 'react'; import ButtonText from '@/components/ButtonText'; import DeleteConfirmationDialog from '@/components/DeleteConfirmationDialog'; import ResourceId from '@/components/ResourceId'; @@ -14,9 +13,16 @@ import { Button } from '@/components/ui/button'; import { useGetSystemAppConfig } from '@/libs/hooks/useGetSystemAppConfig'; import { getErrorMessage } from '@/util/util'; import { useMutation, useQuery } from '@connectrpc/connect-query'; -import { Job, JobService, JobSourceOptions, JobStatus } from '@neosync/sdk'; +import { + Job, + JobService, + JobSourceOptions, + JobStatus, + JobTypeConfig, +} from '@neosync/sdk'; import { LightningBoltIcon, TrashIcon } from '@radix-ui/react-icons'; import { useRouter } from 'next/navigation'; +import { use } from 'react'; import { toast } from 'sonner'; import JobIdSkeletonForm from './JobIdSkeletonForm'; import JobCloneButton from './components/JobCloneButton'; @@ -30,7 +36,7 @@ export default function JobIdLayout(props: LayoutProps) { const id = params?.id ?? ''; const router = useRouter(); const { account } = useAccount(); - const { data, isLoading } = useQuery( + const { data, isLoading, isPending } = useQuery( JobService.method.getJob, { id }, { enabled: !!id } @@ -50,8 +56,7 @@ export default function JobIdLayout(props: LayoutProps) { { id: { case: 'jobId', value: id } }, { enabled: !!id } ); - const { data: systemAppConfigData, isLoading: isSystemConfigLoading } = - useGetSystemAppConfig(); + const { mutateAsync: removeJob } = useMutation(JobService.method.deleteJob); const { mutateAsync: triggerJobRun } = useMutation( JobService.method.createJobRun @@ -93,7 +98,9 @@ export default function JobIdLayout(props: LayoutProps) { mutateJobStatus(); } - if (isLoading) { + const sidebarNavItems = useGetSidebarNavItems(data?.job); + + if (isLoading || isPending) { return (
@@ -111,14 +118,7 @@ export default function JobIdLayout(props: LayoutProps) { ); } - const sidebarNavItems = getSidebarNavItems( - account?.name ?? '', - data?.job, - !isSystemConfigLoading && systemAppConfigData?.isMetricsServiceEnabled, - !isSystemConfigLoading && systemAppConfigData?.isJobHooksEnabled - ); - - const badgeValue = getBadgeText(data.job.source?.options); + const badgeValue = getBadgeText(data.job.jobType, data.job.source?.options); return (
@@ -178,66 +178,133 @@ export default function JobIdLayout(props: LayoutProps) { ); } -function getBadgeText( +function getLabeledJobType( + jobTypeConfig?: JobTypeConfig, options?: JobSourceOptions -): 'Sync Job' | 'Generate Job' | 'AI Generate Job' { +): 'Sync Job' | 'Generate Job' | 'AI Generate Job' | 'PII Detect Job' { switch (options?.config.case) { case 'generate': return 'Generate Job'; case 'aiGenerate': return 'AI Generate Job'; default: - return 'Sync Job'; + switch (jobTypeConfig?.jobType.case) { + case 'sync': + return 'Sync Job'; + case 'piiDetect': + return 'PII Detect Job'; + default: + return 'Sync Job'; + } } } +const getBadgeText = getLabeledJobType; + interface SidebarNav { title: string; href: string; } -function getSidebarNavItems( - accountName: string, - job?: Job, - isMetricsServiceEnabled?: boolean, - isJobHooksEnabled?: boolean -): SidebarNav[] { - if (!job) { +function useGetSidebarNavItems(job?: Job): SidebarNav[] { + const { account } = useAccount(); + const { data: systemAppConfigData, isLoading: isSystemConfigLoading } = + useGetSystemAppConfig(); + + if (!account || !job) { return [{ title: 'Overview', href: `` }]; } - const basePath = `/${accountName}/jobs/${job.id}`; - - const nav = [ - { - title: 'Overview', - href: `${basePath}`, - }, - { - title: 'Source', - href: `${basePath}/source`, - }, - { - title: 'Destinations', - href: `${basePath}/destinations`, - }, - ]; - - if (isJobSubsettable(job)) { - nav.push({ - title: 'Subsets', - href: `${basePath}/subsets`, - }); - } + const badgeText = getLabeledJobType(job.jobType, job.source?.options); + const basePath = `/${account.name}/jobs/${job.id}`; + const isMetricsServiceEnabled = + !isSystemConfigLoading && systemAppConfigData?.isMetricsServiceEnabled; + const isJobHooksEnabled = + !isSystemConfigLoading && systemAppConfigData?.isJobHooksEnabled; - if (isMetricsServiceEnabled) { - nav.push({ - title: 'Usage', - href: `${basePath}/usage`, - }); - } + switch (badgeText) { + case 'Sync Job': { + const nav = [ + { title: 'Overview', href: basePath }, + { title: 'Source', href: `${basePath}/source` }, + { title: 'Destinations', href: `${basePath}/destinations` }, + ]; + if (isJobSubsettable(job)) { + nav.push({ + title: 'Subsets', + href: `${basePath}/subsets`, + }); + } + if (isMetricsServiceEnabled) { + nav.push({ + title: 'Usage', + href: `${basePath}/usage`, + }); + } + if (isJobHooksEnabled) { + nav.push({ + title: 'Hooks', + href: `${basePath}/hooks`, + }); + } - if (isJobHooksEnabled) { - nav.push({ title: 'Hooks', href: `${basePath}/hooks` }); - } + return nav; + } + + case 'Generate Job': { + const nav = [ + { title: 'Overview', href: basePath }, + { title: 'Source', href: `${basePath}/source` }, + { title: 'Destinations', href: `${basePath}/destinations` }, + ]; + if (isMetricsServiceEnabled) { + nav.push({ + title: 'Usage', + href: `${basePath}/usage`, + }); + } + if (isJobHooksEnabled) { + nav.push({ + title: 'Hooks', + href: `${basePath}/hooks`, + }); + } - return nav; + return nav; + } + case 'AI Generate Job': { + const nav = [ + { title: 'Overview', href: basePath }, + { title: 'Source', href: `${basePath}/source` }, + { title: 'Destinations', href: `${basePath}/destinations` }, + ]; + if (isMetricsServiceEnabled) { + nav.push({ + title: 'Usage', + href: `${basePath}/usage`, + }); + } + if (isJobHooksEnabled) { + nav.push({ + title: 'Hooks', + href: `${basePath}/hooks`, + }); + } + return nav; + } + case 'PII Detect Job': { + const nav = [ + { title: 'Overview', href: basePath }, + { title: 'Source', href: `${basePath}/source` }, + ]; + if (isMetricsServiceEnabled) { + nav.push({ + title: 'Usage', + href: `${basePath}/usage`, + }); + } + return nav; + } + default: { + return [{ title: 'Overview', href: basePath }]; + } + } } From f53f8e649269cae9b9d480080b954b4d4395662f Mon Sep 17 00:00:00 2001 From: Nick Z <2420177+nickzelei@users.noreply.github.com> Date: Wed, 12 Mar 2025 11:58:40 -0700 Subject: [PATCH 07/19] Filling out remaining form --- .../components/PiiDetectConnectionCard.tsx | 170 ++++++++++++++++++ .../components/SourceConnectionCard.tsx | 6 +- .../app/(mgmt)/[account]/jobs/[id]/util.ts | 4 + .../new/job/piidetect/schema/FormInputs.tsx | 75 +++++++- .../new/job/piidetect/schema/page.tsx | 22 ++- .../new/job/piidetect/schema/stores.ts | 59 +++++- .../apps/web/components/forms/FormHeader.tsx | 7 +- 7 files changed, 336 insertions(+), 7 deletions(-) create mode 100644 frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/PiiDetectConnectionCard.tsx diff --git a/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/PiiDetectConnectionCard.tsx b/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/PiiDetectConnectionCard.tsx new file mode 100644 index 0000000000..3274950b21 --- /dev/null +++ b/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/PiiDetectConnectionCard.tsx @@ -0,0 +1,170 @@ +import { FilterPatternTableIdentifier } from '@/app/(mgmt)/[account]/new/job/job-form-validations'; +import { + DataSampling, + TableScanFilterMode, + TableScanFilterPatterns, + UserPrompt, +} from '@/app/(mgmt)/[account]/new/job/piidetect/schema/FormInputs'; +import { usePiiDetectionSchemaStore } from '@/app/(mgmt)/[account]/new/job/piidetect/schema/stores'; +import ButtonText from '@/components/ButtonText'; +import { useAccount } from '@/components/providers/account-provider'; +import Spinner from '@/components/Spinner'; +import { Button } from '@/components/ui/button'; +import { useQuery } from '@connectrpc/connect-query'; +import { + Connection, + ConnectionDataService, + ConnectionService, + JobService, +} from '@neosync/sdk'; +import { FormEvent, ReactElement, useEffect, useMemo } from 'react'; +import { getConnectionIdFromSource } from './util'; + +interface Props { + jobId: string; +} + +export default function PiiDetectConnectionCard({ + jobId, +}: Props): ReactElement { + const { account } = useAccount(); + const { + data, + refetch: mutate, + isLoading: isJobDataLoading, + } = useQuery(JobService.method.getJob, { id: jobId }, { enabled: !!jobId }); + const sourceConnectionId = getConnectionIdFromSource(data?.job?.source); + + const { data: connectionsData } = useQuery( + ConnectionService.method.getConnections, + { accountId: account?.id }, + { enabled: !!account?.id } + ); + const connections = connectionsData?.connections ?? []; + const connectionsRecord = connections.reduce( + (record, conn) => { + record[conn.id] = conn; + return record; + }, + {} as Record + ); + + const { + data: connectionSchemaDataResp, + isPending, + isFetching, + } = useQuery( + ConnectionDataService.method.getConnectionSchema, + { connectionId: sourceConnectionId }, + { enabled: !!sourceConnectionId } + ); + + const availableSchemas = useMemo(() => { + if (isPending || !connectionSchemaDataResp) { + return []; + } + const uniqueSchemas = new Set(); + connectionSchemaDataResp?.schemas?.forEach((schema) => { + uniqueSchemas.add(schema.schema); + }); + return Array.from(uniqueSchemas); + }, [connectionSchemaDataResp, isPending, isFetching]); + + const availableTableIdentifiers = useMemo(() => { + if (isPending || !connectionSchemaDataResp) { + return []; + } + const uniqueTableIdentifiers = new Map< + string, + FilterPatternTableIdentifier + >(); + connectionSchemaDataResp?.schemas?.forEach((schema) => { + uniqueTableIdentifiers.set(`${schema.schema}.${schema.table}`, { + schema: schema.schema, + table: schema.table, + }); + }); + return Array.from(uniqueTableIdentifiers.values()); + }, [connectionSchemaDataResp, isPending, isFetching]); + + const { + formData, + setFormData, + errors, + setErrors, + isSubmitting, + setSubmitting, + sourcedFromRemote, + setFromRemoteJob: setFromRemote, + } = usePiiDetectionSchemaStore(); + + useEffect(() => { + if (sourcedFromRemote || isJobDataLoading || !data?.job) { + return; + } + setFromRemote(data.job); + }, [sourcedFromRemote, isJobDataLoading, data?.job, setFromRemote]); + + async function onSubmit(e: FormEvent) { + e.preventDefault(); + if (isSubmitting) { + return; + } + + // todo + } + + return ( +
+ setFormData({ ...formData, userPrompt: value })} + error={errors['userPrompt']} + /> + setFormData({ ...formData, dataSampling: value })} + errors={errors} + /> + + setFormData({ + ...formData, + tableScanFilter: { ...formData.tableScanFilter, mode: value }, + }) + } + error={errors['tableScanFilter.mode']} + /> + + + setFormData({ + ...formData, + tableScanFilter: { + ...formData.tableScanFilter, + patterns: value, + }, + }) + } + availableSchemas={availableSchemas} + availableTableIdentifiers={availableTableIdentifiers} + errors={errors} + mode={formData.tableScanFilter.mode} + /> +
+ +
+ + ); +} diff --git a/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/SourceConnectionCard.tsx b/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/SourceConnectionCard.tsx index 73e48cd381..e8d7e5cd11 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/SourceConnectionCard.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/source/components/SourceConnectionCard.tsx @@ -2,10 +2,11 @@ import { useQuery } from '@connectrpc/connect-query'; import { JobService } from '@neosync/sdk'; import { ReactElement } from 'react'; -import { isAiDataGenJob, isDataGenJob } from '../../util'; +import { isAiDataGenJob, isDataGenJob, isPiiDetectJob } from '../../util'; import AiDataGenConnectionCard from './AiDataGenConnectionCard'; import DataGenConnectionCard from './DataGenConnectionCard'; import DataSyncConnectionCard from './DataSyncConnectionCard'; +import PiiDetectConnectionCard from './PiiDetectConnectionCard'; import SchemaPageSkeleton from './SchemaPageSkeleton'; interface Props { @@ -28,5 +29,8 @@ export default function SourceConnectionCard({ jobId }: Props): ReactElement { if (isAiDataGenJob(data?.job)) { return ; } + if (isPiiDetectJob(data?.job)) { + return ; + } return ; } diff --git a/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/util.ts b/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/util.ts index 539d26b713..44dfc608ba 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/util.ts +++ b/frontend/apps/web/app/(mgmt)/[account]/jobs/[id]/util.ts @@ -7,3 +7,7 @@ export function isDataGenJob(job?: Job): boolean { export function isAiDataGenJob(job?: Job): boolean { return job?.source?.options?.config.case === 'aiGenerate'; } + +export function isPiiDetectJob(job?: Job): boolean { + return job?.jobType?.jobType.case === 'piiDetect'; +} diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/FormInputs.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/FormInputs.tsx index a6b817250b..eca73ea4a4 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/FormInputs.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/new/job/piidetect/schema/FormInputs.tsx @@ -10,15 +10,77 @@ import { CardHeader, CardTitle, } from '@/components/ui/card'; +import { Textarea } from '@/components/ui/textarea'; import { ToggleGroup } from '@/components/ui/toggle-group'; import { TableIcon } from '@radix-ui/react-icons'; import { ReactElement, useCallback, useMemo } from 'react'; import { + DataSamplingFormValue, FilterPatternTableIdentifier, TableScanFilterModeFormValue, TableScanFilterPatternsFormValue, } from '../../job-form-validations'; +interface UserPromptProps { + error?: string; + value: string; + onChange(value: string): void; +} + +export function UserPrompt(props: UserPromptProps): ReactElement { + const { error, value, onChange } = props; + + return ( +
+ +