[Cloud Security] Metering integration tests

x-pack/plugins/security_solution_serverless/server/cloud_security/defend_for_containers_metering.ts

@@ -76,7 +76,8 @@ export const getUsageRecords = async (
             {
               range: {
                 'event.ingested': {
-                  gt: searchFrom.toISOString(),
+                  // gt: searchFrom.toISOString()
+                  gte: `now-30m`,
                 },
               },
             },

x-pack/plugins/security_solution_serverless/server/common/services/usage_reporting_service.ts

@@ -14,17 +14,33 @@ import type { UsageRecord } from '../../types';
 
 // TODO remove once we have the CA available
 const agent = new https.Agent({ rejectUnauthorized: false });
+
 export class UsageReportingService {
   public async reportUsage(
     records: UsageRecord[],
     url = USAGE_SERVICE_USAGE_URL
   ): Promise<Response> {
-    return fetch(url, {
-      method: 'post',
-      body: JSON.stringify(records),
-      headers: { 'Content-Type': 'application/json' },
-      agent,
-    });
+    try {
+      const isHttps = url.includes('https');
+      const response = await fetch(url, {
+        method: 'post',
+        body: JSON.stringify(records),
+        headers: { 'Content-Type': 'application/json' },
+        ...(isHttps && { agent }), // Conditionally add agent if URL is HTTPS for supporting integration tests.
+      });
+
+      if (!response.ok) {
+        console.error(`Error: ${response.statusText}`);
+        throw new Error(`HTTP error! status: ${response.status}`);
+      }
+
+      console.log('Usage report completed successfully.');
+      return response;
+    } catch (error) {
+      console.error('Error during usage report:', JSON.stringify(error));
+      console.error('Error:', error.erros);
+      throw error;
+    }
   }
 }
 

x-pack/plugins/security_solution_serverless/server/config.ts

@@ -17,11 +17,17 @@ export const configSchema = schema.object({
   enabled: schema.boolean({ defaultValue: false }),
   productTypes,
   /**
-   * Usage Reporting: the interval between runs of the task
+   * Usage Reporting: the interval between runs of the endpoint task
    */
 
   usageReportingTaskInterval: schema.string({ defaultValue: '5m' }),
 
+  /**
+   * Usage Reporting: the interval between runs of the cloud security task
+   */
+
+  cloudSecurityUsageReportingTaskInterval: schema.string({ defaultValue: '30m' }),
+
   /**
    * Usage Reporting: timeout value for how long the task should run.
    */

x-pack/plugins/security_solution_serverless/server/plugin.ts

@@ -125,7 +125,7 @@ export class SecuritySolutionServerlessPlugin
     this.cloudSecurityUsageReportingTask
       ?.start({
         taskManager: pluginsSetup.taskManager,
-        interval: cloudSecurityMetringTaskProperties.interval,
+        interval: this.config.cloudSecurityUsageReportingTaskInterval,
       })
       .catch(() => {});
 

x-pack/test/api_integration/apis/cloud_security_posture/mock_data.ts

@@ -8,6 +8,87 @@ import Chance from 'chance';
 
 const chance = new Chance();
 
+export const cspmFindingsMockDataForMetering = [
+  {
+    resource: { id: chance.guid(), name: `Pod`, sub_type: 'aws-s3' },
+    rule: {
+      benchmark: {
+        posture_type: 'cspm',
+      },
+      type: 'process',
+    },
+  },
+  {
+    resource: { id: chance.guid(), name: `Pod`, sub_type: 'aws-rds' },
+    rule: {
+      benchmark: {
+        posture_type: 'cspm',
+      },
+      type: 'process',
+    },
+  },
+  {
+    resource: { id: chance.guid(), name: `Pod`, sub_type: 'not-billable-asset' },
+    rule: {
+      benchmark: {
+        posture_type: 'cspm',
+      },
+      type: 'process',
+    },
+  },
+];
+export const kspmFindingsMockDataForMetering = [
+  {
+    resource: { id: chance.guid(), name: `kubelet`, sub_type: 'node' },
+    rule: {
+      benchmark: {
+        posture_type: 'kspm',
+      },
+    },
+    agent: { id: chance.guid() },
+  },
+  {
+    resource: { id: chance.guid(), name: `kubelet`, sub_type: 'not billable resource' },
+    rule: {
+      benchmark: {
+        posture_type: 'kspm',
+      },
+    },
+    agent: { id: chance.guid() },
+  },
+];
+
+export const cnvmFindingsMockDataForMetering = [
+  {
+    cloud: {
+      instance: {
+        id: chance.guid(),
+      },
+    },
+  },
+  {
+    cloud: {
+      instance: {
+        id: chance.guid(),
+      },
+    },
+  },
+];
+
+export const defendForContainersHeartbeatsForMetering = [
+  {
+    agent: {
+      id: chance.guid(),
+    },
+    cloud_defend: {
+      block_action_enabled: true,
+    },
+    event: {
+      ingested: new Date().toISOString(),
+    },
+  },
+];
+
 export const findingsMockData = [
   {
     resource: { id: chance.guid(), name: `kubelet`, sub_type: 'lower case sub type' },

x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/index.ts

@@ -17,6 +17,7 @@ export default function ({ loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./benchmark/v2'));
     loadTestFile(require.resolve('./find_csp_benchmark_rule'));
     loadTestFile(require.resolve('./telemetry'));
+    loadTestFile(require.resolve('./serverless_metering/cloud_security_metering'));
 
     // TODO: migrate status_unprivileged tests from stateful, if it feasible in serverless with the new security model
     // loadTestFile(require.resolve('./status/status_unprivileged'));

x-pack/test_serverless/api_integration/test_suites/security/cloud_security_posture/serverless_metering/cloud_security_metering.ts

@@ -0,0 +1,206 @@
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../../../ftr_provider_context';
+
+import { RoleCredentials } from 'x-pack/test_serverless/shared/services';
+import {
+  deleteIndex,
+  addIndex,
+  createPackagePolicy,
+} from '../../../../../../test/api_integration/apis/cloud_security_posture/helper'; // eslint-disable-line @kbn/imports/no_boundary_crossing
+import {
+  cnvmFindingsMockDataForMetering,
+  cspmFindingsMockDataForMetering,
+  defendForContainersHeartbeatsForMetering,
+  kspmFindingsMockDataForMetering,
+} from '../../../../../../test/api_integration/apis/cloud_security_posture/mock_data'; // eslint-disable-line @kbn/imports/no_boundary_crossing
+import {
+  LATEST_FINDINGS_INDEX_DEFAULT_NS,
+  LATEST_VULNERABILITIES_INDEX_DEFAULT_NS,
+} from '@kbn/cloud-security-posture-plugin/common/constants';
+import { UsageRecord, getInterceptedRequestPayload, setupMockServer } from './mock_usage_server';
+
+const CLOUD_DEFEND_HEARTBEAT_INDEX_DEFAULT_NS = 'metrics-cloud_defend.heartbeat-default';
+
+export default function (providerContext: FtrProviderContext) {
+  const mockUsageApiServer = setupMockServer();
+  const { getService } = providerContext;
+  const retry = getService('retry');
+  const kibanaServer = getService('kibanaServer');
+  const esArchiver = getService('esArchiver');
+  const es = getService('es');
+  const svlCommonApi = getService('svlCommonApi');
+  const svlUserManager = getService('svlUserManager');
+  const supertestWithoutAuth = getService('supertestWithoutAuth');
+
+  /*
+  This test aims to intercept the usage API request sent by the metering background task manager.
+  The task manager is running by default in security serverless project in the background and sending usage API requests to the usage API.
+   This test mocks the usage API server and intercepts the usage API request sent by the metering background task manager.
+  */
+  describe('Intercept the usage API request sent by the metering background task manager', function () {
+    let agentPolicyId: string;
+    let roleAuthc: RoleCredentials;
+    let internalRequestHeader: { 'x-elastic-internal-origin': string; 'kbn-xsrf': string };
+
+    before(async () => {
+      mockUsageApiServer.listen(8081); // Start the usage api mock server on port 8081
+    });
+
+    after(async () => {
+      // server.disable();
+      console.log('Mock server is stopped');
+    });
+
+    beforeEach(async () => {
+      roleAuthc = await svlUserManager.createApiKeyForRole('admin');
+      internalRequestHeader = svlCommonApi.getInternalRequestHeader();
+
+      await kibanaServer.savedObjects.cleanStandardList();
+      await esArchiver.load('x-pack/test/functional/es_archives/fleet/empty_fleet_server');
+
+      const { body: agentPolicyResponse } = await supertestWithoutAuth
+        .post(`/api/fleet/agent_policies`)
+        .set(internalRequestHeader)
+        .set(roleAuthc.apiKeyHeader)
+        .send({
+          name: 'Test policy',
+          namespace: 'default',
+        });
+
+      agentPolicyId = agentPolicyResponse.item.id;
+
+      await deleteIndex(es, [
+        LATEST_FINDINGS_INDEX_DEFAULT_NS,
+        LATEST_VULNERABILITIES_INDEX_DEFAULT_NS,
+        CLOUD_DEFEND_HEARTBEAT_INDEX_DEFAULT_NS,
+      ]);
+    });
+
+    afterEach(async () => {
+      await deleteIndex(es, [
+        LATEST_FINDINGS_INDEX_DEFAULT_NS,
+        LATEST_VULNERABILITIES_INDEX_DEFAULT_NS,
+      ]);
+      await kibanaServer.savedObjects.cleanStandardList();
+      await esArchiver.unload('x-pack/test/functional/es_archives/fleet/empty_fleet_server');
+      await deleteIndex(es, [
+        LATEST_FINDINGS_INDEX_DEFAULT_NS,
+        LATEST_VULNERABILITIES_INDEX_DEFAULT_NS,
+        CLOUD_DEFEND_HEARTBEAT_INDEX_DEFAULT_NS,
+      ]);
+    });
+
+    it('Should intercept usage API request for CSPM', async () => {
+      await createPackagePolicy(
+        supertestWithoutAuth,
+        agentPolicyId,
+        'cspm',
+        'cloudbeat/cis_aws',
+        'aws',
+        'cspm',
+        'CSPM-1',
+        roleAuthc,
+        internalRequestHeader
+      );
+      await addIndex(es, cspmFindingsMockDataForMetering, LATEST_FINDINGS_INDEX_DEFAULT_NS);
+      // await new Promise(() => {});
+      let interceptedRequestBody: UsageRecord[] = [];
+      await retry.try(async () => {
+        interceptedRequestBody = getInterceptedRequestPayload();
+        expect(interceptedRequestBody.length).to.greaterThan(0);
+        if (interceptedRequestBody.length > 0) {
+          const usageSubTypes = interceptedRequestBody.map((record) => record.usage.sub_type);
+          expect(usageSubTypes).to.contain('cspm');
+        }
+      });
+
+      expect(interceptedRequestBody[0].usage.type).to.be('cloud_security');
+      expect(interceptedRequestBody[0].usage.quantity).to.be(2);
+    });
+
+    it('Should intercept usage API request for KSPM', async () => {
+      await createPackagePolicy(
+        supertestWithoutAuth,
+        agentPolicyId,
+        'kspm',
+        'cloudbeat/cis_k8s',
+        'vanilla',
+        'kspm',
+        'KSPM-1',
+        roleAuthc,
+        internalRequestHeader
+      );
+      await addIndex(es, kspmFindingsMockDataForMetering, LATEST_FINDINGS_INDEX_DEFAULT_NS);
+
+      let interceptedRequestBody: UsageRecord[] = [];
+
+      await retry.try(async () => {
+        interceptedRequestBody = getInterceptedRequestPayload();
+        expect(interceptedRequestBody.length).to.greaterThan(0);
+        if (interceptedRequestBody.length > 0) {
+          const usageSubTypes = interceptedRequestBody.map((record) => record.usage.sub_type);
+          expect(usageSubTypes).to.contain('kspm');
+        }
+      });
+
+      expect(interceptedRequestBody[0].usage.type).to.be('cloud_security');
+      expect(interceptedRequestBody[0].usage.quantity).to.be(1);
+    });
+
+    it('Should intercept usage API request for CNVM', async () => {
+      await createPackagePolicy(
+        supertestWithoutAuth,
+        agentPolicyId,
+        'vuln_mgmt',
+        'cloudbeat/vuln_mgmt_aws',
+        'aws',
+        'vuln_mgmt',
+        'CNVM-1',
+        roleAuthc,
+        internalRequestHeader
+      );
+      await addIndex(es, cnvmFindingsMockDataForMetering, LATEST_VULNERABILITIES_INDEX_DEFAULT_NS);
+
+      let interceptedRequestBody: UsageRecord[] = [];
+
+      await retry.try(async () => {
+        interceptedRequestBody = getInterceptedRequestPayload();
+        expect(interceptedRequestBody.length).to.greaterThan(0);
+        if (interceptedRequestBody.length > 0) {
+          const usageSubTypes = interceptedRequestBody.map((record) => record.usage.sub_type);
+          expect(usageSubTypes).to.contain('cnvm');
+        }
+      });
+
+      expect(interceptedRequestBody[0].usage.type).to.be('cloud_security');
+      expect(interceptedRequestBody[0].usage.quantity).to.be(2);
+    });
+
+    it('Should intercept usage API request for Defend for Containers', async () => {
+      await es.indices.putMapping({
+        index: CLOUD_DEFEND_HEARTBEAT_INDEX_DEFAULT_NS,
+        properties: { event: { properties: { ingested: { type: 'date' } } } },
+      });
+      await addIndex(
+        es,
+        defendForContainersHeartbeatsForMetering,
+        CLOUD_DEFEND_HEARTBEAT_INDEX_DEFAULT_NS
+      );
+
+      let interceptedRequestBody: UsageRecord[] = [];
+
+      await retry.try(async () => {
+        interceptedRequestBody = getInterceptedRequestPayload();
+        expect(interceptedRequestBody.length).to.greaterThan(0);
+        if (interceptedRequestBody.length > 0) {
+          const usageSubTypes = interceptedRequestBody.map((record) => record.usage.sub_type);
+          expect(usageSubTypes).to.contain('cloud_defend');
+        }
+      });
+
+      expect(interceptedRequestBody[0].usage.type).to.be('cloud_security');
+      expect(interceptedRequestBody[0].usage.quantity).to.be(1);
+      expect(interceptedRequestBody[0].usage.period_seconds).to.be(3600);
+    });
+  });
+}