From 817331d6800409632d349a13a1f67da18468c152 Mon Sep 17 00:00:00 2001 From: Koushik Dutta Date: Mon, 25 Mar 2024 23:04:43 -0700 Subject: [PATCH] hikvision: implement smart events --- common/src/read-stream.ts | 11 +- plugins/hikvision/package-lock.json | 65 +++++- plugins/hikvision/package.json | 2 + plugins/hikvision/src/hikvision-camera-api.ts | 115 +++++++++-- plugins/hikvision/src/main.ts | 185 ++++++++++++++---- plugins/snapshot/src/image-reader.ts | 14 +- 6 files changed, 328 insertions(+), 64 deletions(-) diff --git a/common/src/read-stream.ts b/common/src/read-stream.ts index 190b6b345a..e73ddbf882 100644 --- a/common/src/read-stream.ts +++ b/common/src/read-stream.ts @@ -136,12 +136,17 @@ export async function readLine(readable: Readable) { } export async function readString(readable: Readable | Promise) { - let data = ''; + const buffer = await readBuffer(readable); + return buffer.toString(); +} + +export async function readBuffer(readable: Readable | Promise) { + const buffers: Buffer[] = []; readable = await readable; readable.on('data', buffer => { - data += buffer.toString(); + buffers.push(buffer); }); readable.resume(); await once(readable, 'end') - return data; + return Buffer.concat(buffers); } diff --git a/plugins/hikvision/package-lock.json b/plugins/hikvision/package-lock.json index 4a293de90a..e9adddaf23 100644 --- a/plugins/hikvision/package-lock.json +++ b/plugins/hikvision/package-lock.json @@ -12,10 +12,12 @@ "@scrypted/common": "file:../../common", "@scrypted/sdk": "file:../../sdk", "@types/xml2js": "^0.4.11", + "content-disposition": "^0.5.4", "lodash": "^4.17.21", "xml2js": "^0.6.0" }, "devDependencies": { + "@types/content-disposition": "^0.5.8", "@types/node": "^18.15.11" } }, @@ -27,17 +29,16 @@ "@scrypted/sdk": "file:../sdk", "@scrypted/server": "file:../server", "http-auth-utils": "^5.0.1", - "node-fetch-commonjs": "^3.1.1", "typescript": "^5.3.3" }, "devDependencies": { - "@types/node": "^20.10.8", + "@types/node": "^20.11.0", "ts-node": "^10.9.2" } }, "../../sdk": { "name": "@scrypted/sdk", - "version": "0.3.4", + "version": "0.3.29", "license": "ISC", "dependencies": { "@babel/preset-typescript": "^7.18.6", @@ -83,6 +84,12 @@ "resolved": "../../sdk", "link": true }, + "node_modules/@types/content-disposition": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/@types/content-disposition/-/content-disposition-0.5.8.tgz", + "integrity": "sha512-QVSSvno3dE0MgO76pJhmv4Qyi/j0Yk9pBp0Y7TJ2Tlj+KCgJWY6qX7nnxCOLkZ3VYRSIk1WTxCvwUSdx6CCLdg==", + "dev": true + }, "node_modules/@types/node": { "version": "18.15.11", "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.11.tgz", @@ -96,11 +103,41 @@ "@types/node": "*" } }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/sax": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", @@ -133,9 +170,8 @@ "requires": { "@scrypted/sdk": "file:../sdk", "@scrypted/server": "file:../server", - "@types/node": "^20.10.8", + "@types/node": "^20.11.0", "http-auth-utils": "^5.0.1", - "node-fetch-commonjs": "^3.1.1", "ts-node": "^10.9.2", "typescript": "^5.3.3" } @@ -164,6 +200,12 @@ "webpack-bundle-analyzer": "^4.5.0" } }, + "@types/content-disposition": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/@types/content-disposition/-/content-disposition-0.5.8.tgz", + "integrity": "sha512-QVSSvno3dE0MgO76pJhmv4Qyi/j0Yk9pBp0Y7TJ2Tlj+KCgJWY6qX7nnxCOLkZ3VYRSIk1WTxCvwUSdx6CCLdg==", + "dev": true + }, "@types/node": { "version": "18.15.11", "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.11.tgz", @@ -177,11 +219,24 @@ "@types/node": "*" } }, + "content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "requires": { + "safe-buffer": "5.2.1" + } + }, "lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + }, "sax": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", diff --git a/plugins/hikvision/package.json b/plugins/hikvision/package.json index b53b1b0169..65261a9dd1 100644 --- a/plugins/hikvision/package.json +++ b/plugins/hikvision/package.json @@ -38,10 +38,12 @@ "@scrypted/common": "file:../../common", "@scrypted/sdk": "file:../../sdk", "@types/xml2js": "^0.4.11", + "content-disposition": "^0.5.4", "lodash": "^4.17.21", "xml2js": "^0.6.0" }, "devDependencies": { + "@types/content-disposition": "^0.5.8", "@types/node": "^18.15.11" } } diff --git a/plugins/hikvision/src/hikvision-camera-api.ts b/plugins/hikvision/src/hikvision-camera-api.ts index 5e2d6bb4fe..1a556632b2 100644 --- a/plugins/hikvision/src/hikvision-camera-api.ts +++ b/plugins/hikvision/src/hikvision-camera-api.ts @@ -1,8 +1,16 @@ import { AuthFetchCredentialState, HttpFetchOptions, authHttpFetch } from '@scrypted/common/src/http-auth-fetch'; +import { readLine } from '@scrypted/common/src/read-stream'; +import { parseHeaders, readBody, readMessage } from '@scrypted/common/src/rtsp-server'; +import contentDisposition from 'content-disposition'; import { IncomingMessage } from 'http'; -import { Readable } from 'stream'; +import { EventEmitter, Readable } from 'stream'; +import { Destroyable } from '../../rtsp/src/rtsp'; import { getDeviceInfo } from './probe'; +export const detectionMap = { + human: 'person', +} + export function getChannel(channel: string) { return channel || '101'; } @@ -15,6 +23,8 @@ export enum HikvisionCameraEvent { // linedetection // inactive LineDetection = "linedetection", + RegionEntrance = "regionEntrance", + RegionExit = "regionExit", // fielddetection // active // fielddetection @@ -31,7 +41,7 @@ export interface HikvisionCameraStreamSetup { export class HikvisionCameraAPI { credential: AuthFetchCredentialState; deviceModel: Promise; - listenerPromise: Promise; + listenerPromise: Promise; constructor(public ip: string, username: string, password: string, public console: Console) { this.credential = { @@ -129,35 +139,106 @@ export class HikvisionCameraAPI { return response.body; } - async listenEvents() { + async listenEvents(): Promise { + const events = new EventEmitter(); + (events as any).destroy = () => { }; // support multiple cameras listening to a single single stream if (!this.listenerPromise) { const url = `http://${this.ip}/ISAPI/Event/notification/alertStream`; + + let lastSmartDetection: string; + this.listenerPromise = this.request({ url, responseType: 'readable', }).then(response => { - const stream = response.body; + const stream: IncomingMessage = response.body; + (events as any).destroy = () => { + stream.destroy(); + events.removeAllListeners(); + }; + stream.on('close', () => { + this.listenerPromise = undefined; + events.emit('close'); + }); + stream.on('end', () => { + this.listenerPromise = undefined; + events.emit('end'); + }); + stream.on('error', e => { + events.emit('error', e); + }); stream.socket.setKeepAlive(true); - stream.on('data', (buffer: Buffer) => { - const data = buffer.toString(); - for (const event of Object.values(HikvisionCameraEvent)) { - if (data.indexOf(event) !== -1) { - const cameraNumber = data.match(/(.*?)(.*?)inactive') !== -1; - stream.emit('event', event, cameraNumber, inactive, data); + const ct = stream.headers['content-type']; + // make content type parsable as content disposition filename + const cd = contentDisposition.parse(ct.replace('/', '')); + let { boundary } = cd.parameters; + boundary = `--${boundary}`; + const boundaryEnd = `${boundary}--`; + + + (async () => { + while (true) { + let ignore = await readLine(stream); + ignore = ignore.trim(); + if (!ignore) + continue; + if (ignore === boundaryEnd) + continue; + if (ignore !== boundary) { + this.console.error('expected boundary but found', ignore); + throw new Error('expected boundary'); + } + + const message = await readMessage(stream); + events.emit('data', message); + message.unshift(''); + const headers = parseHeaders(message); + const body = await readBody(stream, headers); + + try { + if (!headers['content-type'].includes('application/xml') && lastSmartDetection) { + const cd = contentDisposition.parse(headers['content-disposition'] || 'empty'); + if (!headers['content-type']?.startsWith('image/jpeg')) { + continue; + } + events.emit('smart', lastSmartDetection, body); + lastSmartDetection = undefined; + continue; + } + + } + finally { + // is it possible that smart detections are sent without images? + // if so, flush this detection. + if (lastSmartDetection) { + events.emit('smart', lastSmartDetection); + } + } + + const data = body.toString(); + events.emit('data', data); + for (const event of Object.values(HikvisionCameraEvent)) { + if (data.indexOf(event) !== -1) { + const cameraNumber = data.match(/(.*?)(.*?)inactive') !== -1; + events.emit('event', event, cameraNumber, inactive, data); + if (event === HikvisionCameraEvent.LineDetection + || event === HikvisionCameraEvent.RegionEntrance + || event === HikvisionCameraEvent.RegionExit + || event === HikvisionCameraEvent.FieldDetection) { + lastSmartDetection = data; + } + } } } - }); - return stream; + })() + .catch(() => stream.destroy()); + return events as any as Destroyable; }); this.listenerPromise.catch(() => this.listenerPromise = undefined); - this.listenerPromise.then(stream => { - stream.on('close', () => this.listenerPromise = undefined); - stream.on('end', () => this.listenerPromise = undefined); - }); } return this.listenerPromise; diff --git a/plugins/hikvision/src/main.ts b/plugins/hikvision/src/main.ts index 17260f7bdc..8fefcc4404 100644 --- a/plugins/hikvision/src/main.ts +++ b/plugins/hikvision/src/main.ts @@ -1,11 +1,12 @@ -import sdk, { Camera, DeviceCreatorSettings, DeviceInformation, FFmpegInput, Intercom, MediaObject, MediaStreamOptions, Reboot, RequestPictureOptions, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting } from "@scrypted/sdk"; +import sdk, { Camera, DeviceCreatorSettings, DeviceInformation, FFmpegInput, Intercom, MediaObject, MediaStreamOptions, ObjectDetectionResult, ObjectDetectionTypes, ObjectDetector, ObjectsDetected, Reboot, RequestPictureOptions, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting } from "@scrypted/sdk"; +import crypto from 'crypto'; import { PassThrough } from "stream"; import xml2js from 'xml2js'; import { RtpPacket } from '../../../external/werift/packages/rtp/src/rtp/rtp'; import { OnvifIntercom } from "../../onvif/src/onvif-intercom"; import { RtspProvider, RtspSmartCamera, UrlMediaStreamOptions } from "../../rtsp/src/rtsp"; import { startRtpForwarderProcess } from '../../webrtc/src/rtp-forwarders'; -import { HikvisionCameraAPI, HikvisionCameraEvent } from "./hikvision-camera-api"; +import { HikvisionCameraAPI, HikvisionCameraEvent, detectionMap } from "./hikvision-camera-api"; const { mediaManager } = sdk; @@ -15,15 +16,17 @@ function channelToCameraNumber(channel: string) { return channel.substring(0, channel.length - 2); } -class HikvisionCamera extends RtspSmartCamera implements Camera, Intercom, Reboot { +class HikvisionCamera extends RtspSmartCamera implements Camera, Intercom, Reboot, ObjectDetector { detectedChannels: Promise>; client: HikvisionCameraAPI; onvifIntercom = new OnvifIntercom(this); activeIntercom: Awaited>; + hasSmartDetection: boolean; constructor(nativeId: string, provider: RtspProvider) { super(nativeId, provider); + this.hasSmartDetection = this.storage.getItem('hasSmartDetection') === 'true'; this.updateDevice(); this.updateDeviceInfo(); } @@ -63,41 +66,52 @@ class HikvisionCamera extends RtspSmartCamera implements Camera, Intercom, Reboo let ignoreCameraNumber: boolean; const motionTimeoutDuration = 20000; - events.on('event', async (event: HikvisionCameraEvent, cameraNumber: string, inactive: boolean) => { - if (event === HikvisionCameraEvent.MotionDetected - || event === HikvisionCameraEvent.LineDetection - || event === HikvisionCameraEvent.FieldDetection) { - // check if the camera+channel field is in use, and filter events. - if (this.getRtspChannel()) { - // it is possible to set it up to use a camera number - // on an nvr IP (which gives RTSP urls through the NVR), but then use a http port - // that gives a filtered event stream from only that camera. - // this this case, the camera numbers will not - // match as they will be always be "1". - // to detect that a camera specific endpoint is being used - // can look at the channel ids, and see if that camera number is found. - // this is different from the use case where the NVR or camera - // is using a port other than 80 (the default). - // could add a setting to have the user explicitly denote nvr usage - // but that is error prone. - const userCameraNumber = this.getCameraNumber(); - if (ignoreCameraNumber === undefined && this.detectedChannels) { - const channelIds = (await this.detectedChannels).keys(); - ignoreCameraNumber = true; - for (const id of channelIds) { - if (channelToCameraNumber(id) === userCameraNumber) { - ignoreCameraNumber = false; - break; - } + // check if the camera+channel field is in use, and filter events. + const checkCameraNumber = async (cameraNumber: string) => { + // check if the camera+channel field is in use, and filter events. + if (this.getRtspChannel()) { + // it is possible to set it up to use a camera number + // on an nvr IP (which gives RTSP urls through the NVR), but then use a http port + // that gives a filtered event stream from only that camera. + // this this case, the camera numbers will not + // match as they will be always be "1". + // to detect that a camera specific endpoint is being used + // can look at the channel ids, and see if that camera number is found. + // this is different from the use case where the NVR or camera + // is using a port other than 80 (the default). + // could add a setting to have the user explicitly denote nvr usage + // but that is error prone. + const userCameraNumber = this.getCameraNumber(); + if (ignoreCameraNumber === undefined && this.detectedChannels) { + const channelIds = (await this.detectedChannels).keys(); + ignoreCameraNumber = true; + for (const id of channelIds) { + if (channelToCameraNumber(id) === userCameraNumber) { + ignoreCameraNumber = false; + break; } } + } - if (!ignoreCameraNumber && cameraNumber !== userCameraNumber) { - // this.console.error(`### Skipping motion event ${cameraNumber} != ${this.getCameraNumber()}`); - return; - } + if (!ignoreCameraNumber && cameraNumber !== userCameraNumber) { + // this.console.error(`### Skipping motion event ${cameraNumber} != ${this.getCameraNumber()}`); + return false; } + } + + return true; + }; + + events.on('event', async (event: HikvisionCameraEvent, cameraNumber: string, inactive: boolean) => { + if (event === HikvisionCameraEvent.MotionDetected + || event === HikvisionCameraEvent.LineDetection + || event === HikvisionCameraEvent.RegionEntrance + || event === HikvisionCameraEvent.RegionExit + || event === HikvisionCameraEvent.FieldDetection) { + + if (!checkCameraNumber(cameraNumber)) + return; this.motionDetected = true; clearTimeout(motionTimeout); @@ -106,11 +120,107 @@ class HikvisionCamera extends RtspSmartCamera implements Camera, Intercom, Reboo this.motionDetected = false; }, motionTimeoutDuration); } - }) + }); + + let inputDimensions: [number, number]; + + events.on('smart', async (data: string, image: Buffer) => { + if (!this.hasSmartDetection) { + this.hasSmartDetection = true; + this.storage.setItem('hasSmartDetection', 'true'); + this.updateDevice(); + } + + const xml = await xml2js.parseStringPromise(data); + + + const [channelId] = xml.EventNotificationAlert.channelID; + if (!checkCameraNumber(channelId)) { + this.console.warn('chann fail') + return; + } + + const now = Date.now(); + let detections: ObjectDetectionResult[] = xml.EventNotificationAlert?.DetectionRegionList?.map(region => { + const { DetectionRegionEntry } = region; + const dre = DetectionRegionEntry[0]; + if (!DetectionRegionEntry) + return; + const { detectionTarget } = dre; + // const { TargetRect } = dre; + // const { X, Y, width, height } = TargetRect[0]; + const [name] = detectionTarget; + return { + score: 1, + className: detectionMap[name] || name, + // boundingBox: [ + // parseInt(X), + // parseInt(Y), + // parseInt(width), + // parseInt(height), + // ], + // movement: { + // moving: true, + // firstSeen: now, + // lastSeen: now, + // } + } as ObjectDetectionResult; + }); + + detections = detections.filter(d => d); + if (!detections.length) + return; + + // if (inputDimensions === undefined && loadSharp()) { + // try { + // const { image: i, metadata } = await loadVipsMetadata(image); + // i.destroy(); + // inputDimensions = [metadata.width, metadata.height]; + // } + // catch (e) { + // inputDimensions = null; + // } + // finally { + // } + // } + + let detectionId: string; + if (image) { + detectionId = crypto.randomBytes(4).toString('hex'); + this.recentDetections.set(detectionId, image); + setTimeout(() => this.recentDetections.delete(detectionId), 10000); + } + + const detected: ObjectsDetected = { + inputDimensions, + detectionId, + timestamp: now, + detections, + }; + + this.onDeviceEvent(ScryptedInterface.ObjectDetector, detected); + }); return events; } + recentDetections = new Map(); + + async getDetectionInput(detectionId: string, eventId?: any): Promise { + const image = this.recentDetections.get(detectionId); + if (!image) + return; + return mediaManager.createMediaObject(image, 'image/jpeg'); + } + + async getObjectTypes(): Promise { + return { + classes: [ + 'person', + ] + } + } + createClient() { return new HikvisionCameraAPI(this.getHttpAddress(), this.getUsername(), this.getPassword(), this.console); } @@ -284,6 +394,9 @@ class HikvisionCamera extends RtspSmartCamera implements Camera, Intercom, Reboo interfaces.push(ScryptedInterface.Intercom); } + if (this.hasSmartDetection) + interfaces.push(ScryptedInterface.ObjectDetector); + this.provider.updateDevice(this.nativeId, this.name, interfaces, type); } @@ -454,7 +567,7 @@ class HikvisionCamera extends RtspSmartCamera implements Camera, Intercom, Reboo if (response.statusCode !== 200) forwarder.kill(); }) - .catch(() => forwarder.kill()); + .catch(() => forwarder.kill()); } async stopIntercom(): Promise { @@ -587,4 +700,4 @@ class HikvisionProvider extends RtspProvider { } } -export default new HikvisionProvider(); +export default HikvisionProvider; diff --git a/plugins/snapshot/src/image-reader.ts b/plugins/snapshot/src/image-reader.ts index 2b903b02c2..6f7e24baa1 100644 --- a/plugins/snapshot/src/image-reader.ts +++ b/plugins/snapshot/src/image-reader.ts @@ -132,13 +132,21 @@ export class VipsImage implements Image { } } -export async function loadVipsImage(data: Buffer | string, sourceId: string) { - loadSharp(); - +export async function loadVipsMetadata(data: Buffer | string) { const image = sharpInstance(data, { failOn: 'none' }); const metadata = await image.metadata(); + return { + image, + metadata, + } +} + +export async function loadVipsImage(data: Buffer | string, sourceId: string) { + loadSharp(); + + const { image, metadata } = await loadVipsMetadata(data); const vipsImage = new VipsImage(image, metadata, sourceId); return vipsImage; }