diff --git a/plugins/objectdetector/README.md b/plugins/objectdetector/README.md index 9762f62b94..6ba22115ea 100644 --- a/plugins/objectdetector/README.md +++ b/plugins/objectdetector/README.md @@ -13,4 +13,8 @@ benefits to HomeKit, which does its own detection processing. ## Smart Motion Sensors -This plugin can be used to create smart motion sensors that trigger when a specific type of object (car, person, dog, etc) triggers movement on a camera. Created sensors can then be synced to other platforms such as HomeKit, Google Home, Alexa, or Home Assistant for use in automations. This feature requires cameras with hardware or software object detection capability. +This plugin can be used to create smart motion sensors that trigger when a specific type of object (vehicle, person, animal, etc) triggers movement on a camera. Created sensors can then be synced to other platforms such as HomeKit, Google Home, Alexa, or Home Assistant for use in automations. This Sensor requires cameras with hardware or software object detection capability. + +## Smart Occupancy Sensors + +This plugin can be used to create smart occupancy sensors remains triggered when a specific type of object (vehicle, person, animal, etc) is detected on a camera. Created sensors can then be synced to other platforms such as HomeKit, Google Home, Alexa, or Home Assistant for use in automations. This Sensor requires an object detector plugin such as Scrypted NVR, OpenVINO, CoreML, ONNX, or Tensorflow-lite. \ No newline at end of file diff --git a/plugins/objectdetector/package-lock.json b/plugins/objectdetector/package-lock.json index 051a01771a..65264faa31 100644 --- a/plugins/objectdetector/package-lock.json +++ b/plugins/objectdetector/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/objectdetector", - "version": "0.1.52", + "version": "0.1.55", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@scrypted/objectdetector", - "version": "0.1.52", + "version": "0.1.55", "license": "Apache-2.0", "dependencies": { "@scrypted/common": "file:../../common", diff --git a/plugins/objectdetector/package.json b/plugins/objectdetector/package.json index 7c9e788fb7..bae3ee240f 100644 --- a/plugins/objectdetector/package.json +++ b/plugins/objectdetector/package.json @@ -1,6 +1,6 @@ { "name": "@scrypted/objectdetector", - "version": "0.1.52", + "version": "0.1.55", "description": "Scrypted Video Analysis Plugin. Installed alongside a detection service like OpenCV or TensorFlow.", "author": "Scrypted", "license": "Apache-2.0", diff --git a/plugins/objectdetector/src/main.ts b/plugins/objectdetector/src/main.ts index a837cf811f..58b3d8170f 100644 --- a/plugins/objectdetector/src/main.ts +++ b/plugins/objectdetector/src/main.ts @@ -6,8 +6,9 @@ import crypto from 'crypto'; import { AutoenableMixinProvider } from "../../../common/src/autoenable-mixin-provider"; import { SettingsMixinDeviceBase } from "../../../common/src/settings-mixin"; import { FFmpegVideoFrameGenerator } from './ffmpeg-videoframes'; -import { insidePolygon, normalizeBox, polygonOverlap } from './polygon'; -import { SMART_MOTIONSENSOR_PREFIX, SmartMotionSensor, createObjectDetectorStorageSetting } from './smart-motionsensor'; +import { fixLegacyClipPath, insidePolygon, normalizeBoxToClipPath, polygonOverlap } from './polygon'; +import { SMART_MOTIONSENSOR_PREFIX, SmartMotionSensor } from './smart-motionsensor'; +import { SMART_OCCUPANCYSENSOR_PREFIX, SmartOccupancySensor } from './smart-occupancy-sensor'; import { getAllDevices, safeParseJson } from './util'; @@ -542,7 +543,7 @@ class ObjectDetectionMixin extends SettingsMixinDeviceBase { @@ -1194,6 +1196,8 @@ export class ObjectDetectionPlugin extends AutoenableMixinProvider implements Se ret = this.devices.get(nativeId) || new FFmpegVideoFrameGenerator('ffmpeg'); if (nativeId?.startsWith(SMART_MOTIONSENSOR_PREFIX)) ret = this.devices.get(nativeId) || new SmartMotionSensor(this, nativeId); + if (nativeId?.startsWith(SMART_OCCUPANCYSENSOR_PREFIX)) + ret = this.devices.get(nativeId) || new SmartOccupancySensor(this, nativeId); if (ret) this.devices.set(nativeId, ret); @@ -1204,6 +1208,13 @@ export class ObjectDetectionPlugin extends AutoenableMixinProvider implements Se if (nativeId?.startsWith(SMART_MOTIONSENSOR_PREFIX)) { const smart = this.devices.get(nativeId) as SmartMotionSensor; smart?.detectionListener?.removeListener(); + smart?.resetMotionTimeout(); + } + if (nativeId?.startsWith(SMART_OCCUPANCYSENSOR_PREFIX)) { + const smart = this.devices.get(nativeId) as SmartOccupancySensor; + smart?.detectionListener?.removeListener(); + smart?.resetOccupiedTimeout(); + smart?.clearOccupancyInterval(); } } @@ -1239,32 +1250,71 @@ export class ObjectDetectionPlugin extends AutoenableMixinProvider implements Se async getCreateDeviceSettings(): Promise { return [ - createObjectDetectorStorageSetting(), + { + key: 'sensorType', + title: 'Sensor Type', + description: 'Select the type of sensor to create.', + choices: [ + 'Smart Motion Sensor', + 'Smart Occupancy Sensor', + ], + }, + { + key: 'camera', + title: 'Camera', + description: 'Select a camera or doorbell.', + type: 'device', + deviceFilter: `type === '${ScryptedDeviceType.Doorbell}' || type === '${ScryptedDeviceType.Camera}'`, + }, ]; } async createDevice(settings: DeviceCreatorSettings): Promise { - const nativeId = SMART_MOTIONSENSOR_PREFIX + crypto.randomBytes(8).toString('hex'); - const objectDetector = sdk.systemManager.getDeviceById(settings.objectDetector as string); - let name = objectDetector.name || 'New'; - name += ' Smart Motion Sensor' - - const id = await sdk.deviceManager.onDeviceDiscovered({ - nativeId, - name, - type: ScryptedDeviceType.Sensor, - interfaces: [ - ScryptedInterface.Camera, - ScryptedInterface.MotionSensor, - ScryptedInterface.Settings, - ScryptedInterface.Readme, - ] - }); + const sensorType = settings.sensorType; + const camera = sdk.systemManager.getDeviceById(settings.camera as string); + if (sensorType === 'Smart Motion Sensor') { + const nativeId = SMART_MOTIONSENSOR_PREFIX + crypto.randomBytes(8).toString('hex'); + let name = camera.name || 'New'; + name += ' Smart Motion Sensor' + + const id = await sdk.deviceManager.onDeviceDiscovered({ + nativeId, + name, + type: ScryptedDeviceType.Sensor, + interfaces: [ + ScryptedInterface.Camera, + ScryptedInterface.MotionSensor, + ScryptedInterface.Settings, + ScryptedInterface.Readme, + ] + }); - const sensor = new SmartMotionSensor(this, nativeId); - sensor.storageSettings.values.objectDetector = objectDetector?.id; + const sensor = new SmartMotionSensor(this, nativeId); + sensor.storageSettings.values.objectDetector = camera?.id; - return id; + return id; + } + else if (sensorType === 'Smart Occupancy Sensor') { + const nativeId = SMART_OCCUPANCYSENSOR_PREFIX + crypto.randomBytes(8).toString('hex'); + let name = camera.name || 'New'; + name += ' Smart Occupancy Sensor' + + const id = await sdk.deviceManager.onDeviceDiscovered({ + nativeId, + name, + type: ScryptedDeviceType.Sensor, + interfaces: [ + ScryptedInterface.OccupancySensor, + ScryptedInterface.Settings, + ScryptedInterface.Readme, + ] + }); + + const sensor = new SmartOccupancySensor(this, nativeId); + sensor.storageSettings.values.camera = camera?.id; + + return id; + } } } diff --git a/plugins/objectdetector/src/polygon.ts b/plugins/objectdetector/src/polygon.ts index 49de13d805..e33cf85669 100644 --- a/plugins/objectdetector/src/polygon.ts +++ b/plugins/objectdetector/src/polygon.ts @@ -1,4 +1,4 @@ -import { Point } from '@scrypted/sdk'; +import type { ClipPath, Point } from '@scrypted/sdk'; import polygonClipping from 'polygon-clipping'; // const polygonOverlap = require('polygon-overlap'); @@ -14,15 +14,36 @@ export function insidePolygon(point: Point, polygon: Point[]) { return !!intersect.length; } -export function normalizeBox(boundingBox: [number, number, number, number], inputDimensions: [number, number], scalar = 100): [Point, Point, Point, Point] { +export function fixLegacyClipPath(clipPath: ClipPath): ClipPath { + if (!clipPath) + return; + + // if any value is over abs 2, then divide by 100. + // this is a workaround for the old scrypted bug where the path was not normalized. + // this is a temporary workaround until the path is normalized in the UI. + let needNormalize = false; + for (const p of clipPath) { + for (const c of p) { + if (Math.abs(c) >= 2) + needNormalize = true; + } + } + + if (!needNormalize) + return clipPath; + + return clipPath.map(p => p.map(c => c / 100)) as ClipPath; +} + +export function normalizeBoxToClipPath(boundingBox: [number, number, number, number], inputDimensions: [number, number]): [Point, Point, Point, Point] { let [x, y, width, height] = boundingBox; let x2 = x + width; let y2 = y + height; // the zones are point paths in percentage format - x = x * scalar / inputDimensions[0]; - y = y * scalar / inputDimensions[1]; - x2 = x2 * scalar / inputDimensions[0]; - y2 = y2 * scalar / inputDimensions[1]; + x = x / inputDimensions[0]; + y = y / inputDimensions[1]; + x2 = x2 / inputDimensions[0]; + y2 = y2 / inputDimensions[1]; return [[x, y], [x2, y], [x2, y2], [x, y2]]; } diff --git a/plugins/objectdetector/src/smart-motionsensor.ts b/plugins/objectdetector/src/smart-motionsensor.ts index 369331beee..d10159fa1e 100644 --- a/plugins/objectdetector/src/smart-motionsensor.ts +++ b/plugins/objectdetector/src/smart-motionsensor.ts @@ -1,24 +1,18 @@ import sdk, { Camera, EventListenerRegister, MediaObject, MotionSensor, ObjectDetector, ObjectsDetected, Readme, RequestPictureOptions, ResponsePictureOptions, ScryptedDevice, ScryptedDeviceBase, ScryptedDeviceType, ScryptedInterface, ScryptedNativeId, Setting, SettingValue, Settings } from "@scrypted/sdk"; -import { StorageSetting, StorageSettings } from "@scrypted/sdk/storage-settings"; +import { StorageSettings } from "@scrypted/sdk/storage-settings"; import { levenshteinDistance } from "./edit-distance"; import type { ObjectDetectionPlugin } from "./main"; export const SMART_MOTIONSENSOR_PREFIX = 'smart-motionsensor-'; -export const SMART_OCCUPANCYSENSOR_PREFIX = 'smart-occupancysensor-'; - -export function createObjectDetectorStorageSetting(): StorageSetting { - return { - key: 'objectDetector', - title: 'Object Detector', - description: 'Select the camera or doorbell that provides smart detection event.', - type: 'device', - deviceFilter: `(type === '${ScryptedDeviceType.Doorbell}' || type === '${ScryptedDeviceType.Camera}') && interfaces.includes('${ScryptedInterface.ObjectDetector}')`, - }; -} export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, Readme, MotionSensor, Camera { storageSettings = new StorageSettings(this, { - objectDetector: createObjectDetectorStorageSetting(), + objectDetector: { + title: 'Camera', + description: 'Select a camera or doorbell that provides smart detection events.', + type: 'device', + deviceFilter: `(type === '${ScryptedDeviceType.Doorbell}' || type === '${ScryptedDeviceType.Camera}') && interfaces.includes('${ScryptedInterface.ObjectDetector}')`, + }, detections: { title: 'Detections', description: 'The detections that will trigger this smart motion sensor.', @@ -145,13 +139,13 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R return; } - resetTrigger() { + resetMotionTimeout() { clearTimeout(this.timeout); this.timeout = undefined; } trigger() { - this.resetTrigger(); + this.resetMotionTimeout(); this.motionDetected = true; const duration: number = this.storageSettings.values.detectionTimeout; if (!duration) @@ -167,7 +161,7 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R this.detectionListener = undefined; this.motionListener?.removeListener(); this.motionListener = undefined; - this.resetTrigger(); + this.resetMotionTimeout(); const objectDetector: ObjectDetector & MotionSensor & ScryptedDevice = this.storageSettings.values.objectDetector; @@ -178,8 +172,6 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R if (!detections?.length) return; - const console = sdk.deviceManager.getMixinConsole(objectDetector.id, this.nativeId); - this.motionListener = objectDetector.listen({ event: ScryptedInterface.MotionSensor, watch: true, @@ -258,7 +250,7 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R if (match) { if (!this.motionDetected) - console.log('Smart Motion Sensor triggered on', match); + this.console.log('Smart Motion Sensor triggered on', match); if (detected.detectionId) this.lastPicture = objectDetector.getDetectionInput(detected.detectionId, details.eventId); this.trigger(); @@ -278,6 +270,6 @@ export class SmartMotionSensor extends ScryptedDeviceBase implements Settings, R return ` ## Smart Motion Sensor -This Smart Motion Sensor can trigger when a specific type of object (car, person, dog, etc) triggers movement on a camera. The sensor can then be synced to other platforms such as HomeKit, Google Home, Alexa, or Home Assistant for use in automations. This Sensor requires a camera with hardware or software object detection capability.`; +This Smart Motion Sensor can trigger when a specific type of object (vehicle, person, animal, etc) triggers movement on a camera. The sensor can then be synced to other platforms such as HomeKit, Google Home, Alexa, or Home Assistant for use in automations. This Sensor requires a camera with hardware or software object detection capability.`; } } diff --git a/plugins/sample-cameraprovider b/plugins/sample-cameraprovider index 51bbc2be20..ce75c61948 160000 --- a/plugins/sample-cameraprovider +++ b/plugins/sample-cameraprovider @@ -1 +1 @@ -Subproject commit 51bbc2be202f41c172ae7b61b52650fe7359a025 +Subproject commit ce75c6194841e84e9c44ed0ce5383c1e74f632db diff --git a/server/package-lock.json b/server/package-lock.json index 4e908b18c1..21a5d7f7f3 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -1,12 +1,12 @@ { "name": "@scrypted/server", - "version": "0.123.67", + "version": "0.123.68", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@scrypted/server", - "version": "0.123.67", + "version": "0.123.68", "hasInstallScript": true, "license": "ISC", "dependencies": {