Add Caption Bot demo

Documentation/CaptionBot.md

@@ -0,0 +1,11 @@
+# Caption Bot
+
+![alt text](https://github.com/Microsoft/Cognitive-Samples-IntelligentKiosk/blob/master/Documentation/CaptionBot.jpg "Caption Bot")
+
+A realtime workflow for processing frames from a web camera to automatically generate a caption that describes what the algorithm sees. It uses the Computer Vision API to identify the components of the captures.
+
+# Key Source Code
+
+* [CaptionKioskPage](../Kiosk/Views/CaptionKioskPage.xaml.cs): Main page that drives the demo. It hosts the CameraControl (see below) to display the live camera feed, configures it to run in autonomous mode and reacts to the state machine changes to capture photos and analyze them.
+
+* [CameraControl](../Kiosk/Controls/CameraControl.xaml.cs): The code that contains the camera feed and runs a background loop to perform several tasks (track faces, draw face rectangles, execute the state machine that drives the autonomous photo capture workflow, etc)

Kiosk/Controls/CameraControl.xaml.cs

@@ -157,6 +157,7 @@ public bool EnableCameraControls
         public int CameraResolutionHeight { get; private set; }
 
         public int NumFacesOnLastFrame { get; set; }
+        public DateTime LastFaceTimestamp { get; set; }
         public int ContinuousModeTimerInSecond { get; set; } = 5;
 
         public CameraStreamState CameraStreamState { get { return captureManager != null ? captureManager.CameraStreamState : CameraStreamState.NotStreaming; } }
@@ -345,6 +346,10 @@ private async void ProcessCurrentVideoFrame(ThreadPoolTimer timer)
                         }
 
                         this.NumFacesOnLastFrame = faces.Count();
+                        if (this.NumFacesOnLastFrame != 0)
+                        {
+                            this.LastFaceTimestamp = DateTime.Now;
+                        }
 
                         if (this.EnableAutoCaptureMode)
                         {

Kiosk/IntelligentKioskSample.csproj

@@ -101,9 +101,11 @@
     <UseDotNetNativeToolchain>true</UseDotNetNativeToolchain>
   </PropertyGroup>
   <ItemGroup>
+    <Content Include="Assets\captionbot.png" />
     <Content Include="Assets\DemoGallery\Anomaly detector.jpg" />
     <Content Include="Assets\DemoGallery\Bing News Analytics.jpg" />
     <Content Include="Assets\DemoGallery\Bing Visual Search.jpg" />
+    <Content Include="Assets\DemoGallery\Caption Bot.jpg" />
     <Content Include="Assets\DemoGallery\Custom Vision Explorer.jpg" />
     <Content Include="Assets\DemoGallery\Face API Explorer.jpg" />
     <Content Include="Assets\DemoGallery\Forms Recognizer.jpg" />
@@ -287,6 +289,9 @@
     <Compile Include="Views\BingVisualSearch.xaml.cs">
       <DependentUpon>BingVisualSearch.xaml</DependentUpon>
     </Compile>
+    <Compile Include="Views\CaptionKioskPage.xaml.cs">
+      <DependentUpon>CaptionKioskPage.xaml</DependentUpon>
+    </Compile>
     <Compile Include="Views\CustomVision\CustomObjectDetectionModel.cs" />
     <Compile Include="Views\CustomVision\CustomVisionDataLoader.cs" />
     <Compile Include="Views\CustomVision\CustomVisionExplorer.xaml.cs">
@@ -601,6 +606,10 @@
       <Generator>MSBuild:Compile</Generator>
       <SubType>Designer</SubType>
     </Page>
+    <Page Include="Views\CaptionKioskPage.xaml">
+      <SubType>Designer</SubType>
+      <Generator>MSBuild:Compile</Generator>
+    </Page>
     <Page Include="Views\CustomVision\CustomVisionExplorer.xaml">
       <Generator>MSBuild:Compile</Generator>
       <SubType>Designer</SubType>

Kiosk/Views/CaptionKioskPage.xaml

@@ -0,0 +1,93 @@
+<Page
+    x:Class="IntelligentKioskSample.Views.CaptionKioskPage"
+    xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+    xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+    xmlns:ctl="using:IntelligentKioskSample.Controls"
+    xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
+    xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
+    SizeChanged="OnPageSizeChanged"
+    mc:Ignorable="d">
+
+    <Grid>
+        <Grid x:Name="MainGrid" Background="#008272" EntranceNavigationTransitionInfo.IsTargetElement="True">
+            <Grid.RowDefinitions>
+                <RowDefinition Height="0.05*" />
+                <RowDefinition Height="0.10*" />
+                <RowDefinition Height="0.05*" />
+                <RowDefinition />
+                <RowDefinition Height="0.015*" />
+            </Grid.RowDefinitions>
+
+            <Grid Grid.Row="1" Grid.Column="1" HorizontalAlignment="Center" VerticalAlignment="Center">
+                <Grid.ColumnDefinitions>
+                    <ColumnDefinition Width="0.3*"/>
+                    <ColumnDefinition/>
+                </Grid.ColumnDefinitions>
+
+                <Image Grid.RowSpan="2" Source="ms-appx:///Assets/captionbot.png" HorizontalAlignment="Center" VerticalAlignment="Center" Margin="0,0,4,0" />
+
+                <Grid Grid.Column="1">
+                    <Grid.RowDefinitions>
+                        <RowDefinition Height="0.6*"/>
+                        <RowDefinition Height="0.2*"/>
+                    </Grid.RowDefinitions>
+
+                    <Grid.ColumnDefinitions>
+                        <ColumnDefinition/>
+                        <ColumnDefinition Width="0.3*"/>
+                    </Grid.ColumnDefinitions>
+
+                    <Viewbox Grid.ColumnSpan="2" HorizontalAlignment="Left" VerticalAlignment="Bottom">
+                        <TextBlock Text="CaptionBot.ai" Foreground="White" FontFamily="Segoe UI Light" />
+                    </Viewbox>
+
+                    <Viewbox  Grid.Row="1" HorizontalAlignment="Left">
+                        <StackPanel Orientation="Horizontal">
+                            <TextBlock Margin="6,0,0,0" Text="I will describe whatever I see!" FontFamily="Segoe UI Semibold" />
+                            <TextBlock Margin="12,0,0,0" Text="#CaptionBot" FontFamily="Segoe UI Semibold" Foreground="LightGray" />
+                        </StackPanel>
+                    </Viewbox>
+                </Grid>
+            </Grid>
+
+            <Grid Grid.Row="3" HorizontalAlignment="Center" VerticalAlignment="Center">
+                <Grid.ColumnDefinitions>
+                    <ColumnDefinition Width="0.075*"/>
+                    <ColumnDefinition />
+                    <ColumnDefinition Width="0.075*"/>
+                </Grid.ColumnDefinitions>
+
+                <Grid x:Name="cameraHostGrid" Grid.Column="1" HorizontalAlignment="Center" VerticalAlignment="Center">
+                    <ctl:CameraControl x:Name="cameraControl" />
+                </Grid>
+
+                <TextBlock Opacity="0.6" Margin="0,0,2,0" Grid.Column="1" x:Name="debugText"  HorizontalAlignment="Right" VerticalAlignment="Bottom" />
+
+                <Grid Grid.Column="1" x:Name="cameraGuideHost">
+                    <Grid.ColumnDefinitions>
+                        <ColumnDefinition/>
+                        <ColumnDefinition/>
+                        <ColumnDefinition/>
+                    </Grid.ColumnDefinitions>
+                    <Grid Name="cameraGuideBallon" VerticalAlignment="Top" Grid.Column="2" Margin="0,0,10,0" >
+                        <Grid.RowDefinitions>
+                            <RowDefinition Height="auto"/>
+                            <RowDefinition Height="auto"/>
+                        </Grid.RowDefinitions>
+
+                        <Polyline StrokeThickness="0" Grid.Row="0" Margin="10,0,0,0"
+                               VerticalAlignment="Top"
+                               HorizontalAlignment="Left"
+                               Points="0,0,0,10,12,10" Fill="White" />
+
+                        <Border Grid.Row="1" Background="White" Padding="10,5,10,5">
+                            <TextBlock x:Name="cameraGuideText" Foreground="#008272" Text="Step in front of the camera to start!" Style="{StaticResource SubtitleTextBlockStyle}" FontStyle="Italic" />
+                        </Border>
+                    </Grid>
+                </Grid>
+
+            </Grid>
+
+        </Grid>
+    </Grid>
+</Page>

Kiosk/Views/CaptionKioskPage.xaml.cs

@@ -0,0 +1,193 @@
+// 
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license.
+// 
+// Microsoft Cognitive Services: http://www.microsoft.com/cognitive
+// 
+// Microsoft Cognitive Services Github:
+// https://github.com/Microsoft/Cognitive
+// 
+// Copyright (c) Microsoft Corporation
+// All rights reserved.
+// 
+// MIT License:
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to
+// permit persons to whom the Software is furnished to do so, subject to
+// the following conditions:
+// 
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED ""AS IS"", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+// 
+
+using ServiceHelpers;
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Windows.UI.Popups;
+using Windows.UI.Xaml;
+using Windows.UI.Xaml.Controls;
+using Windows.UI.Xaml.Navigation;
+
+namespace IntelligentKioskSample.Views
+{
+    [KioskExperience(Id = "CaptionBot Kiosk",
+        DisplayName = "Caption Bot",
+        Description = "Get a description of the content of a webcam image",
+        ImagePath = "ms-appx:/Assets/DemoGallery/Caption Bot.jpg",
+        ExperienceType = ExperienceType.Automated | ExperienceType.Business,
+        TechnologiesUsed = TechnologyType.Vision,
+        TechnologyArea = TechnologyAreaType.Vision,
+        DateAdded = "2016/06/10")]
+    public sealed partial class CaptionKioskPage : Page
+    {
+        private Task processingLoopTask;
+        private bool isProcessingLoopInProgress;
+        private bool isProcessingPhoto;
+
+        public CaptionKioskPage()
+        {
+            this.InitializeComponent();
+
+            Window.Current.Activated += CurrentWindowActivationStateChanged;
+            this.cameraControl.FilterOutSmallFaces = true;
+            this.cameraControl.CameraAspectRatioChanged += CameraControl_CameraAspectRatioChanged;
+            this.cameraControl.ShowDialogOnApiErrors = SettingsHelper.Instance.ShowDialogOnApiErrors;
+            this.cameraControl.HideCameraControls();
+        }
+
+        private void CameraControl_CameraAspectRatioChanged(object sender, EventArgs e)
+        {
+            this.UpdateCameraHostSize();
+        }
+
+        private async void CurrentWindowActivationStateChanged(object sender, Windows.UI.Core.WindowActivatedEventArgs e)
+        {
+            if ((e.WindowActivationState == Windows.UI.Core.CoreWindowActivationState.CodeActivated ||
+                e.WindowActivationState == Windows.UI.Core.CoreWindowActivationState.PointerActivated) &&
+                this.cameraControl.CameraStreamState == Windows.Media.Devices.CameraStreamState.Shutdown)
+            {
+                // When our Window loses focus due to user interaction Windows shuts it down, so we 
+                // detect here when the window regains focus and trigger a restart of the camera.
+                await this.cameraControl.StartStreamAsync(isForRealTimeProcessing: true);
+            }
+        }
+
+        protected override async void OnNavigatedTo(NavigationEventArgs e)
+        {
+            if (string.IsNullOrEmpty(SettingsHelper.Instance.VisionApiKey))
+            {
+                await new MessageDialog("Missing Vision API Key. Please enter a key in the Settings page.", "Missing Vision API Key").ShowAsync();
+            }
+            else
+            {
+                await this.cameraControl.StartStreamAsync(isForRealTimeProcessing: true);
+                this.StartProcessingLoop();
+            }
+
+            base.OnNavigatedTo(e);
+        }
+
+        protected override async void OnNavigatingFrom(NavigatingCancelEventArgs e)
+        {
+            this.isProcessingLoopInProgress = false;
+
+            Window.Current.Activated -= CurrentWindowActivationStateChanged;
+            this.cameraControl.CameraAspectRatioChanged -= CameraControl_CameraAspectRatioChanged;
+
+            await this.cameraControl.StopStreamAsync();
+            base.OnNavigatingFrom(e);
+        }
+
+        private void StartProcessingLoop()
+        {
+            this.isProcessingLoopInProgress = true;
+
+            if (this.processingLoopTask == null || this.processingLoopTask.Status != TaskStatus.Running)
+            {
+                this.processingLoopTask = Task.Run(() => this.ProcessingLoop());
+            }
+        }
+
+        private async void ProcessingLoop()
+        {
+            while (this.isProcessingLoopInProgress)
+            {
+                await this.Dispatcher.RunAsync(Windows.UI.Core.CoreDispatcherPriority.Normal, async () =>
+                {
+                    if (!this.isProcessingPhoto)
+                    {
+                        this.isProcessingPhoto = true;
+                        if ((DateTime.Now - this.cameraControl.LastFaceTimestamp).TotalSeconds >= 60)
+                        {
+                            await ProcessCameraCapture(null);
+                        }
+                        else
+                        {
+                            await this.ProcessCameraCapture(await this.cameraControl.TakeAutoCapturePhoto());
+                        }
+                    }
+                });
+
+                await Task.Delay(1000);
+            }
+        }
+
+        private async Task ProcessCameraCapture(ImageAnalyzer e)
+        {
+            if (e == null)
+            {
+                this.isProcessingPhoto = false;
+                this.cameraGuideText.Text = "I haven't seen a human face in a while...\r\nIs anybody there? Please wake me up!";
+
+                return;
+            }
+
+            DateTime start = DateTime.Now;
+
+            await Task.WhenAll(e.DescribeAsync(), e.DetectFacesAsync(detectFaceAttributes: true));
+
+            this.debugText.Text = string.Format("Latency: {0}ms", (int)(DateTime.Now - start).TotalMilliseconds);
+
+            string caption = e.AnalysisResult.Description?.Captions?.FirstOrDefault()?.Text;
+            if (!string.IsNullOrEmpty(caption))
+            {
+                if (!SettingsHelper.Instance.ShowAgeAndGender && Util.ContainsGenderRelatedKeyword(caption))
+                {
+                    this.cameraGuideText.Text = "Please enable Age/Gender prediction in the Settings Page to see the results";
+                }
+                else
+                {
+                    this.cameraGuideText.Text = string.Format("I think it's {0}", caption);
+                }
+            }
+            else
+            {
+                this.cameraGuideText.Text = "I am not sure what that is";
+            }
+
+            this.isProcessingPhoto = false;
+        }
+
+        private void OnPageSizeChanged(object sender, SizeChangedEventArgs e)
+        {
+            this.UpdateCameraHostSize();
+        }
+
+        private void UpdateCameraHostSize()
+        {
+            this.cameraHostGrid.Height = this.cameraHostGrid.ActualWidth / (this.cameraControl.CameraAspectRatio != 0 ? this.cameraControl.CameraAspectRatio : 1.777777777777);
+        }
+    }
+}

README.md

@@ -23,6 +23,7 @@ The Intelligent Kiosk Sample is a collection of demos showcasing workflows and e
 | [Anomaly Detector](Documentation/AnomalyDetectorDemo.md)      | Showcases anomaly detection with three pre-selected data streams, as well as a Live Sound scenario that samples the audio level of your microphone as the input data stream. | Anomaly Detector API |
 | [Bing News Analytics](Documentation/BingNewsAnalytics.md)      | Connecting the Bing News APIs with the Text Analytics APIs to create a visualization of the news based on their sentiment and most common topics | Bing News API, Bing AutoSuggestion API, Text Sentiment and Text KeyPhrase Extraction |
 | [Bing Visual Search](Documentation/BingVisualSearch.md)      | Shows how to use the power of Bing Image Insights to perform visual search and find visually similar images, products or celebrities. | Bing Search API;  Bing AutoSuggestion API |
+| [Caption Bot](Documentation/CaptionBot.md)      | Get a description of the content of a webcam image. | Computer Vision API; Windows 10 Face Tracking; |
 | [Custom Vision Explorer](Documentation/CustomVisionExplorer.md)      | Shows how to use the Custom Vision Service to create a custom image classifier or object detector and score images against it. | Custom Vision API, Bing Image Search API;  Bing AutoSuggestion API |
 | [Custom Vision Setup](Documentation/CustomVisionSetup.md)      | Shows how to train a machine learning model using the Custom Vision Service. | Custom Vision API, Bing Image Search API;  Bing AutoSuggestion API |
 | [Face API Explorer](Documentation/FaceAPIExplorer.md)          | A playground for the Face APIs used for extracting face-related attributes, such as head pose, gender, age, emotion, facial hair, and glasses, as well as face identification. | Windows 10 Face Tracking; Face API; Bing Image Search API; Bing AutoSuggestion API |