Skype Calling Through Kinect Gesture Recognition

 

Gone are the days when people preferred to tap the keys for sending commands to their computers. Today’s is the time of touch technology and future is gesture sensing technology. Gesture sensing technologies involve extensive combination of hardware and software application to come up with outstanding results. Gesture sensing is pacing up widely to be used for touchless computer operations.

This blog shows how this latest technology can be used to make Skype calls with ease using Kinect. This app can be used to make Skype calls to predefined users with just wave of hand.

Create a windows application. Connect the Kinect device to your system and use below code snippet to set up the Kinect with your application.

Add Microsoft.Kinect.dll and SKYPE4COMLib.dll to your application.

 using System;
 using System.Drawing;
 using System.Windows.Forms;
 using System.Drawing.Imaging;
 using System.Linq;
 using System.Drawing.Drawing2D;
 using System.Collections.Generic;
 using Microsoft.Kinect;
 using SKYPE4COMLib;

 public Form1()
        {
            InitializeComponent();
            SetupKinect();
        }

        private void SetupKinect()
        {
            if (_isInit)
                StopKinect();

            if (KinectSensor.KinectSensors.Count > 0)
            {
                //pull the first Kinect
                _sensor = KinectSensor.KinectSensors[0];
            }
            if (_sensor.Status != KinectStatus.Connected || KinectSensor.KinectSensors.Count == 0)
            {
                MessageBox.Show("No Kinect connected");
                return;
            }

            _sensor.SkeletonStream.Enable();
            _sensor.DepthStream.Enable(DepthImageFormat.Resolution320x240Fps30);
            _sensor.ColorStream.Enable(ColorImageFormat.RgbResolution640x480Fps30);
            _sensor.AllFramesReady += new EventHandler<AllFramesReadyEventArgs>(_sensor_AllFramesReady);

            _sensor.Start();
            _isInit = true;
        }

 void _sensor_AllFramesReady(object sender, AllFramesReadyEventArgs e)
        {
            RuntimeColorFrameReady(e); 
            Sensor_SkeletonFrameReady(e);
        }

        void RuntimeColorFrameReady(AllFramesReadyEventArgs e)
        {
            using (ColorImageFrame colorFrame = e.OpenColorImageFrame())
            {
                if (colorFrame == null)
                {
                    return;
                }

                ColorImage.Image = colorFrame.ToBitmap();

                if (_saveColorFrame)
                {
                    _saveColorFrame = false;
                    colorFrame.ToBitmap().Save(DateTime.Now.ToString("yyyyMMddHHmmss") + "_color.jpg",
                    ImageFormat.Jpeg);
                }
            }
        }

        public void StopKinect()
        {
            if (_sensor != null)
            {
                _sensor.Stop();
            }

            _isInit = false;
        }

        private void ReinitRuntime_Click(object sender, EventArgs e)
        {
            SetupKinect();
        }

        private void Form1_FormClosing(object sender, FormClosingEventArgs e)
        {
            StopKinect();
        }

Now to track the hand gesture below event handler will be called each time the gesture is changed. First of all it creates the skeleton of the object(the human body) in front of the Kinect device.

 void Sensor_SkeletonFrameReady(AllFramesReadyEventArgs e)
        {
            using (var frame = e.OpenSkeletonFrame())
            {
                if (frame != null)
                {
                    Skeleton[] skeletons = new Skeleton[frame.SkeletonArrayLength];

                    frame.CopySkeletonDataTo(skeletons);

                    var skeleton = skeletons.Where(s => s.TrackingState == 
                    SkeletonTrackingState.Tracked).FirstOrDefault();

                    if (skeleton != null)
                    {
                        //call the function to check the hand position                       
                        TrackHand(skeleton);
                    }
                }
            }
        }

 

The below function will now track the joints of the passed skeleton and then compares the coordinates of the right hand with respect to the head and shoulder.

 public bool TrackHand(Skeleton skeleton)
        {
            Joint rightHand = skeleton.Joints[JointType.HandRight];
            Joint leftHand = skeleton.Joints[JointType.HandLeft];
            Joint head = skeleton.Joints[JointType.Head];
            Joint rightShoulder = skeleton.Joints[JointType.ShoulderRight];

            string userName;

            if (rightHand.Position.Y > head.Position.Y) //if right hand is above head
            {
                userName = "david.skypeID";
                MessageBox.Show("Calling " + userName);
                CallSkype(userName);
                return true;
            }
            else
                if (leftHand.Position.Y > head.Position.Y) //if left hand is above head
                {
                    userName = "tom.skypeID";
                    MessageBox.Show("Calling " + userName);
                    CallSkype(userName);
                    return true;
                }
                else
                    if (rightHand.Position.X > rightShoulder.Position.X) //if right hand is in right 
                    direction to the shoulder
                    {
                        userName = "steve.skypeID";
                        MessageBox.Show("Calling " + userName);
                        CallSkype(userName);
                        return true;
                    }
                    else
                        if (rightHand.Position.X < rightShoulder.Position.X) //if right hand is in 
                        right direction to the shoulder
                        {
                            userName = "andy.skypeID";
                            MessageBox.Show("Calling " + userName);
                            CallSkype(userName);
                            return true;
                        }

            return false;
        }

 

Based on different gestures of hand as mentioned in above method the current user can call to different Skype users.

The CallSkype() function is actually used here to make the call based on the online status of users, and prompts accordingly.

/**************************Skype***********************/

        public void CallSkype(string userNameToCall)
        {
            Call call = new Call();
            try
            {
                Start();

                // wait for the client to be connected and ready 
  skype.Attach(7, true); // 7 = protocol

                if (skype.CurrentUserStatus == TUserStatus.cusOnline) // check if calling user
                is online
                {
                    //check if user is online
                    User user = new User(skype.SearchForUsers(userNameToCall));
                    if (user.OnlineStatus == TOnlineStatus.olsOnline)
                    {
                        call = skype.PlaceCall(userNameToCall);
                        if (call.Status != TCallStatus.clsInProgress)
                        {
                            //some message
                            MessageBox.Show(call.Status.ToString());
                        }
                        MessageBox.Show(call.Status.ToString());
                    }
                    else
                    {
                        MessageBox.Show(userNameToCall + " is " + call.Status.ToString());
                    }
                }
                else
                {
                    MessageBox.Show("Please get online to Skype.");
                }
                Shutdown();
            }
            catch (Exception ex)
            {
                MessageBox.Show("Error: " + ex.Message);
            }
        }

        public void Start()
        {
            if (!skype.Client.IsRunning)
            {
                skype.Client.Start(true, true);
            }
        }

        public void Shutdown()
        {
            if (skype.Client.IsRunning)
            {
                skype.Client.Shutdown();
            }
        }
        /*************************Skype Ends**********************************/

Leave a comment