From 680b1ca2bb49de297ae28b945325cc412ee2c1fe Mon Sep 17 00:00:00 2001
From: BlubbFish <git@blubbfish.net>
Date: Wed, 9 Nov 2016 14:18:10 +0000
Subject: [PATCH]
---
Demo1/Demo1/Demo1.csproj | 6 +-
Demo1/Demo1/DepthFrameConverter.cs | 153 ++++++++++---------
Demo1/Demo1/MainWindow.xaml | 7 +-
Demo1/Demo1/MainWindow.xaml.cs | 189 ++++++++++++++++++------
Demo1/Demo1/SkeletalCommonExtensions.cs | 9 +-
5 files changed, 241 insertions(+), 123 deletions(-)
diff --git a/Demo1/Demo1/Demo1.csproj b/Demo1/Demo1/Demo1.csproj
index 6d7bde2..e49a923 100644
--- a/Demo1/Demo1/Demo1.csproj
+++ b/Demo1/Demo1/Demo1.csproj
@@ -36,9 +36,13 @@
<WarningLevel>4</WarningLevel>
</PropertyGroup>
<ItemGroup>
- <Reference Include="Microsoft.Research.Kinect, Version=1.0.0.45, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL" />
+ <Reference Include="Microsoft.Kinect, Version=1.7.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <SpecificVersion>False</SpecificVersion>
+ <HintPath>C:\Program Files\Microsoft SDKs\Kinect\v1.7\Assemblies\Microsoft.Kinect.dll</HintPath>
+ </Reference>
<Reference Include="System" />
<Reference Include="System.Data" />
+ <Reference Include="System.Drawing" />
<Reference Include="System.Xml" />
<Reference Include="Microsoft.CSharp" />
<Reference Include="System.Core" />
diff --git a/Demo1/Demo1/DepthFrameConverter.cs b/Demo1/Demo1/DepthFrameConverter.cs
index 2bea220..32d3bbb 100644
--- a/Demo1/Demo1/DepthFrameConverter.cs
+++ b/Demo1/Demo1/DepthFrameConverter.cs
@@ -1,11 +1,9 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
+using Microsoft.Kinect;
+using System.Drawing;
namespace KinectWorkshop
{
- public class DepthFrameConverter
+ /*public class DepthFrameConverter
{
const int RED_IDX = 2;
const int GREEN_IDX = 1;
@@ -14,71 +12,94 @@ namespace KinectWorkshop
// Converts a 16-bit grayscale depth frame which includes player indexes into a 32-bit frame
// that displays different players in different colors
- public byte[] ConvertDepthFrameWithUser(byte[] depthFrame16)
+ public byte[] ConvertDepthFrameWithUser(DepthImagePixel[] depthFrame, int width, int height)
{
- for (int i16 = 0, i32 = 0; i16 < depthFrame16.Length && i32 < depthFrame32.Length; i16 += 2, i32 += 4)
+ Color c = Color.Empty;
+ byte[] ret = new byte[width * height * 4];
+ for (int x = 1; x < width; x++)
{
- int player = depthFrame16[i16] & 0x07;
- //if ((y - 1) * 320 + x * 4 == i16)
- //{
- // player = 8;
- //}
- int realDepth = (depthFrame16[i16 + 1] << 5) | (depthFrame16[i16] >> 3);
- // transform 13-bit depth information into an 8-bit intensity appropriate
- // for display (we disregard information in most significant bit)
- byte intensity = (byte)(255 - (255 * realDepth / 0x0fff));
-
- depthFrame32[i32 + RED_IDX] = 0;
- depthFrame32[i32 + GREEN_IDX] = 0;
- depthFrame32[i32 + BLUE_IDX] = 0;
-
- // choose different display colors based on player
- switch (player)
+ for (int y = 0; y < height; y++)
{
- case 0:
- depthFrame32[i32 + RED_IDX] = (byte)(intensity / 2);
- depthFrame32[i32 + GREEN_IDX] = (byte)(intensity / 2);
- depthFrame32[i32 + BLUE_IDX] = (byte)(intensity / 2);
- break;
- case 1:
- depthFrame32[i32 + RED_IDX] = intensity;
- break;
- case 2:
- depthFrame32[i32 + GREEN_IDX] = intensity;
- break;
- case 3:
- depthFrame32[i32 + RED_IDX] = (byte)(intensity / 4);
- depthFrame32[i32 + GREEN_IDX] = (byte)(intensity);
- depthFrame32[i32 + BLUE_IDX] = (byte)(intensity);
- break;
- case 4:
- depthFrame32[i32 + RED_IDX] = (byte)(intensity);
- depthFrame32[i32 + GREEN_IDX] = (byte)(intensity);
- depthFrame32[i32 + BLUE_IDX] = (byte)(intensity / 4);
- break;
- case 5:
- depthFrame32[i32 + RED_IDX] = (byte)(intensity);
- depthFrame32[i32 + GREEN_IDX] = (byte)(intensity / 4);
- depthFrame32[i32 + BLUE_IDX] = (byte)(intensity);
- break;
- case 6:
- depthFrame32[i32 + RED_IDX] = (byte)(intensity / 2);
- depthFrame32[i32 + GREEN_IDX] = (byte)(intensity / 2);
- depthFrame32[i32 + BLUE_IDX] = (byte)(intensity);
- break;
- case 7:
- depthFrame32[i32 + RED_IDX] = (byte)(255 - intensity);
- depthFrame32[i32 + GREEN_IDX] = (byte)(255 - intensity);
- depthFrame32[i32 + BLUE_IDX] = (byte)(255 - intensity);
- break;
- case 8:
- depthFrame32[i32 + RED_IDX] = 255;
- depthFrame32[i32 + GREEN_IDX] = 0;
- depthFrame32[i32 + BLUE_IDX] = 0;
- break;
+ int i = ((y * width) + x) * 4;
+ short depth = (depthFrame[x + y * width]).Depth;
+ c = Color.Empty;
+ if(depth > 0 && depth <= 4000)
+ {
+ int depthdist = (int)((depth / 4090f) * 255f);
+ c = System.Drawing.Color.FromArgb((int)(depthdist / 2f), depthdist, (int)(depthdist * 0.7f));
+ }
+ ret[i] = c.B;
+ ret[i] = c.G;
+ ret[i] = c.R;
+ ret[i] = 255;
}
+
}
- return depthFrame32;
+
+ /* for (int i16 = 0, i32 = 0; i16 < depthFrame16.Length && i32 < depthFrame32.Length; i16 += 2, i32 += 4)
+ {
+ int player = depthFrame16[i16] & 0x07;
+ //if ((y - 1) * 320 + x * 4 == i16)
+ //{
+ // player = 8;
+ //}
+ int realDepth = (depthFrame16[i16 + 1] << 5) | (depthFrame16[i16] >> 3);
+ // transform 13-bit depth information into an 8-bit intensity appropriate
+ // for display (we disregard information in most significant bit)
+ byte intensity = (byte)(255 - (255 * realDepth / 0x0fff));
+
+ depthFrame32[i32 + RED_IDX] = 0;
+ depthFrame32[i32 + GREEN_IDX] = 0;
+ depthFrame32[i32 + BLUE_IDX] = 0;
+
+ // choose different display colors based on player
+ switch (player)
+ {
+ case 0:
+ depthFrame32[i32 + RED_IDX] = (byte)(intensity / 2);
+ depthFrame32[i32 + GREEN_IDX] = (byte)(intensity / 2);
+ depthFrame32[i32 + BLUE_IDX] = (byte)(intensity / 2);
+ break;
+ case 1:
+ depthFrame32[i32 + RED_IDX] = intensity;
+ break;
+ case 2:
+ depthFrame32[i32 + GREEN_IDX] = intensity;
+ break;
+ case 3:
+ depthFrame32[i32 + RED_IDX] = (byte)(intensity / 4);
+ depthFrame32[i32 + GREEN_IDX] = (byte)(intensity);
+ depthFrame32[i32 + BLUE_IDX] = (byte)(intensity);
+ break;
+ case 4:
+ depthFrame32[i32 + RED_IDX] = (byte)(intensity);
+ depthFrame32[i32 + GREEN_IDX] = (byte)(intensity);
+ depthFrame32[i32 + BLUE_IDX] = (byte)(intensity / 4);
+ break;
+ case 5:
+ depthFrame32[i32 + RED_IDX] = (byte)(intensity);
+ depthFrame32[i32 + GREEN_IDX] = (byte)(intensity / 4);
+ depthFrame32[i32 + BLUE_IDX] = (byte)(intensity);
+ break;
+ case 6:
+ depthFrame32[i32 + RED_IDX] = (byte)(intensity / 2);
+ depthFrame32[i32 + GREEN_IDX] = (byte)(intensity / 2);
+ depthFrame32[i32 + BLUE_IDX] = (byte)(intensity);
+ break;
+ case 7:
+ depthFrame32[i32 + RED_IDX] = (byte)(255 - intensity);
+ depthFrame32[i32 + GREEN_IDX] = (byte)(255 - intensity);
+ depthFrame32[i32 + BLUE_IDX] = (byte)(255 - intensity);
+ break;
+ case 8:
+ depthFrame32[i32 + RED_IDX] = 255;
+ depthFrame32[i32 + GREEN_IDX] = 0;
+ depthFrame32[i32 + BLUE_IDX] = 0;
+ break;
+ }
+ }
+ return depthFrame32;*
+ return ret;
}
internal byte[] ConvertDepthFrame(byte[] depthFrame16)
@@ -112,5 +133,5 @@ namespace KinectWorkshop
}
return depthFrame32;
}
- }
+ }*/
}
diff --git a/Demo1/Demo1/MainWindow.xaml b/Demo1/Demo1/MainWindow.xaml
index 71a6b8e..4d70c16 100644
--- a/Demo1/Demo1/MainWindow.xaml
+++ b/Demo1/Demo1/MainWindow.xaml
@@ -4,12 +4,11 @@
Title="MainWindow" Height="520" Width="660" Loaded="Window_Loaded" KeyUp="Window_KeyUp">
<Grid>
<Image Name="imgDepth" Width="640" Height="480" />
- <!--<StackPanel Orientation="Vertical">
+ <!-- <StackPanel Orientation="Vertical">
<Image Name="imgRgb" Width="320" Height="240" />
<Image Name="imgDepthPlayer" Width="320" Height="240" />
- </StackPanel>-->
+ </StackPanel> -->
- <Canvas Name="MainCanvas">
- </Canvas>
+ <Canvas Name="MainCanvas"/>
</Grid>
</Window>
diff --git a/Demo1/Demo1/MainWindow.xaml.cs b/Demo1/Demo1/MainWindow.xaml.cs
index 7bf2cc7..e782f50 100644
--- a/Demo1/Demo1/MainWindow.xaml.cs
+++ b/Demo1/Demo1/MainWindow.xaml.cs
@@ -11,7 +11,7 @@ using System.Windows.Media;
using System.Windows.Media.Imaging;
using System.Windows.Navigation;
using System.Windows.Shapes;
-using Microsoft.Research.Kinect.Nui;
+using Microsoft.Kinect;
namespace Demo1
{
@@ -21,7 +21,14 @@ namespace Demo1
public partial class MainWindow : Window
{
private KinectWorkshop.DepthFrameConverter cv = new KinectWorkshop.DepthFrameConverter();
- private Camera _cam;
+ private KinectSensor r;
+ //private DepthImagePixel[] depthPixels;
+ //private byte[] colorPixels;
+ //private DepthImageFormat DepthFormat = DepthImageFormat.Resolution320x240Fps30;
+ //private ColorImageFormat ColorFormat = ColorImageFormat.RgbResolution640x480Fps30;
+ //private ColorImagePoint[] colorCoordinates;
+ //private bool init = false;
+
public MainWindow()
{
InitializeComponent();
@@ -29,24 +36,29 @@ namespace Demo1
private void Window_Loaded(object sender, RoutedEventArgs e)
{
- if (Runtime.Kinects.Count == 0)
+ if (KinectSensor.KinectSensors.Count == 0)
{
MessageBox.Show("Keine Kinect");
return;
}
try
{
- Runtime r = Runtime.Kinects[0];
- r.Initialize(RuntimeOptions.UseSkeletalTracking | RuntimeOptions.UseDepthAndPlayerIndex);
+ r = KinectSensor.KinectSensors[0];
+ r.Start();
+ //r.Initialize(RuntimeOptions.UseSkeletalTracking | RuntimeOptions.UseDepthAndPlayerIndex);
//r.VideoFrameReady += new EventHandler<ImageFrameReadyEventArgs>(r_VideoFrameReady);
//r.VideoStream.Open(ImageStreamType.Video, 2, ImageResolution.Resolution640x480, ImageType.Color);
-
- r.DepthFrameReady += new EventHandler<ImageFrameReadyEventArgs>(r_DepthFrameReady);
- r.DepthStream.Open(ImageStreamType.Depth, 2, ImageResolution.Resolution320x240, ImageType.DepthAndPlayerIndex);
- r.SkeletonEngine.TransformSmooth = true;
- r.SkeletonFrameReady += new EventHandler<SkeletonFrameReadyEventArgs>(r_SkeletonFrameReady);
- _cam = r.NuiCamera;
+ //r.DepthFrameReady += r_DepthFrameReady;
+ //r.DepthStream.Enable();
+ //r.DepthStream.Open(ImageStreamType.Depth, 2, ImageResolution.Resolution320x240, ImageType.DepthAndPlayerIndex);
+ //r.SkeletonFrameReady += r_SkeletonFrameReady;
+ r.SkeletonStream.Enable(new TransformSmoothParameters() { Smoothing = 0.1f });
+ //r.SkeletonEngine.TransformSmooth = true;
+ //r.SkeletonFrameReady += new EventHandler<SkeletonFrameReadyEventArgs>(r_SkeletonFrameReady);
+ //_cam = r.NuiCamera;
+ //r.AllFramesReady += R_AllFramesReady;
+ r.SkeletonFrameReady += this.r_SkeletonFrameReady;
}
catch (Exception ex)
{
@@ -54,46 +66,118 @@ namespace Demo1
}
}
+ /*private void R_AllFramesReady(object sender, AllFramesReadyEventArgs e)
+ {
+ if(r == null)
+ {
+ return;
+ }
+ bool depthReceived = false;
+ bool colorReceived = false;
+
+ using (DepthImageFrame depthFrame = e.OpenDepthImageFrame())
+ {
+ if (null != depthFrame)
+ {
+ // Copy the pixel data from the image to a temporary array
+ depthFrame.CopyDepthImagePixelDataTo(this.depthPixels);
+
+ depthReceived = true;
+ if(!init) {
+ this.colorCoordinates = new ColorImagePoint[r.DepthStream.FramePixelDataLength];
+ init = true;
+ }
+ }
+ }
+
+ using (ColorImageFrame colorFrame = e.OpenColorImageFrame())
+ {
+ if (null != colorFrame)
+ {
+ // Copy the pixel data from the image to a temporary array
+ colorFrame.CopyPixelDataTo(this.colorPixels);
+
+ colorReceived = true;
+ }
+ }
+
+ if (true == depthReceived)
+ {
+ r.CoordinateMapper.MapDepthFrameToColorFrame(
+ DepthFormat,
+ this.depthPixels,
+ ColorFormat,
+ this.colorCoordinates);
+
+ // ...
+
+ int depthIndex = x + (y * this.depthWidth);
+ DepthImagePixel depthPixel = this.depthPixels[depthIndex];
+
+ // scale color coordinates to depth resolution
+ int X = colorImagePoint.X / this.colorToDepthDivisor;
+ int Y = colorImagePoint.Y / this.colorToDepthDivisor;
+
+ // depthPixel is the depth for the (X,Y) pixel in the color frame
+ }
+
+ }*/
+
void r_SkeletonFrameReady(object sender, SkeletonFrameReadyEventArgs e)
{
+ SkeletonFrame skf = e.OpenSkeletonFrame();
+ if(skf == null)
+ {
+ return;
+ }
+ Skeleton[] sks = new Skeleton[skf.SkeletonArrayLength];
+ skf.CopySkeletonDataTo(sks);
this.MainCanvas.Children.Clear();
- foreach (SkeletonData sk in e.SkeletonFrame.Skeletons)
+ foreach (Skeleton sk in sks)
{
if (sk.TrackingState == SkeletonTrackingState.Tracked)
{
- this.SetLines(sk);
- this.SetPoints(sk);
+ this.drawPlayer(sk);
+
}
}
}
- private void SetLines(SkeletonData sk)
+ private void drawPlayer(Skeleton sk)
{
- JointID[,] id = {
- { JointID.Head, JointID.ShoulderCenter },
- { JointID.ShoulderCenter, JointID.ShoulderLeft },
- { JointID.ShoulderCenter, JointID.ShoulderRight },
- { JointID.ShoulderLeft, JointID.ElbowLeft },
- { JointID.ShoulderRight, JointID.ElbowRight },
- { JointID.ElbowLeft, JointID.WristLeft },
- { JointID.ElbowRight, JointID.WristRight },
- { JointID.WristLeft, JointID.HandLeft },
- { JointID.WristRight, JointID.HandRight },
- { JointID.ShoulderCenter, JointID.Spine },
- { JointID.Spine, JointID.HipCenter },
- { JointID.HipCenter, JointID.HipLeft },
- { JointID.HipCenter, JointID.HipRight },
- { JointID.HipLeft, JointID.KneeLeft },
- { JointID.HipRight, JointID.KneeRight },
- { JointID.KneeLeft, JointID.AnkleLeft },
- { JointID.KneeRight, JointID.AnkleRight },
- { JointID.AnkleLeft, JointID.FootLeft },
- { JointID.AnkleRight, JointID.FootRight }
+ this.SetPoints(sk);
+ this.SetLines(sk);
+ }
+
+ private void SetLines(Skeleton sk)
+ {
+ JointType[,] id = {
+ { JointType.Head, JointType.ShoulderCenter },
+ { JointType.ShoulderCenter, JointType.ShoulderLeft },
+ { JointType.ShoulderCenter, JointType.ShoulderRight },
+ { JointType.ShoulderLeft, JointType.ElbowLeft },
+ { JointType.ShoulderRight, JointType.ElbowRight },
+ { JointType.ElbowLeft, JointType.WristLeft },
+ { JointType.ElbowRight, JointType.WristRight },
+ { JointType.WristLeft, JointType.HandLeft },
+ { JointType.WristRight, JointType.HandRight },
+ { JointType.ShoulderCenter, JointType.Spine },
+ { JointType.Spine, JointType.HipCenter },
+ { JointType.HipCenter, JointType.HipLeft },
+ { JointType.HipCenter, JointType.HipRight },
+ { JointType.HipLeft, JointType.KneeLeft },
+ { JointType.HipRight, JointType.KneeRight },
+ { JointType.KneeLeft, JointType.AnkleLeft },
+ { JointType.KneeRight, JointType.AnkleRight },
+ { JointType.AnkleLeft, JointType.FootLeft },
+ { JointType.AnkleRight, JointType.FootRight }
};
for (int i = 0; i < id.Length / 2; i++)
{
Joint r1 = sk.Joints[id[i, 0]].ScaleTo(640, 480);
Joint r2 = sk.Joints[id[i, 1]].ScaleTo(640, 480);
+
+ /**** */
Line l = new Line();
l.Stroke = Brushes.Blue;
l.StrokeThickness = 5.0;
@@ -101,11 +185,13 @@ namespace Demo1
l.X1 = r1.Position.X;
l.Y2 = r2.Position.Y;
l.X2 = r2.Position.X;
+ /**** */
+
this.MainCanvas.Children.Add(l);
}
}
- private void SetPoints(SkeletonData sk)
+ private void SetPoints(Skeleton sk)
{
foreach (Joint joint in sk.Joints)
{
@@ -121,28 +207,37 @@ namespace Demo1
}
}
- void r_DepthFrameReady(object sender, ImageFrameReadyEventArgs e)
+ /*void r_DepthFrameReady(object sender, DepthImageFrameReadyEventArgs e)
{
- PlanarImage img = e.ImageFrame.Image;
- byte[] bh = cv.ConvertDepthFrameWithUser(img.Bits);
- imgDepth.Source = BitmapSource.Create(img.Width, img.Height,
- 96, 96, PixelFormats.Bgr32, null,
- bh, img.Width * 4);
+ //PlanarImage img = e.ImageFrame.Image;
+ DepthImageFrame img = e.OpenDepthImageFrame();
+ if(img == null)
+ {
+ return;
+ }
+ DepthImagePixel[] bs = new DepthImagePixel[img.PixelDataLength];
+ img.CopyDepthImagePixelDataTo(bs);
+ byte[] bh = cv.ConvertDepthFrameWithUser(bs, img.Width, img.Height);
+ imgDepth.Source = BitmapSource.Create(img.Width, img.Height, 96, 96, PixelFormats.Bgr32, null, bh, img.Width * 4);
//bh = cv.ConvertDepthFrameHiddenPlayer(img.Bits);
//byte[] bh = cv.ConvertDepthFrame(img.Bits);
//imgDepthPlayer.Source = BitmapSource.Create(img.Width, img.Height,
// 96, 96, PixelFormats.Bgr32, null,
// bh, img.Width * 4);
- }
+ }*/
private void Window_KeyUp(object sender, KeyEventArgs e)
{
try
{
- if (e.Key == Key.OemPlus)
- _cam.ElevationAngle += 1;
- if (e.Key == Key.OemMinus)
- _cam.ElevationAngle -= 1;
+ if (e.Key == Key.Up)
+ {
+ int k = r.ElevationAngle;
+ k++;
+ r.ElevationAngle = k;
+ }
+ if (e.Key == Key.Down)
+ r.ElevationAngle -= 1;
}
catch (ArgumentOutOfRangeException outOfRangeException)
{
diff --git a/Demo1/Demo1/SkeletalCommonExtensions.cs b/Demo1/Demo1/SkeletalCommonExtensions.cs
index 2f5b2b3..445f250 100644
--- a/Demo1/Demo1/SkeletalCommonExtensions.cs
+++ b/Demo1/Demo1/SkeletalCommonExtensions.cs
@@ -1,4 +1,5 @@
-using Microsoft.Research.Kinect.Nui;
+using Microsoft.Kinect;
+using System.Windows;
namespace Demo1
{
@@ -6,17 +7,15 @@ namespace Demo1
{
public static Joint ScaleTo(this Joint joint, int width, int height, float skeletonMaxX, float skeletonMaxY)
{
- Vector pos = new Vector()
+ SkeletonPoint pos = new SkeletonPoint()
{
X = Scale(width, skeletonMaxX, joint.Position.X),
Y = Scale(height, skeletonMaxY, -joint.Position.Y),
- Z = joint.Position.Z,
- W = joint.Position.W
+ Z = joint.Position.Z
};
Joint j = new Joint()
{
- ID = joint.ID,
TrackingState = joint.TrackingState,
Position = pos
};