I'm adding a feature to a project that will allow users to see a heat map representation of their mouse movements on the screen in real time. My goal is to make this API as dynamic as possible.
By dynamic, I mean I would like users to be able to use this API to generate the heat map in real time and then plug that heat map into their third party graphics software to view that heat map. (i.e Unity, React, Mobile, etc.. )
For testing purposes the third party graphics software that I am using is Unity. I have created a .cs unity script that does the following per frame:
- Start recording mouse locations.
- Have ExternalProgram.exe generate a bitmap image using the points that unity just recorded.
- Then have Unity display the updated .bmp image on the screen.
Right now the problem that I am having is that the .bmp file is not being created when I use ProcessStartInfo in my unity script to run the .exe that is in charge of creating the .bmp image.
I've been debugging this code for the past week trying to figure out what is wrong with it. I know for a fact that Unity is successfully recording the mouse's location and passing those values to the .exe after calling ProcessStartInfo.
But for some reason it doesn't actually create the .bmp file. This is weird because if I independently run the ExternalProject in Visual Studio then it works just fine and it creates the .bmp file and shows the correct heat map representation on the image.
I figured that maybe starting a program and passing it tons of data and having that program create a file would be a lot of work for unity to do every single frame. (I am open to suggestions on ways to get around having to do that) So I decided to just have the script record points for the first 15 seconds and then try to write that data to the .bmp file but that didn't work either.
Main program file for ExternalProject.exe
class Program
{
public static void Main(string[] args)
{
string arguments = "";
foreach (string arg in args)
{
arguments += arg;
}
Console.WriteLine("My Args: " + arguments + "--EOF");
bool noArguments = String.IsNullOrEmpty(arguments);
if (noArguments)
{
// Test input data
arguments = "(111,222)|(333,444)|(555,777)|(888,999)|(1000,1000)|(1000,1000)|(1000,1000)";
}
// The method ConvertStringToSignalsList() has already been tested and it works.
List<MouseMovementSignal> signals = ConvertStringToSignalsList(arguments);
CreateMouseHeatmap(signals);
Console.WriteLine("Press Enter to close...");
Console.ReadLine();
}
private static void CreateMouseHeatmap(List<MouseMovementSignal> argumentSignals)
{
int Height = 2100;
int Width = 3800;
List<MouseMovementSignal> mouseSignals
= argumentSignals; // Program perameters
//= getRecordedMouseData(); // DB
//= createFakeMouseSignals(Height, Width); // Generates fake signals
try
{
HeatmapStructure<MouseMovementSignal> mapper =
new HeatmapStructure<MouseMovementSignal>(Height, Width);
mapper.LoadSignalsFromObjects(mouseSignals);
// .BMP Image is created inside of this method !!
mapper.IdentifyHeatRegions();
//MouseMovementSignal mms = argumentSignals[argumentSignals.Count - 1];
//Console.WriteLine("Last: " + mms.Channels[0].Name + ": " + mms.Channels[0].Values[0] + ", "
// + mms.Channels[1].Name + ": " + mms.Channels[1].Values[0]);
//finalHeatmap.Save("MyFirstBitmap.bmp", System.Drawing.Imaging.ImageFormat.Bmp);
Console.WriteLine("Image actually Complete!!!!!!!!!!!!!!");
}
catch (Exception e)
{
Console.WriteLine("Error: " + e);
}
}
}
My Unity Script
public class HeatmapScript : MonoBehaviour {
private List<string> points;
private Camera cam;
private RawImage ri;
private string heatmapBmpPath = @"C:\Users\not-showing-you-my-file-structure-LOL\MyFirstBitmap.bmp";
private int frameCount = 0;
string pointsAsString = "";
Stopwatch sw;
// Use this for initialization
void Start() {
// Initialize the raw image.
cam = Camera.main;
GameObject imageObject = GameObject.FindGameObjectWithTag("imageView");
ri = imageObject.GetComponent<RawImage>();
// Initialize points list.
points = new List<string>();
sw = new Stopwatch();
sw.Start();
}
bool stop = false;
// Update is called once per frame.
void Update() {
float xValue = Input.mousePosition.x;
float yValue = Input.mousePosition.y;
Vector2 newPoint = new Vector2(xValue, yValue);
points.Add(newPoint.ToString());
int tSecs = 15000;// 15 seconds
// After 15 seconds of recording points pass them to the program that creates the heat map.
if (stop == false && sw.ElapsedMilliseconds > tSecs)
{
StartCoroutine("UpdateBMP");
UnityEngine.Debug.Log(points.Count);
stop = true;
}
//Update the raw image on the screen.
UnityEngine.Texture2D newTexture = CreateTextureFromBitmap(heatmapBmpPath);
//Set the RawImage to the size of the scren.
ri.texture = newTexture;
ri.rectTransform.sizeDelta = new Vector2(Screen.width, Screen.height);
frameCount++;
}
IEnumerator UpdateBMP()
{
// Show mouse position in unity environment
float xValue = Input.mousePosition.x;
float yValue = Input.mousePosition.y;
Vector2 newPoint = new Vector2(xValue, yValue);
points.Add(newPoint.ToString());
// EX:
// (123,123)|(123,123)|(123,123)|(123,123)|(123,123)|(123,123)
// display list contents without loop
pointsAsString = string.Join("|", points.ToArray());
// Every frame call Behavior's Program.cs that calls HeatmapStructure.cs to update .bmp file
ProcessStartInfo processInfo = new ProcessStartInfo();
processInfo.FileName = @"C:\Users\not-showing-you-my-file-structure-LOL\ExternalProgram.exe";
processInfo.UseShellExecute = false;
processInfo.Arguments = pointsAsString;
Process process = Process.Start(processInfo);
yield return null;
}
private UnityEngine.Texture2D CreateTextureFromBitmap(string completeFilePath)
{
BMPLoader loader = new BMPLoader();
BMPImage img = loader.LoadBMP(completeFilePath);
UnityEngine.Texture2D myTexture = img.ToTexture2D();
UnityEngine.Debug.Log("File Size: " + img.header.filesize);
return myTexture;
}
}
HeatmapStructure.cs class
public class HeatmapStructure<T> where T : ISignal
{
public class COGPoint
{
public double X, Y, Z;
//public Color Color;
public byte Intensity;
public bool isD3Point = false; // 3D Point check
public const double DEFAULT_AXIS_LOC = 0.0001;
public COGPoint()
{
//Color = Color.Blue;
Intensity = 0;
}
// NOTE: double z has a default value therefore it is optional
public COGPoint(byte intensity, double x, double y, double z = DEFAULT_AXIS_LOC)
{
this.X = x;
this.Y = y;
this.Z = z; // Optional
//Color = Color.Blue; // Cold: Blue / Hot: Red
this.Intensity = intensity;
if (z != DEFAULT_AXIS_LOC)
{
isD3Point = true;
}
}
public override string ToString()
{
string output = (isD3Point == true) ?
("(x,y,z) " + X + "," + Y + "," + Z) : ("(x,y) " + X + "," + Y); // 3D : 2D
output += //" Color: " + Color.ToString() +
" Intensity: " + Intensity;
return output;
}
}
private List<COGPoint> points;
private int Height;
private int Width;
public HeatmapStructure(int Height, int Width)
{
this.Height = Height;
this.Width = Width;
points = new List<COGPoint>();
}
private Bitmap CreateIntensityMask(Bitmap bSurface, List<COGPoint> aHeatPoints)
{
// Create new graphics surface from memory bitmap
Graphics DrawSurface = Graphics.FromImage(bSurface);
// Set background color to white so that pixels can be correctly colorized
DrawSurface.Clear(Color.White);
// Traverse heat point data and draw masks for each heat point
foreach (COGPoint DataPoint in aHeatPoints)
{
// Render current heat point on draw surface
DrawHeatPoint(DrawSurface, DataPoint, 45);
}
return bSurface;
}
// TODO: How to draw updating Bitmap in unity in real time ??
private void DrawHeatPoint(Graphics Canvas, COGPoint HeatPoint, int Radius)
{
// Create points generic list of points to hold circumference points
List<Point> CircumferencePointsList = new List<Point>();
// Create an empty point to predefine the point struct used in the circumference loop
Point CircumferencePoint;
// Create an empty array that will be populated with points from the generic list
Point[] CircumferencePointsArray;
// Calculate ratio to scale byte intensity range from 0-255 to 0-1
float fRatio = 1F / Byte.MaxValue;
// Precalulate half of byte max value
byte bHalf = Byte.MaxValue / 2;
// Flip intensity on it's center value from low-high to high-low
int iIntensity = (byte)(HeatPoint.Intensity - ((HeatPoint.Intensity - bHalf) * 2));
// Store scaled and flipped intensity value for use with gradient center location
float fIntensity = iIntensity * fRatio;
// Loop through all angles of a circle
// Define loop variable as a double to prevent casting in each iteration
// Iterate through loop on 10 degree deltas, this can change to improve performance
for (double i = 0; i <= 360; i += 10)
{
// Replace last iteration point with new empty point struct
CircumferencePoint = new Point();
// Plot new point on the circumference of a circle of the defined radius
// Using the point coordinates, radius, and angle
// Calculate the position of this iterations point on the circle
CircumferencePoint.X = Convert.ToInt32(HeatPoint.X + Radius * Math.Cos(ConvertDegreesToRadians(i)));
CircumferencePoint.Y = Convert.ToInt32(HeatPoint.Y + Radius * Math.Sin(ConvertDegreesToRadians(i)));
// Add newly plotted circumference point to generic point list
CircumferencePointsList.Add(CircumferencePoint);
}
// Populate empty points system array from generic points array list
// Do this to satisfy the datatype of the PathGradientBrush and FillPolygon methods
CircumferencePointsArray = CircumferencePointsList.ToArray();
// Create new PathGradientBrush to create a radial gradient using the circumference points
PathGradientBrush GradientShaper = new PathGradientBrush(CircumferencePointsArray);
// Create new color blend to tell the PathGradientBrush what colors to use and where to put them
ColorBlend GradientSpecifications = new ColorBlend(3);
// Define positions of gradient colors, use intesity to adjust the middle color to
// show more mask or less mask
GradientSpecifications.Positions = new float[3] { 0, fIntensity, 1 };
// Define gradient colors and their alpha values, adjust alpha of gradient colors to match intensity
GradientSpecifications.Colors = new Color[3]
{
Color.FromArgb(0, Color.White),
Color.FromArgb(HeatPoint.Intensity, Color.Black),
Color.FromArgb(HeatPoint.Intensity, Color.Black)
};
// Pass off color blend to PathGradientBrush to instruct it how to generate the gradient
GradientShaper.InterpolationColors = GradientSpecifications;
// Draw polygon (circle) using our point array and gradient brush
Canvas.FillPolygon(GradientShaper, CircumferencePointsArray);
}
private double ConvertDegreesToRadians(double degrees)
{
double radians = (Math.PI / 180) * degrees;
return (radians);
}
// old name : button1_Click
public Bitmap IdentifyHeatRegions()
{
// Create new memory bitmap the same size as the picture box
Bitmap bMap = new Bitmap(Width, Height);
// Call CreateIntensityMask, give it the memory bitmap, and use it's output to set the picture box image
Bitmap bm = CreateIntensityMask(bMap, points);
Bitmap coloredBitmap = Colorize(bm, 243); // <-- NOTE: should be 255. But my palette.bmp is 243x5
coloredBitmap.Save("MyFirstBitmap.bmp", System.Drawing.Imaging.ImageFormat.Bmp);
return coloredBitmap;
}
public static Bitmap Colorize(Bitmap Mask, byte Alpha)
{
// Create new bitmap to act as a work surface for the colorization process
Bitmap Output = new Bitmap(Mask.Width, Mask.Height, PixelFormat.Format32bppArgb);
// Create a graphics object from our memory bitmap so we can draw on it and clear it's drawing surface
Graphics Surface = Graphics.FromImage(Output);
Surface.Clear(Color.Transparent);
// Build an array of color mappings to remap our greyscale mask to full color
// Accept an alpha byte to specify the transparancy of the output image
ColorMap[] Colors = CreatePaletteIndex(Alpha);
// Create new image attributes class to handle the color remappings
// Inject our color map array to instruct the image attributes class how to do the colorization
ImageAttributes Remapper = new ImageAttributes();
try
{
Remapper.SetRemapTable(Colors);
}
catch (Exception e)
{
Console.WriteLine(e);
}
// Draw our mask onto our memory bitmap work surface using the new color mapping scheme
Surface.DrawImage(Mask, new Rectangle(0, 0, Mask.Width, Mask.Height), 0, 0, Mask.Width, Mask.Height, GraphicsUnit.Pixel, Remapper);
// Send back newly colorized memory bitmap
return Output;
}
private static ColorMap[] CreatePaletteIndex(byte Alpha)
{
ColorMap[] OutputMap = new ColorMap[Alpha + 1];
// Change this path to wherever you saved the palette image.
Bitmap Palette = (Bitmap)Bitmap.FromFile(@"C:\Users\cdowns\Desktop\palette.bmp");
// Loop through each pixel and create a new color mapping
try
{
for (int X = 0; X <= Alpha; X++)
{
OutputMap[X] = new ColorMap();
OutputMap[X].OldColor = Color.FromArgb(X, X, X);
OutputMap[X].NewColor = Color.FromArgb(Alpha, Palette.GetPixel(X, 0));
}
}
catch (Exception e) {
Console.WriteLine(e);
}
return OutputMap;
}
public void LoadSignalsFromObjects(List<T> allSignals) // ISignal Object
{
Random random = new Random();
foreach (T signal in allSignals)
{
COGPoint newPoint = new COGPoint();
if (allSignals[0].GetType() == typeof(MouseMovementSignal))
{
string axis1 = signal.Channels[0].Name;
List<double> value1 = signal.Channels[0].Values;
string axis2 = signal.Channels[1].Name;
List<double> value2 = signal.Channels[1].Values;
// Make sure to enter signals into database in this format
//Console.WriteLine(axis1 + " " + axis2);
newPoint.X = value1[0];
newPoint.Y = value2[0];
// TOOD: Implement
newPoint.Intensity = (byte)random.Next(0, 120);
// Display newPoint values
//Console.WriteLine("COGPoint Numbers: X: " + newPoint.X + " , Y: " + newPoint.Y
// + /*" Color: " + newPoint.Color + */" Intensity: " + newPoint.Intensity);
}
else if (allSignals[0].GetType() == typeof(EyePosition))
{
}
// Add to main list of heat points
points.Add(newPoint);
}
}
}
Expected result is that the .bmp image is created after the first 15 seconds.
(P.S. I am very new to both Unity and C# to I am probably doing this completely wrong. I am open to an entirely new idea for going about making this work. Thanks)
