Node-REDでHoloLensをWebSocket連携して複数オブジェクトを同時操作するメモ
Node-REDでHoloLensをWebSocket連携して複数オブジェクトを同時操作するメモです。
WebSocketのメリットは手軽に双方向通信ができるので複数操作を試しつつ、マテリアル操作のトゥイーンまわりを確認したいということもあって、ためしてみます。
HoloLensアプリ（Unity）とNode-REDで動くWebSocketサーバーをつなぐメモ をベースに発展させていきます。
やりたいこと
複数オブジェクトを同時操作でやりたいことは以下のとおりです。
以前の記事 を元に5つのCubeをWebSocketと連携できるようにしておきます。
まずに全オブジェクト対して、RGBAカラー情報・命令先オブジェクトのJSON命令を送って、全オブジェクトが動くようにします。
先に完成した動画を。タブレット経由でのNode-REDから命令して動かします。
つづいて一部のオブジェクト対して、同様にRGBAカラー情報・命令先オブジェクトのJSON命令を送って、たとえば、CUbe2とCube4だけ動くようにします。
こちらも先に完成した動画を。タブレット経由でのNode-REDから命令して動かします。
では、早速準備していきましょう。
Unityの準備
このように5つCubeが並び、各Cubeに連番で命名されています。
各Cubeには以下のソースコーを割り当ててます。HoloLensアプリ（Unity）とNode-REDで動くWebSocketサーバーをつなぐメモを元にしています。
using UnityEngine;
using System.Collections;
using UnityEngine.Networking;
using System;
using System.Net;
using System.Net.Sockets;
using System.Threading;
using System.Collections.Generic;
using MiniJSON;
#if UNITY_UWP
using Windows.Foundation;
using Windows.Networking.Sockets;
using Windows.Security.Cryptography.Certificates;
using Windows.Storage.Streams;
using System.Threading.Tasks;
#else
using System.Text;
#endif
public class SphereCommands2 : MonoBehaviour
{
Vector3 originalPosition;
Vector3 originalRotation;
Vector3 originalScale;
public bool flagOnOff = false;
#if UNITY_UWP
private MessageWebSocket messageWebSocket;
private DataWriter messageWriter;
private bool busy;
#endif
// Use this for initialization
void Start()
{
// 初期の位置記憶
originalPosition = this.transform.localPosition;
originalRotation = this.transform.localRotation.eulerAngles;
originalScale = this.transform.localScale;
#if UNITY_UWP
// HoloLens実機でWebSocket接続開始
OnConnect();
#endif
}
// Called by GazeGestureManager when the user performs a Select gesture
void OnSelect()
{
if (flagOnOff)
{
flagOnOff = false;
// もとに戻る移動の動作
iTween.MoveTo(this.gameObject, iTween.Hash(
"y", originalPosition.y,
"time", 2f,
"oncomplete", "AnimationEnd",
"oncompletetarget", this.gameObject,
"easeType", "easeInOutBack"
));
// もとに戻る回転の動作
iTween.RotateTo(this.gameObject, iTween.Hash(
"x", originalRotation.x,
"time", 2f
));
// もとに戻る大きさの動作
iTween.ScaleTo(this.gameObject, iTween.Hash(
"x", originalScale.x,
"y", originalScale.y,
"z", originalScale.z,
"time", 2f
));
}
else
{
flagOnOff = true;
var dic = Json.Deserialize("{\"value1\":\"\",\"value2\":2,\"value3\":3}") as Dictionary<string, object>;
dic["value1"] = System.DateTime.Now.ToString();
dic["value2"] = this.gameObject.name;
dic["value3"] = flagOnOff;
var json = Json.Serialize(dic);
Debug.Log(json);
#if UNITY_UWP
Task.Run(async () =>
{
AppendOutputLine("Send JSON : " + json);
UnityEngine.WSA.Application.InvokeOnAppThread(() => {
AppendOutputLine("InvokeOnAppThread : iTween");
// クリック時にふわっと浮く動作
iTween.MoveTo(this.gameObject, iTween.Hash(
"y", originalPosition.y + 0.3,
"time", 2f,
"oncomplete", "AnimationEnd",
"oncompletetarget", this.gameObject,
"easeType", "easeInOutBack"
));
}, true);
await WebSock_SendMessage(messageWebSocket, json);
});
#endif
}
}
// Called by SpeechManager when the user says the "Reset world" command
void OnReset()
{
}
// Called by SpeechManager when the user says the "Drop sphere" command
void OnDrop()
{
}
#if UNITY_UWP
private void OnConnect()
{
AppendOutputLine("OnConnect");
messageWebSocket = new MessageWebSocket();
//In this case we will be sending/receiving a string so we need to set the MessageType to Utf8.
messageWebSocket.Control.MessageType = SocketMessageType.Utf8;
//Add the MessageReceived event handler.
messageWebSocket.MessageReceived += WebSock_MessageReceived;
//Add the Closed event handler.
messageWebSocket.Closed += WebSock_Closed;
Uri serverUri = new Uri("ws://192.168.XX.XX:1880"); // 別PCのNode-REDのWebSocketにつながる
try
{
Task.Run(async () => {
//Connect to the server.
AppendOutputLine("Connect to the server...." + serverUri.ToString());
await Task.Run(async () =>
{
await messageWebSocket.ConnectAsync(serverUri);
AppendOutputLine("ConnectAsync OK");
await WebSock_SendMessage(messageWebSocket, "Connect Start");
});
});
}
catch (Exception ex)
{
AppendOutputLine("error : " + ex.ToString());
//Add code here to handle any exceptions
}
}
private async Task WebSock_SendMessage(MessageWebSocket webSock, string message)
{
AppendOutputLine("WebSock_SendMessage : " + message);
DataWriter messageWriter = new DataWriter(webSock.OutputStream);
messageWriter.WriteString(message);
await messageWriter.StoreAsync();
}
private void WebSock_MessageReceived(MessageWebSocket sender, MessageWebSocketMessageReceivedEventArgs args)
{
DataReader messageReader = args.GetDataReader();
messageReader.UnicodeEncoding = UnicodeEncoding.Utf8;
string messageString = messageReader.ReadString(messageReader.UnconsumedBufferLength);
AppendOutputLine("messageString : " + messageString);
//JSON convert
var dic = Json.Deserialize(messageString) as Dictionary<string, object>;
Debug.Log(dic);
// オブジェクト名が一致したら動作
// allだと全オブジェクト動作
Task.Run(async () =>
{
UnityEngine.WSA.Application.InvokeOnAppThread(() => {
if (dic["obj"].ToString() == this.gameObject.name || dic["obj"].ToString() == "all")
{
AppendOutputLine("InvokeOnAppThread : iTween");
// WebSocket受信時に回転が変わる
iTween.RotateTo(this.gameObject, iTween.Hash(
"x", UnityEngine.Random.Range(1, 5) * 20,
"y", UnityEngine.Random.Range(1, 5) * 20,
"z", UnityEngine.Random.Range(1, 5) * 20,
"time", 2f
));
// WebSocket受信時に大きさが変わる
var scale = UnityEngine.Random.Range(1, 5) * 0.02;
iTween.ScaleTo(this.gameObject, iTween.Hash(
"x", originalScale.x + scale,
"y", originalScale.y + scale,
"z", originalScale.z + scale,
"time", 2f
));
// カラー変更をdoubleで整える
// Parseから再度変換するのは 0.0 , 1.0 のときにInt64に勝手に変わる対策
double a = double.Parse(dic["a"].ToString());
double r = double.Parse(dic["r"].ToString());
double g = double.Parse(dic["g"].ToString());
double b = double.Parse(dic["b"].ToString());
Debug.Log(a);
AppendOutputLine(a.GetType().Name);
// カラー変更
iTween.ColorTo(this.gameObject, iTween.Hash("a", a, "r", r, "g", g, "b", b, "time", 1.0f));
}
}, true);
await Task.Delay(100);
});
}
private void WebSock_Closed(IWebSocket sender, WebSocketClosedEventArgs args)
{
//Add code here to do something when the connection is closed locally or by the server
}
private void AppendOutputLine(string value)
{
// OutputField.Text += value + "\r\n";
Debug.Log(value);
}
#endif
}追加ポイントは、Node-REDの命令の詳細化。JSONによるより細かな命令（オブジェクト名・カラーRGBA命令）と、それに応じたカラー調整トゥイーンColorToを利用しています。
//JSON convert
var dic = Json.Deserialize(messageString) as Dictionary<string, object>;
Debug.Log(dic);
// オブジェクト名が一致したら動作
// allだと全オブジェクト動作
Task.Run(async () =>
{
UnityEngine.WSA.Application.InvokeOnAppThread(() => {
if (dic["obj"].ToString() == this.gameObject.name || dic["obj"].ToString() == "all")
{JSONデータの obj 値を元に「オブジェクト名が一致したら動作」「allだと全オブジェクト動作」を実現してます。
// カラー変更をdoubleで整える
// Parseから再度変換するのは 0.0 , 1.0 のときにInt64に勝手に変わる対策
double a = double.Parse(dic["a"].ToString());
double r = double.Parse(dic["r"].ToString());
double g = double.Parse(dic["g"].ToString());
double b = double.Parse(dic["b"].ToString());
Debug.Log(a);
AppendOutputLine(a.GetType().Name);
// カラー変更
iTween.ColorTo(this.gameObject, iTween.Hash("a", a, "r", r, "g", g, "b", b, "time", 1.0f));こちらがカラー変更の値取得とカラー変更のアニメーション部分です。
iTweenの変化値がdouble値なんですが、miniJSONで1.0や0.0を変換するとInt64の整数値になって弾かれてしまうので、一旦Object.Parseから再度変換して、double値への整理するようにしています。（もっと良いやり方があるかも）
Node-REDの準備
Node-REDのほうの準備は、JSONによるより細かな命令（オブジェクト名・カラーRGBA命令）を送るようにしつつ、全オブジェクト操作、一部のオブジェクト操作を実現します。
このような構成になります。
全オブジェクトのマテリアル操作
all objectは全オブジェクトを変更するものです。
例えばa:0.5 color:redなら50%アルファの赤マテリアルにします。
a:0.5 color:red から a:1 color:blueを操作した状態です。
all resetは100%に戻しつつマテリアルの色もグレーに戻します。
一部のオブジェクト操作のマテリアル操作
multi objectは一部のオブジェクト操作を実現しています。
Cube2 yellow, Cube4 blueはCube2 Cube4 に同時に命令を出しています。
命令の内容は、objeは個別のオブジェクト名で以降のargb値はカラー設定です。
実際にCube2 Cube4の色が変わっています。
Cube1 gray, Cube3 gray, Cube5 grayもCube1 Cube3 Cube5 へ同時に命令を出しています。
aがゼロなので、完全に透明になります。
先ほどの続きですが、動かしてみるとCube1 Cube3 Cube5が透明に変わっています。
こちらのJSONファイルも置いておきます。Node-REDにインポートして試してみてください。
[{"id":"7c0dc9e3.ac18b8","type":"websocket in","z":"dfdb32be.f5a9a","name":"","server":"84ef16d9.c09ac8","client":"","x":523,"y":113,"wires":[["99da5529.8e5358"]]},{"id":"99da5529.8e5358","type":"debug","z":"dfdb32be.f5a9a","name":"","active":true,"console":"false","complete":"false","x":778,"y":161,"wires":[]},{"id":"336baf5.05d3f5","type":"websocket out","z":"dfdb32be.f5a9a","name":"","server":"84ef16d9.c09ac8","client":"","x":793,"y":437,"wires":[]},{"id":"16a315c5.4b26ca","type":"inject","z":"dfdb32be.f5a9a","name":"a:0.5 color:red","topic":"","payload":"{\"obj\":\"all\",\"a\":0.5,\"r\":0.5,\"g\":0,\"b\":0}","payloadType":"json","repeat":"","crontab":"","once":false,"x":572,"y":341,"wires":[["336baf5.05d3f5"]]},{"id":"713266c2.1571e8","type":"inject","z":"dfdb32be.f5a9a","name":"a:0 color:green","topic":"","payload":"{\"obj\":\"all\",\"a\":0,\"r\":0.0,\"g\":0.5,\"b\":0.0}","payloadType":"json","repeat":"","crontab":"","once":false,"x":571,"y":439,"wires":[["336baf5.05d3f5"]]},{"id":"2132c0ed.3afc9","type":"inject","z":"dfdb32be.f5a9a","name":"a:1 color:blue","topic":"","payload":"{\"obj\":\"all\",\"a\":1,\"r\":0.0,\"g\":0,\"b\":0.5}","payloadType":"json","repeat":"","crontab":"","once":false,"x":568,"y":538,"wires":[["336baf5.05d3f5"]]},{"id":"f781bf87.9b3fa","type":"comment","z":"dfdb32be.f5a9a","name":"all object","info":"","x":519,"y":262,"wires":[]},{"id":"303a24c1.bfb24c","type":"comment","z":"dfdb32be.f5a9a","name":"multi object","info":"","x":533,"y":613,"wires":[]},{"id":"9de502fd.edf16","type":"inject","z":"dfdb32be.f5a9a","name":"Cube2,Cube4","topic":"","payload":"{}","payloadType":"json","repeat":"","crontab":"","once":false,"x":571,"y":701,"wires":[["c8b6aa22.41b768","aef8a115.684cb"]]},{"id":"c8b6aa22.41b768","type":"change","z":"dfdb32be.f5a9a","name":"Cube2 yellow","rules":[{"t":"set","p":"payload","pt":"msg","to":"{\"obj\":\"Cube2\",\"a\":1,\"r\":0.5,\"g\":0.5,\"b\":0}","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":786,"y":701,"wires":[["23b952a3.8ac98e"]]},{"id":"23b952a3.8ac98e","type":"websocket out","z":"dfdb32be.f5a9a","name":"","server":"84ef16d9.c09ac8","client":"","x":1012,"y":700,"wires":[]},{"id":"aef8a115.684cb","type":"change","z":"dfdb32be.f5a9a","name":"Cube4 blue","rules":[{"t":"set","p":"payload","pt":"msg","to":"{\"obj\":\"Cube4\",\"a\":1,\"r\":0,\"g\":0,\"b\":0.5}","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":779,"y":765,"wires":[["23b952a3.8ac98e"]]},{"id":"132f5397.b245dc","type":"comment","z":"dfdb32be.f5a9a","name":"all reset","info":"","x":933,"y":260,"wires":[]},{"id":"32979886.fad148","type":"inject","z":"dfdb32be.f5a9a","name":"a:1 color:gray","topic":"","payload":"{\"obj\":\"all\",\"a\":1,\"r\":0.5,\"g\":0.5,\"b\":0.5}","payloadType":"json","repeat":"","crontab":"","once":false,"x":979,"y":325,"wires":[["a57e3846.d6c528"]]},{"id":"a57e3846.d6c528","type":"websocket out","z":"dfdb32be.f5a9a","name":"","server":"84ef16d9.c09ac8","client":"","x":1211,"y":323,"wires":[]},{"id":"2d18a888.f127e8","type":"inject","z":"dfdb32be.f5a9a","name":"Cube1,Cube3,Cube5 alpha","topic":"","payload":"{}","payloadType":"json","repeat":"","crontab":"","once":false,"x":526,"y":821,"wires":[["bfc4757a.083668","9077e805.c4d048","eaf2b5ba.6db848"]]},{"id":"bfc4757a.083668","type":"change","z":"dfdb32be.f5a9a","name":"Cube1 gray","rules":[{"t":"set","p":"payload","pt":"msg","to":"{\"obj\":\"Cube1\",\"a\":0,\"r\":0.5,\"g\":0.5,\"b\":0.5}","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":775,"y":827,"wires":[["4f9aad99.40fc34"]]},{"id":"9077e805.c4d048","type":"change","z":"dfdb32be.f5a9a","name":"Cube3 gray","rules":[{"t":"set","p":"payload","pt":"msg","to":"{\"obj\":\"Cube3\",\"a\":0,\"r\":0.5,\"g\":0.5,\"b\":0.5}","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":778,"y":891,"wires":[["4f9aad99.40fc34"]]},{"id":"eaf2b5ba.6db848","type":"change","z":"dfdb32be.f5a9a","name":"Cube5 gray","rules":[{"t":"set","p":"payload","pt":"msg","to":"{\"obj\":\"Cube5\",\"a\":0,\"r\":0.5,\"g\":0.5,\"b\":0.5}","tot":"str"}],"action":"","property":"","from":"","to":"","reg":false,"x":776,"y":962,"wires":[["4f9aad99.40fc34"]]},{"id":"4f9aad99.40fc34","type":"websocket out","z":"dfdb32be.f5a9a","name":"","server":"84ef16d9.c09ac8","client":"","x":1011,"y":826,"wires":[]},{"id":"84ef16d9.c09ac8","type":"websocket-listener","z":"","path":"/","wholemsg":"false"}]動かしてみる
全オブジェクトのマテリアル操作
このような手順で動かしたものが以下の動画です。
一部のオブジェクト操作のマテリアル操作
このような手順で動かしたものが以下の動画です。
うまくいきました！
WebSocketのメリットを生かしつつマテリアル操作のトゥイーンを複数オブジェクトに適用できました。
今は各オブジェクトが律儀に通信を張る仕様なので、いずれシングルトン実装をしたいですが、Unityでオブジェクトさえ用意すれば、Node-REDからWEBプロトコルでいい感じに調整していけばよいという可能性が見えました。
やっぱり開発しやすいのは得意分野に引き込んでいるのは楽しいですね。
それでは、よきUnity & HoloLens & Node-RED & WebSocket Lifeを。