using System; using System.Collections; using System.Text; using System.Collections.Generic; using System.IO; using System.Net.Sockets; using UnityEngine; using System.Threading; namespace UnityEditor.Build.CacheServer { /// /// Options for the type of a particular file. /// public enum FileType { /// /// Use to indicate that the file is an asset. /// Asset = 'a', /// /// Use to indicate that the file holds information for an asset/resource. /// Info = 'i', /// /// Use to indicate that the file is a resource. /// Resource = 'r' } /// /// Options for the result returned by a download operation. /// public enum DownloadResult { /// /// Use to indicate that the operation failed. /// Failure = 0, /// /// Use to indicate that the operation failed because it could not locate the specified file. /// FileNotFound = 1, /// /// Use to indicate that the operation succedeed. /// Success = 2 } /// /// A GUID/Hash pair that uniquely identifies a particular file. For each FileId, the Cache Server can store a separate /// binary stream for each FileType. /// public struct FileId : IEqualityComparer { /// /// The guid byte array. /// public readonly byte[] guid; /// /// The hash code byte array. /// public readonly byte[] hash; /// /// A structure used to identify a file by guid and hash code. /// /// File GUID. /// File hash code. private FileId(byte[] guid, byte[] hash) { this.guid = guid; this.hash = hash; } /// /// Create a FileId given a string guid and string hash code representation. /// /// GUID string representation. /// Hash code string representation. /// public static FileId From(string guidStr, string hashStr) { if (guidStr.Length != 32) throw new ArgumentException("Length != 32", "guidStr"); if (hashStr.Length != 32) throw new ArgumentException("Length != 32", "hashStr"); return new FileId(Util.StringToGuid(guidStr), Util.StringToHash(hashStr)); } /// /// Create a FileId given a byte array guid and byte array hash code. /// /// GUID byte array. /// Hash code byte array. /// public static FileId From(byte[] guid, byte[] hash) { if (guid.Length != 16) throw new ArgumentException("Length != 32", "guid"); if (hash.Length != 16) throw new ArgumentException("Length != 32", "hash"); return new FileId(guid, hash); } /// /// Check equality of two objects given their guid and hash code. /// /// lhs object. /// rhs object. /// public new bool Equals(object x, object y) { var hash1 = (byte[])x; var hash2 = (byte[])y; if (hash1.Length != hash2.Length) return false; for (var i = 0; i < hash1.Length; i++) if (hash1[i] != hash2[i]) return false; return true; } /// /// Get the hash code for a specific object. /// /// The object you want the hash code for. /// public int GetHashCode(object obj) { var hc = 17; hc = hc * 23 + guid.GetHashCode(); hc = hc * 23 + hash.GetHashCode(); return hc; } } /// /// Exception thrown when an upload operation is not properly isolated within a begin/end transaction /// public class TransactionIsolationException : Exception { /// /// Creates a new exception for when an upload operation is not properly isolated within a begin/end transaction. /// /// The text containing information to display. public TransactionIsolationException(string msg) : base(msg) {} } /// /// EventArgs passed to the DownloadFinished event handler /// public class DownloadFinishedEventArgs : EventArgs { /// /// EventArgs download result code. /// public DownloadResult Result { get; set; } /// /// The downloaded item. /// public IDownloadItem DownloadItem { get; set; } /// /// The size of the downloaded item. /// public long Size { get; set; } /// /// The length of the download queue. /// public long DownloadQueueLength { get; set; } } /// /// A client API for uploading and downloading files from a Cache Server /// public class Client { private enum StreamReadState { Response, Size, Id } private const int ProtocolVersion = 254; private const string CmdTrxBegin = "ts"; private const string CmdTrxEnd = "te"; private const string CmdGet = "g"; private const string CmdPut = "p"; private const string CmdQuit = "q"; private const int ResponseLen = 2; private const int SizeLen = 16; private const int GuidLen = 16; private const int HashLen = 16; private const int IdLen = GuidLen + HashLen; private const int ReadBufferLen = 64 * 1024; private readonly Queue m_downloadQueue; private readonly TcpClient m_tcpClient; private readonly string m_host; private readonly int m_port; internal Stream m_stream; private Mutex m_mutex; private readonly byte[] m_streamReadBuffer; private int m_streamBytesRead; private int m_streamBytesNeeded; private StreamReadState m_streamReadState = StreamReadState.Response; private DownloadFinishedEventArgs m_nextFileCompleteEventArgs; private Stream m_nextWriteStream; private bool m_inTrx; /// /// Returns the number of items in the download queue /// public int DownloadQueueLength { get { return m_downloadQueue.Count; } } /// /// Event fired when a queued download request finishes. /// public event EventHandler DownloadFinished; /// /// Remove all listeners from the DownloadFinished event /// public void ResetDownloadFinishedEventHandler() { DownloadFinished = null; } /// /// Create a new Cache Server client /// /// The host name or IP of the Cache Server. /// The port number of the Cache Server. Default port is 8126. public Client(string host, int port = 8126) { m_streamReadBuffer = new byte[ReadBufferLen]; m_downloadQueue = new Queue(); m_tcpClient = new TcpClient(); m_host = host; m_port = port; } /// /// Connects to the Cache Server and sends a protocol version handshake. /// /// public void Connect() { var client = m_tcpClient; client.Connect(m_host, m_port); m_stream = client.GetStream(); m_stream.ReadTimeout = 10000; m_stream.WriteTimeout = 10000; SendVersion(); m_mutex = new Mutex(); } /// /// Connects to the Cache Server and sends a protocol version handshake. A TimeoutException is thrown if the connection cannot /// be established within milliseconds. /// /// /// /// public void Connect(int timeoutMs) { var client = m_tcpClient; var op = client.BeginConnect(m_host, m_port, null, null); var connected = op.AsyncWaitHandle.WaitOne(TimeSpan.FromMilliseconds(timeoutMs)); if (!connected) throw new TimeoutException(); m_stream = client.GetStream(); SendVersion(); m_mutex = new Mutex(); } /// /// Begin an upload transaction for an asset. Transactions in process can be interupted by calling BeginTransaction /// again before calling EndTransaction. /// /// public void BeginTransaction(FileId fileId) { m_inTrx = true; m_stream.Write(Encoding.ASCII.GetBytes(CmdTrxBegin), 0, 2); m_stream.Write(fileId.guid, 0, GuidLen); m_stream.Write(fileId.hash, 0, HashLen); } /// /// Upload from the given stream for the given FileType. Will throw an exception if not preceeded by BeginTransaction. /// /// /// /// /// public void Upload(FileType type, Stream readStream) { if (!m_inTrx) throw new TransactionIsolationException("Upload without BeginTransaction"); if (!readStream.CanRead || !readStream.CanSeek) throw new ArgumentException(); m_stream.Write(Encoding.ASCII.GetBytes(CmdPut + (char)type), 0, 2); m_stream.Write(Util.EncodeInt64(readStream.Length), 0, SizeLen); var buf = new byte[ReadBufferLen]; while (readStream.Position < readStream.Length - 1) { var len = readStream.Read(buf, 0, ReadBufferLen); m_stream.Write(buf, 0, len); } } /// /// Commit the uploaded files to the Cache Server. Will throw an exception if not preceeded by BeginTransaction. /// /// public void EndTransaction() { if (!m_inTrx) throw new TransactionIsolationException("EndTransaction without BeginTransaction"); m_inTrx = false; m_stream.Write(Encoding.ASCII.GetBytes(CmdTrxEnd), 0, 2); } /// /// Send a download request to the Cache Server. Listen to the DownloadComplete event to read the results. /// /// The IDownloadItem that specifies which file to download public void QueueDownload(IDownloadItem downloadItem) { m_stream.Write(Encoding.ASCII.GetBytes(CmdGet + (char)downloadItem.Type), 0, 2); m_stream.Write(downloadItem.Id.guid, 0, GuidLen); m_stream.Write(downloadItem.Id.hash, 0, HashLen); m_mutex.WaitOne(); m_downloadQueue.Enqueue(downloadItem); int count = m_downloadQueue.Count; m_mutex.ReleaseMutex(); if (count == 1) ReadNextDownloadResult(); } /// /// Close the connection to the Cache Server. Sends the 'quit' command and closes the network stream. /// public void Close() { if (m_stream != null) m_stream.Write(Encoding.ASCII.GetBytes(CmdQuit), 0, 1); if (m_tcpClient != null) m_tcpClient.Close(); if (m_mutex != null) { m_mutex.Dispose(); m_mutex = null; } } private void SendVersion() { var encodedVersion = Util.EncodeInt32(ProtocolVersion, true); m_stream.Write(encodedVersion, 0, encodedVersion.Length); var versionBuf = new byte[8]; var pos = 0; while (pos < versionBuf.Length - 1) { pos += m_stream.Read(versionBuf, 0, versionBuf.Length); } if (Util.ReadUInt32(versionBuf, 0) != ProtocolVersion) throw new Exception("Server version mismatch"); } private void OnDownloadFinished(DownloadFinishedEventArgs e) { m_mutex.WaitOne(); m_downloadQueue.Dequeue(); int count = m_downloadQueue.Count; m_mutex.ReleaseMutex(); e.DownloadQueueLength = count; if (DownloadFinished != null) DownloadFinished(this, e); if (count > 0) ReadNextDownloadResult(); } internal void ReadNextDownloadResult() { m_streamReadState = StreamReadState.Response; m_streamBytesNeeded = ResponseLen; m_streamBytesRead = 0; m_nextFileCompleteEventArgs = new DownloadFinishedEventArgs { Result = DownloadResult.Failure }; BeginReadHeader(); } private void BeginReadHeader() { m_stream.BeginRead(m_streamReadBuffer, m_streamBytesRead, m_streamBytesNeeded - m_streamBytesRead, EndReadHeader, m_stream); } internal Action OnReadHeader; private void EndReadHeader(IAsyncResult r) { var bytesRead = m_stream.EndRead(r); if (bytesRead <= 0) return; m_streamBytesRead += bytesRead; if (OnReadHeader != null) OnReadHeader(m_streamBytesRead, m_streamReadBuffer); if (m_streamBytesRead < m_streamBytesNeeded) { BeginReadHeader(); return; } switch (m_streamReadState) { case StreamReadState.Response: if (Convert.ToChar(m_streamReadBuffer[0]) == '+') { m_streamReadState = StreamReadState.Size; m_streamBytesNeeded = SizeLen; } else { m_nextFileCompleteEventArgs.Result = DownloadResult.FileNotFound; m_streamReadState = StreamReadState.Id; m_streamBytesNeeded = IdLen; } break; case StreamReadState.Size: m_nextFileCompleteEventArgs.Size = Util.ReadUInt64(m_streamReadBuffer, 0); m_streamReadState = StreamReadState.Id; m_streamBytesNeeded = IdLen; break; case StreamReadState.Id: m_mutex.WaitOne(); var next = m_downloadQueue.Peek(); m_mutex.ReleaseMutex(); m_nextFileCompleteEventArgs.DownloadItem = next; var match = Util.ByteArraysAreEqual(next.Id.guid, 0, m_streamReadBuffer, 0, GuidLen) && Util.ByteArraysAreEqual(next.Id.hash, 0, m_streamReadBuffer, GuidLen, HashLen); if (!match) { Close(); throw new InvalidDataException(); } if (m_nextFileCompleteEventArgs.Result == DownloadResult.FileNotFound) { OnDownloadFinished(m_nextFileCompleteEventArgs); } else { var size = m_nextFileCompleteEventArgs.Size; m_nextWriteStream = next.GetWriteStream(size); m_streamBytesNeeded = (int)size; m_streamBytesRead = 0; BeginReadData(); } return; default: throw new ArgumentOutOfRangeException(); } m_streamBytesRead = 0; BeginReadHeader(); } private void BeginReadData() { var len = Math.Min(ReadBufferLen, m_streamBytesNeeded - m_streamBytesRead); m_stream.BeginRead(m_streamReadBuffer, 0, len, EndReadData, null); } private void EndReadData(IAsyncResult readResult) { var bytesRead = m_stream.EndRead(readResult); Debug.Assert(bytesRead > 0); m_streamBytesRead += bytesRead; var writeResult = m_nextWriteStream.BeginWrite(m_streamReadBuffer, 0, bytesRead, null, null); m_nextWriteStream.EndWrite(writeResult); if (m_streamBytesRead < m_streamBytesNeeded) { BeginReadData(); } else { m_nextFileCompleteEventArgs.DownloadItem.Finish(); m_nextFileCompleteEventArgs.Result = DownloadResult.Success; OnDownloadFinished(m_nextFileCompleteEventArgs); } } } }