using System; using System.Globalization; using System.Text; using System.Text.RegularExpressions; using System.Web; using System.Net; using System.IO; using Amundsen.Utilities; // http://code.google.com/u/mca.amundsen/updates using System.Xml; using System.Collections; namespace Finsel.AzureCommands { /// /// Interface to Microsoft's Azure Blob Storage /// public class AzureBlobStorage { private Authentication pAuth = new Authentication(); /// /// Contains the authentication information /// public Authentication auth { get { return pAuth; } set { pAuth = value; pAuth.EndPoint = string.Format(EndpointFormat, pAuth.Account); ad.auth = pAuth; ; } } azureCommon ac = new azureCommon(); azureDirect ad = new azureDirect(); private string EndpointFormat = "http://{0}.blob.core.windows.net/"; private int maxNonBlockBytes = 4194304; HttpClient client = new HttpClient(); Hashing h = new Hashing(); /// /// Defines the UserAgent string passed in when making HTTP requests /// string UserAgent = "amundsen-finsel/1.0"; /// /// Create a new AzureCommands object /// public AzureBlobStorage() { client.UserAgent = UserAgent; } /// /// Create a new AzureCommands object with default settings /// /// The account is generally the first part of the Endpoint. /// The Endpoint for Azure Table Storage as defined for your account /// The SharedKey for access /// Shared public AzureBlobStorage(string account, string endPoint, string sharedKey, string keyType) { client.UserAgent = UserAgent; pAuth = new Authentication(account, string.Format(EndpointFormat, account), sharedKey); ad.auth = pAuth; client.UserAgent = UserAgent; } /// /// Create a new AzureCommands object with an auth object /// /// Azure Authetication Object public AzureBlobStorage(Authentication auth) { pAuth = auth; ad.auth = pAuth; } /// /// Results of request /// public azureResults RequestResults = new azureResults(); /// /// ETag for caching /// public string ETag = string.Empty; /// /// Data used in POST to ATS /// public string PostData = string.Empty; private string contentType = ""; private DateTime requestDate = DateTime.UtcNow; private string contentMD5 = string.Empty; private string authHeader = string.Empty; private string method = string.Empty; /// /// Get a list of tables defined for an ATS instance /// /// string array of table names public azureResults GetContainerList(string parameters) { string listParameters = "comp=list"; if (parameters != string.Empty && parameters != null) { if (parameters.StartsWith("?")) parameters = parameters.Substring(1); if (!parameters.StartsWith("&")) parameters = string.Format(CultureInfo.CurrentCulture, "&{0}", parameters); listParameters = string.Format(CultureInfo.CurrentCulture, "{0}{1}", listParameters, parameters); } return Containers(cmdType.get, "", new Hashtable(), listParameters); } /// /// Containers handles container level operations against Azure Blob Storage /// /// The type of operation you want to commit: Delete, Post, Put, Get /// Name of the container to perform the command agains /// A hashtable of Name-Value pairs containing header information /// public azureResults Containers(cmdType cmd, string containerName, Hashtable headers) { return Containers(cmd, containerName, headers, ""); } /// /// Containers handles container level operations against Azure Blob Storage /// /// The type of operation you want to commit: Delete, Post, Put, Get /// Name of the container to perform the command agains /// A hashtable of Name-Value pairs containing header information /// Parameters to add to the URI /// public azureResults Containers(cmdType cmd, string containerName, Hashtable headers, string parameters) { azureResults retVal = new azureResults(); HttpStatusCode success = HttpStatusCode.NotImplemented; try { StringBuilder sb = new StringBuilder(); string sendBody = string.Empty; string rtnBody = string.Empty; string requestUrl = string.Format(CultureInfo.CurrentCulture, "{0}{1}", auth.EndPoint, containerName); requestDate = DateTime.UtcNow; // handle version 2009-0-17 if (parameters == string.Empty) { parameters = "?restype=container"; } else { if (!parameters.ToLower().Contains("comp=list")) { parameters = string.Format("{0}&{1}", parameters, "restype=container"); } } switch (cmd) { case cmdType.get: case cmdType.head: method = cmd.ToString().ToUpper(); // do GET success = HttpStatusCode.OK; break; case cmdType.delete: method = "DELETE"; // do DELETE success = HttpStatusCode.Accepted; break; case cmdType.put: method = "PUT"; success = HttpStatusCode.Created; break; default: //retVal.StatusCode = HttpStatusCode.NotImplemented; break; } if (parameters != string.Empty && parameters != null) { if (!parameters.StartsWith("?")) parameters = string.Format("?{0}", parameters); requestUrl += string.Format(CultureInfo.CurrentCulture, "{0}", parameters); retVal.Url = requestUrl; } retVal = ad.ProcessRequest(cmd, requestUrl, "", headers); retVal.Succeeded = (retVal.StatusCode == success); } catch (HttpException hex) { retVal.StatusCode = (HttpStatusCode)hex.GetHttpCode(); retVal.Succeeded = false; retVal.Body = hex.GetHtmlErrorMessage(); } catch (Exception ex) { retVal.StatusCode = HttpStatusCode.SeeOther; retVal.Body = ex.ToString(); retVal.Succeeded = false; } return retVal; } /// /// Blobs handles blob level operations against Azure Blob Storage /// /// The type of operation you want to commit: Delete, Post, Put, Get /// Name of the container where blob exists /// Name of blob to perform command against /// A hashtable of Name-Value pairs containing header information /// Parameters to add to the URI /// public azureResults Blobs(cmdType cmd, string containerName, string blobName, Hashtable headers, string parameters) { azureResults retVal = new azureResults(); HttpStatusCode success = HttpStatusCode.NotImplemented; try { StringBuilder sb = new StringBuilder(); string sendBody = string.Empty; string rtnBody = string.Empty; string requestUrl = string.Format(CultureInfo.CurrentCulture, "{0}{1}/{2}", auth.EndPoint, containerName, blobName); requestDate = DateTime.UtcNow; switch (cmd) { case cmdType.get: case cmdType.head: method = cmd.ToString().ToUpper(); // do GET success = HttpStatusCode.OK; break; case cmdType.delete: method = "DELETE"; // do DELETE success = HttpStatusCode.Accepted; break; case cmdType.put: method = "PUT"; success = HttpStatusCode.OK; break; default: //retVal.StatusCode = HttpStatusCode.NotImplemented; break; } if (parameters != string.Empty && parameters != null) { if (!parameters.StartsWith("?")) parameters = string.Format("?{0}", parameters); requestUrl += string.Format(CultureInfo.CurrentCulture, "{0}", parameters); } retVal = ad.ProcessRequest(cmd, requestUrl, "", headers); retVal.Succeeded = (retVal.StatusCode == success); } catch (HttpException hex) { retVal.StatusCode = (HttpStatusCode)hex.GetHttpCode(); retVal.Succeeded = false; retVal.Body = hex.GetHtmlErrorMessage(); } catch (Exception ex) { retVal.StatusCode = HttpStatusCode.SeeOther; retVal.Body = ex.ToString(); retVal.Succeeded = false; } return retVal; } /// /// Defines whether a container is accessible without authentication /// /// Name of the container /// If true, all blobs in the container will be available through standard http without authentication /// An azureResults showing the results of the request. public azureResults SetContainerAccess(string containerName, bool MakePublic) { Hashtable ht = new Hashtable(); ht.Add("x-ms-prop-publicaccess", (MakePublic ? "True" : "False")); azureResults ar = Containers(cmdType.put, containerName, ht, "?comp=acl"); ar.Succeeded = (ar.StatusCode == HttpStatusCode.OK); return ar; } /// /// Delete a blob stored in a container. /// /// Container name /// Blob name, including paths /// An azureResults showing the results of the request. public azureResults DeleteBlob(string containerName, string blobName) { azureResults retVal = new azureResults(); Hashtable ht = new Hashtable(); ht.Add("If-Match", "*"); retVal = Blobs(cmdType.delete, containerName, blobName, ht, ""); retVal.Succeeded = (retVal.StatusCode == HttpStatusCode.OK || retVal.StatusCode == HttpStatusCode.Accepted); return retVal; } /// /// GetBlobList returns a list of the blobs in a container. /// /// Name of the container /// Parameters include: prefix, marker and maxresults. /// azureResults object with results of the request. public azureResults GetBlobList(string containerName, string parameters) { azureResults retVal = new azureResults(); retVal = Blobs(cmdType.get, containerName, "", new Hashtable(), "?restype=container&comp=list"); retVal.Succeeded = (retVal.StatusCode == HttpStatusCode.OK); return retVal; } /// /// GetBlob gets a blob from Microsoft's Azure Blob Storage /// /// Name of the container holding the blob. /// Name of the blob, can include paths /// Used to get a range of the blob rather than the whole thing. /// byte array containing the blob. public byte[] GetBlob(string containerName, string blobName, string Range) { azureResults ar = new azureResults(); return GetBlob(containerName, blobName, Range, ref ar); } /// /// GetBlob gets a blob from Microsoft's Azure Blob Storage /// /// Name of the container holding the blob. /// Name of the blob, can include paths /// Used to get a range of the blob rather than the whole thing. /// Passed in by ref, will contain the results of the request. /// byte array containing the blob. public byte[] GetBlob(string containerName, string blobName, string Range, ref azureResults ar) { MemoryStream retVal = null; try { StringBuilder sb = new StringBuilder(); string sendBody = string.Empty; string rtnBody = string.Empty; string requestUrl = string.Format(CultureInfo.CurrentCulture, "{0}{1}/{2}", auth.EndPoint, containerName, blobName); requestDate = DateTime.UtcNow; method = "GET"; if (Range != string.Empty) client.RequestHeaders["range"] = Range; // do GET client.RequestHeaders["x-ms-date"] = string.Format(CultureInfo.CurrentCulture, "{0:R}", requestDate); ar.CanonicalResource = ac.CanonicalizeUrl(requestUrl); authHeader = ac.CreateSharedKeyAuth(method, ar.CanonicalResource, contentMD5, requestDate, client, auth); client.RequestHeaders["authorization"] = authHeader; client.UseBinaryStream = true; client.Execute(requestUrl, method, contentType, "", ref retVal); this.ETag = client.ResponseHeaders["etag"]; ar.Url = requestUrl; ar.Body = ""; ar.StatusCode = client.ResponseStatusCode; ar.Headers = ac.Headers2Hash(client.ResponseHeaders); ar.Succeeded = (ar.StatusCode == HttpStatusCode.OK); } catch (HttpException hex) { ar.StatusCode = (HttpStatusCode)hex.GetHttpCode(); ar.Succeeded = false; ar.Body = hex.GetHtmlErrorMessage(); } catch (Exception ex) { ar.StatusCode = HttpStatusCode.SeeOther; ar.Body = ex.ToString(); ar.Succeeded = false; } if (retVal != null) return retVal.ToArray();// retVal.ToArray(); else return null; } /// /// PutBlob creates/updates a blob within Microsoft's Azure Blob Storage /// /// How many bytes are in the Content /// Content type of the blob. application/bin is used by default if nothing else is passed in /// A byte array representing the blob being stored. /// The name of the container to store the blob in. /// The name of the blob. Can inlcude paths (/path/blob.txt, for instance) /// A hashtable containing the Name-Value pairs of MetaData. /// public azureResults PutBlob(Int64 ContentLength, string ContentType, byte[] Content, string containerName, string blobName, Hashtable htMetaData) { azureResults retVal = new azureResults(); try { StringBuilder sb = new StringBuilder(); string sendBody = string.Empty; string rtnBody = string.Empty; //if (containerName == "$root") // containerName = string.Empty; string requestUrl = string.Format(CultureInfo.CurrentCulture, "{0}{1}/{2}", auth.EndPoint, containerName, blobName); requestDate = DateTime.UtcNow; if (ContentLength < maxNonBlockBytes) { HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(requestUrl); req.Method = "PUT"; req.ContentType = ContentType; req.ContentLength = ContentLength; if (htMetaData.Count > 0) foreach (DictionaryEntry item in htMetaData) { string metaDataName = item.Key.ToString().ToLower().Replace(" ", "-").Replace("\r", ""); if (!metaDataName.StartsWith("x-ms-meta-")) metaDataName = "x-ms-meta-" + metaDataName; try { if (item.Value.ToString().Trim() != string.Empty) req.Headers[metaDataName] = item.Value.ToString(); } catch (Exception ex) { } } req.Headers["x-ms-date"] = string.Format(CultureInfo.CurrentCulture, "{0:R}", requestDate); req.Headers["x-ms-version"] = "2009-07-17"; retVal.CanonicalResource = ac.CanonicalizeUrl(requestUrl); authHeader = ac.CreateSharedKeyAuth(req.Method, retVal.CanonicalResource, contentMD5, requestDate, req, auth); req.Headers["authorization"] = authHeader; Stream requestStream = req.GetRequestStream(); requestStream.Write(Content, 0, (int)ContentLength); requestStream.Flush(); HttpWebResponse response = (HttpWebResponse)req.GetResponse(); response.Close(); retVal.Url = requestUrl; retVal.Body = ""; retVal.StatusCode = response.StatusCode; retVal.Headers = ac.Headers2Hash(response.Headers); retVal.Succeeded = (retVal.StatusCode == HttpStatusCode.Created); } else { string blockURI = string.Empty; Hashtable htHeaders = htMetaData; htHeaders.Add("Content-Type", ContentType); azureDirect ad = new azureDirect(auth.Account, auth.EndPoint, auth.SharedKey, auth.KeyType); int blocksCount = (int)Math.Ceiling((double)ContentLength / maxNonBlockBytes); string[] blockIds = new string[blocksCount]; int startPosition = 0; for (int i = 0; i < blocksCount; i++) { blockIds[i] = Convert.ToBase64String(BitConverter.GetBytes(i)); blockURI = string.Format("{0}?comp=block&blockid={1}", requestUrl, blockIds[i]); byte[] blockContent = new byte[maxNonBlockBytes]; Array.Copy(Content, startPosition, blockContent, 0, (startPosition + maxNonBlockBytes > Content.Length ? Content.Length - startPosition : maxNonBlockBytes)); retVal = ad.ProcessRequest(cmdType.put, blockURI, blockContent, htHeaders); startPosition += maxNonBlockBytes; } blockURI = string.Format("{0}?comp=blocklist", requestUrl); StringBuilder sbBlockList = new StringBuilder(); sbBlockList.Append("\n"); foreach (string id in blockIds) { sbBlockList.AppendFormat("{0}\n", id); } sbBlockList.Append(""); retVal = ad.ProcessRequest(cmdType.put, blockURI, new System.Text.ASCIIEncoding().GetBytes(sbBlockList.ToString()), htHeaders); } } catch (HttpException hex) { retVal.StatusCode = (HttpStatusCode)hex.GetHttpCode(); retVal.Succeeded = false; retVal.Body = hex.GetHtmlErrorMessage(); } catch (Exception ex) { retVal.StatusCode = HttpStatusCode.SeeOther; retVal.Body = ex.ToString(); retVal.Succeeded = false; } finally { } return retVal; } /// /// PutBlob creates/updates a blob within Microsoft's Azure Blob Storage /// /// How many bytes are in the Content /// Content type of the blob. application/bin is used by default if nothing else is passed in /// A byte array representing the blob being stored. /// The name of the container to store the blob in. /// The name of the blob. Can inlcude paths (/path/blob.txt, for instance) /// A hashtable containing the Name-Value pairs of MetaData. /// public azureResults PutBlobWithBlocks(string ContentType, byte[] Content, string containerName, string blobName, Hashtable htMetaData, int BlockLength, int StartBlock) { azureResults retVal = new azureResults(); try { StringBuilder sb = new StringBuilder(); string sendBody = string.Empty; string rtnBody = string.Empty; string requestUrl = string.Format(CultureInfo.CurrentCulture, "{0}{1}/{2}", auth.EndPoint, containerName, blobName); requestDate = DateTime.UtcNow; string blockURI = string.Empty; Hashtable htHeaders = htMetaData; htHeaders.Add("Content-Type", ContentType); azureDirect ad = new azureDirect(auth.Account, auth.EndPoint, auth.SharedKey, auth.KeyType); int blocksCount = (int)Math.Ceiling((double)Content.Length / BlockLength ) ; string[] blockIds = new string[blocksCount]; int startPosition = StartBlock*BlockLength ; int blockIdPosition = 0; for (int i = StartBlock; i < blocksCount; i++) { blockIds[blockIdPosition] = Convert.ToBase64String(BitConverter.GetBytes(i)); blockURI = string.Format("{0}?comp=block&blockid={1}", requestUrl, blockIds[blockIdPosition]); blockIdPosition++; byte[] blockContent = new byte[BlockLength]; Array.Copy(Content, startPosition, blockContent, 0, (startPosition + BlockLength > Content.Length ? Content.Length - startPosition : BlockLength)); retVal = ad.ProcessRequest(cmdType.put, blockURI, blockContent, htHeaders); while(retVal.StatusCode != HttpStatusCode.Created) retVal = ad.ProcessRequest(cmdType.put, blockURI, blockContent, htHeaders); startPosition += BlockLength; Console.WriteLine(i); } blockURI = string.Format("{0}?comp=blocklist", requestUrl); StringBuilder sbBlockList = new StringBuilder(); sbBlockList.Append("\n"); foreach (string id in blockIds) { sbBlockList.AppendFormat("{0}\n", id); } sbBlockList.Append(""); retVal = ad.ProcessRequest(cmdType.put, blockURI, new System.Text.ASCIIEncoding().GetBytes(sbBlockList.ToString()), htHeaders); } catch (HttpException hex) { retVal.StatusCode = (HttpStatusCode)hex.GetHttpCode(); retVal.Succeeded = false; retVal.Body = hex.GetHtmlErrorMessage(); } catch (Exception ex) { retVal.StatusCode = HttpStatusCode.SeeOther; retVal.Body = ex.ToString(); retVal.Succeeded = false; } return retVal; } public azureResults CopyBlob(string OriginalBlobUrl, string NewBlobUrl, Hashtable htHeaders) { if (htHeaders.ContainsKey("x-ms-copy-source")) htHeaders["x-ms-copy-source"] = new azureCommon().CanonicalizeUrl(OriginalBlobUrl); else htHeaders.Add("x-ms-copy-source", new azureCommon().CanonicalizeUrl(OriginalBlobUrl)); azureResults retVal = new azureResults(); azureDirect ad = new azureDirect(auth); retVal = ad.ProcessRequest(cmdType.put, NewBlobUrl, "", htHeaders); return retVal; } /// /// Metadata is a quick way to get/set the metadata about a container or a blob /// /// The type of command you want to execute. GET, PUT and DELETE are supported /// Name of the container. If no blobName is passed in, will apply Metadata /// command to the Container. /// The name of the blob in the container, if you want to apply the Metadata command to a blob. /// A hashtable containing the Name-Value pairs of MetaData. /// An azureResults showing the results of the request. public azureResults MetaData(cmdType cmd, string containerName, string blobName, Hashtable htMetaData) { azureResults retVal = new azureResults(); Hashtable headers = new Hashtable(); if (htMetaData != null) if (htMetaData.Count > 0) foreach (DictionaryEntry item in htMetaData) { string metaDataName = item.Key.ToString().ToLower().Replace(" ", "-").Replace("\r", ""); if (!metaDataName.StartsWith("x-ms-meta-")) metaDataName = "x-ms-meta-" + metaDataName; headers.Add(metaDataName, item.Value); } if (blobName == string.Empty) { retVal = Containers(cmd, containerName, headers, "?comp=metadata"); retVal.Succeeded = (retVal.StatusCode == HttpStatusCode.OK); } else { retVal = Blobs(cmd, containerName, blobName, headers, "?comp=metadata"); } return retVal; } } }