using System;
using System.Globalization;
using System.Text;
using System.Text.RegularExpressions;
using System.Web;
using System.Net;
using System.IO;
using Amundsen.Utilities; // http://code.google.com/u/mca.amundsen/updates
using System.Xml;
using System.Collections;
namespace Finsel.AzureCommands
{
///
/// Interface to Microsoft's Azure Blob Storage
///
public class AzureBlobStorage
{
private Authentication pAuth = new Authentication();
///
/// Contains the authentication information
///
public Authentication auth { get { return pAuth; } set { pAuth = value; pAuth.EndPoint = string.Format(EndpointFormat, pAuth.Account); ad.auth = pAuth; ; } }
azureCommon ac = new azureCommon();
azureDirect ad = new azureDirect();
private string EndpointFormat = "http://{0}.blob.core.windows.net/";
private int maxNonBlockBytes = 4194304;
HttpClient client = new HttpClient();
Hashing h = new Hashing();
///
/// Defines the UserAgent string passed in when making HTTP requests
///
string UserAgent = "amundsen-finsel/1.0";
///
/// Create a new AzureCommands object
///
public AzureBlobStorage()
{
client.UserAgent = UserAgent;
}
///
/// Create a new AzureCommands object with default settings
///
/// The account is generally the first part of the Endpoint.
/// The Endpoint for Azure Table Storage as defined for your account
/// The SharedKey for access
/// Shared
public AzureBlobStorage(string account, string endPoint, string sharedKey, string keyType)
{
client.UserAgent = UserAgent;
pAuth = new Authentication(account, string.Format(EndpointFormat, account), sharedKey);
ad.auth = pAuth;
client.UserAgent = UserAgent;
}
///
/// Create a new AzureCommands object with an auth object
///
/// Azure Authetication Object
public AzureBlobStorage(Authentication auth)
{
pAuth = auth;
ad.auth = pAuth;
}
///
/// Results of request
///
public azureResults RequestResults = new azureResults();
///
/// ETag for caching
///
public string ETag = string.Empty;
///
/// Data used in POST to ATS
///
public string PostData = string.Empty;
private string contentType = "";
private DateTime requestDate = DateTime.UtcNow;
private string contentMD5 = string.Empty;
private string authHeader = string.Empty;
private string method = string.Empty;
///
/// Get a list of tables defined for an ATS instance
///
/// string array of table names
public azureResults GetContainerList(string parameters)
{
string listParameters = "comp=list";
if (parameters != string.Empty && parameters != null)
{
if (parameters.StartsWith("?"))
parameters = parameters.Substring(1);
if (!parameters.StartsWith("&"))
parameters = string.Format(CultureInfo.CurrentCulture, "&{0}", parameters);
listParameters = string.Format(CultureInfo.CurrentCulture, "{0}{1}", listParameters, parameters);
}
return Containers(cmdType.get, "", new Hashtable(), listParameters);
}
///
/// Containers handles container level operations against Azure Blob Storage
///
/// The type of operation you want to commit: Delete, Post, Put, Get
/// Name of the container to perform the command agains
/// A hashtable of Name-Value pairs containing header information
///
public azureResults Containers(cmdType cmd, string containerName, Hashtable headers)
{
return Containers(cmd, containerName, headers, "");
}
///
/// Containers handles container level operations against Azure Blob Storage
///
/// The type of operation you want to commit: Delete, Post, Put, Get
/// Name of the container to perform the command agains
/// A hashtable of Name-Value pairs containing header information
/// Parameters to add to the URI
///
public azureResults Containers(cmdType cmd, string containerName, Hashtable headers, string parameters)
{
azureResults retVal = new azureResults();
HttpStatusCode success = HttpStatusCode.NotImplemented;
try
{
StringBuilder sb = new StringBuilder();
string sendBody = string.Empty;
string rtnBody = string.Empty;
string requestUrl = string.Format(CultureInfo.CurrentCulture, "{0}{1}", auth.EndPoint, containerName);
requestDate = DateTime.UtcNow;
// handle version 2009-0-17
if (parameters == string.Empty)
{
parameters = "?restype=container";
}
else
{
if (!parameters.ToLower().Contains("comp=list"))
{
parameters = string.Format("{0}&{1}", parameters, "restype=container");
}
}
switch (cmd)
{
case cmdType.get:
case cmdType.head:
method = cmd.ToString().ToUpper();
// do GET
success = HttpStatusCode.OK;
break;
case cmdType.delete:
method = "DELETE";
// do DELETE
success = HttpStatusCode.Accepted;
break;
case cmdType.put:
method = "PUT";
success = HttpStatusCode.Created;
break;
default:
//retVal.StatusCode = HttpStatusCode.NotImplemented;
break;
}
if (parameters != string.Empty && parameters != null)
{
if (!parameters.StartsWith("?"))
parameters = string.Format("?{0}", parameters);
requestUrl += string.Format(CultureInfo.CurrentCulture, "{0}", parameters);
retVal.Url = requestUrl;
}
retVal = ad.ProcessRequest(cmd, requestUrl, "", headers);
retVal.Succeeded = (retVal.StatusCode == success);
}
catch (HttpException hex)
{
retVal.StatusCode = (HttpStatusCode)hex.GetHttpCode();
retVal.Succeeded = false;
retVal.Body = hex.GetHtmlErrorMessage();
}
catch (Exception ex)
{
retVal.StatusCode = HttpStatusCode.SeeOther;
retVal.Body = ex.ToString();
retVal.Succeeded = false;
}
return retVal;
}
///
/// Blobs handles blob level operations against Azure Blob Storage
///
/// The type of operation you want to commit: Delete, Post, Put, Get
/// Name of the container where blob exists
/// Name of blob to perform command against
/// A hashtable of Name-Value pairs containing header information
/// Parameters to add to the URI
///
public azureResults Blobs(cmdType cmd, string containerName, string blobName, Hashtable headers, string parameters)
{
azureResults retVal = new azureResults();
HttpStatusCode success = HttpStatusCode.NotImplemented;
try
{
StringBuilder sb = new StringBuilder();
string sendBody = string.Empty;
string rtnBody = string.Empty;
string requestUrl = string.Format(CultureInfo.CurrentCulture, "{0}{1}/{2}", auth.EndPoint, containerName, blobName);
requestDate = DateTime.UtcNow;
switch (cmd)
{
case cmdType.get:
case cmdType.head:
method = cmd.ToString().ToUpper();
// do GET
success = HttpStatusCode.OK;
break;
case cmdType.delete:
method = "DELETE";
// do DELETE
success = HttpStatusCode.Accepted;
break;
case cmdType.put:
method = "PUT";
success = HttpStatusCode.OK;
break;
default:
//retVal.StatusCode = HttpStatusCode.NotImplemented;
break;
}
if (parameters != string.Empty && parameters != null)
{
if (!parameters.StartsWith("?"))
parameters = string.Format("?{0}", parameters);
requestUrl += string.Format(CultureInfo.CurrentCulture, "{0}", parameters);
}
retVal = ad.ProcessRequest(cmd, requestUrl, "", headers);
retVal.Succeeded = (retVal.StatusCode == success);
}
catch (HttpException hex)
{
retVal.StatusCode = (HttpStatusCode)hex.GetHttpCode();
retVal.Succeeded = false;
retVal.Body = hex.GetHtmlErrorMessage();
}
catch (Exception ex)
{
retVal.StatusCode = HttpStatusCode.SeeOther;
retVal.Body = ex.ToString();
retVal.Succeeded = false;
}
return retVal;
}
///
/// Defines whether a container is accessible without authentication
///
/// Name of the container
/// If true, all blobs in the container will be available through standard http without authentication
/// An azureResults showing the results of the request.
public azureResults SetContainerAccess(string containerName, bool MakePublic)
{
Hashtable ht = new Hashtable();
ht.Add("x-ms-prop-publicaccess", (MakePublic ? "True" : "False"));
azureResults ar = Containers(cmdType.put, containerName, ht, "?comp=acl");
ar.Succeeded = (ar.StatusCode == HttpStatusCode.OK);
return ar;
}
///
/// Delete a blob stored in a container.
///
/// Container name
/// Blob name, including paths
/// An azureResults showing the results of the request.
public azureResults DeleteBlob(string containerName, string blobName)
{
azureResults retVal = new azureResults();
Hashtable ht = new Hashtable();
ht.Add("If-Match", "*");
retVal = Blobs(cmdType.delete, containerName, blobName, ht, "");
retVal.Succeeded = (retVal.StatusCode == HttpStatusCode.OK || retVal.StatusCode == HttpStatusCode.Accepted);
return retVal;
}
///
/// GetBlobList returns a list of the blobs in a container.
///
/// Name of the container
/// Parameters include: prefix, marker and maxresults.
/// azureResults object with results of the request.
public azureResults GetBlobList(string containerName, string parameters)
{
azureResults retVal = new azureResults();
retVal = Blobs(cmdType.get, containerName, "", new Hashtable(), "?restype=container&comp=list");
retVal.Succeeded = (retVal.StatusCode == HttpStatusCode.OK);
return retVal;
}
///
/// GetBlob gets a blob from Microsoft's Azure Blob Storage
///
/// Name of the container holding the blob.
/// Name of the blob, can include paths
/// Used to get a range of the blob rather than the whole thing.
/// byte array containing the blob.
public byte[] GetBlob(string containerName, string blobName, string Range)
{
azureResults ar = new azureResults();
return GetBlob(containerName, blobName, Range, ref ar);
}
///
/// GetBlob gets a blob from Microsoft's Azure Blob Storage
///
/// Name of the container holding the blob.
/// Name of the blob, can include paths
/// Used to get a range of the blob rather than the whole thing.
/// Passed in by ref, will contain the results of the request.
/// byte array containing the blob.
public byte[] GetBlob(string containerName, string blobName, string Range, ref azureResults ar)
{
MemoryStream retVal = null;
try
{
StringBuilder sb = new StringBuilder();
string sendBody = string.Empty;
string rtnBody = string.Empty;
string requestUrl = string.Format(CultureInfo.CurrentCulture, "{0}{1}/{2}", auth.EndPoint, containerName, blobName);
requestDate = DateTime.UtcNow;
method = "GET";
if (Range != string.Empty)
client.RequestHeaders["range"] = Range;
// do GET
client.RequestHeaders["x-ms-date"] = string.Format(CultureInfo.CurrentCulture, "{0:R}", requestDate);
ar.CanonicalResource = ac.CanonicalizeUrl(requestUrl);
authHeader = ac.CreateSharedKeyAuth(method, ar.CanonicalResource, contentMD5, requestDate, client, auth);
client.RequestHeaders["authorization"] = authHeader;
client.UseBinaryStream = true;
client.Execute(requestUrl, method, contentType, "", ref retVal);
this.ETag = client.ResponseHeaders["etag"];
ar.Url = requestUrl;
ar.Body = "";
ar.StatusCode = client.ResponseStatusCode;
ar.Headers = ac.Headers2Hash(client.ResponseHeaders);
ar.Succeeded = (ar.StatusCode == HttpStatusCode.OK);
}
catch (HttpException hex)
{
ar.StatusCode = (HttpStatusCode)hex.GetHttpCode();
ar.Succeeded = false;
ar.Body = hex.GetHtmlErrorMessage();
}
catch (Exception ex)
{
ar.StatusCode = HttpStatusCode.SeeOther;
ar.Body = ex.ToString();
ar.Succeeded = false;
}
if (retVal != null)
return retVal.ToArray();// retVal.ToArray();
else return null;
}
///
/// PutBlob creates/updates a blob within Microsoft's Azure Blob Storage
///
/// How many bytes are in the Content
/// Content type of the blob. application/bin is used by default if nothing else is passed in
/// A byte array representing the blob being stored.
/// The name of the container to store the blob in.
/// The name of the blob. Can inlcude paths (/path/blob.txt, for instance)
/// A hashtable containing the Name-Value pairs of MetaData.
///
public azureResults PutBlob(Int64 ContentLength, string ContentType, byte[] Content, string containerName, string blobName, Hashtable htMetaData)
{
azureResults retVal = new azureResults();
try
{
StringBuilder sb = new StringBuilder();
string sendBody = string.Empty;
string rtnBody = string.Empty;
//if (containerName == "$root")
// containerName = string.Empty;
string requestUrl = string.Format(CultureInfo.CurrentCulture, "{0}{1}/{2}", auth.EndPoint, containerName, blobName);
requestDate = DateTime.UtcNow;
if (ContentLength < maxNonBlockBytes)
{
HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(requestUrl);
req.Method = "PUT";
req.ContentType = ContentType;
req.ContentLength = ContentLength;
if (htMetaData.Count > 0)
foreach (DictionaryEntry item in htMetaData)
{
string metaDataName = item.Key.ToString().ToLower().Replace(" ", "-").Replace("\r", "");
if (!metaDataName.StartsWith("x-ms-meta-"))
metaDataName = "x-ms-meta-" + metaDataName;
try
{
if (item.Value.ToString().Trim() != string.Empty)
req.Headers[metaDataName] = item.Value.ToString();
}
catch (Exception ex) { }
}
req.Headers["x-ms-date"] = string.Format(CultureInfo.CurrentCulture, "{0:R}", requestDate);
req.Headers["x-ms-version"] = "2009-07-17";
retVal.CanonicalResource = ac.CanonicalizeUrl(requestUrl);
authHeader = ac.CreateSharedKeyAuth(req.Method, retVal.CanonicalResource, contentMD5, requestDate, req, auth);
req.Headers["authorization"] = authHeader;
Stream requestStream = req.GetRequestStream();
requestStream.Write(Content, 0, (int)ContentLength);
requestStream.Flush();
HttpWebResponse response = (HttpWebResponse)req.GetResponse();
response.Close();
retVal.Url = requestUrl;
retVal.Body = "";
retVal.StatusCode = response.StatusCode;
retVal.Headers = ac.Headers2Hash(response.Headers);
retVal.Succeeded = (retVal.StatusCode == HttpStatusCode.Created);
}
else
{
string blockURI = string.Empty;
Hashtable htHeaders = htMetaData;
htHeaders.Add("Content-Type", ContentType);
azureDirect ad = new azureDirect(auth.Account, auth.EndPoint, auth.SharedKey, auth.KeyType);
int blocksCount = (int)Math.Ceiling((double)ContentLength / maxNonBlockBytes);
string[] blockIds = new string[blocksCount];
int startPosition = 0;
for (int i = 0; i < blocksCount; i++)
{
blockIds[i] = Convert.ToBase64String(BitConverter.GetBytes(i));
blockURI = string.Format("{0}?comp=block&blockid={1}", requestUrl, blockIds[i]);
byte[] blockContent = new byte[maxNonBlockBytes];
Array.Copy(Content, startPosition, blockContent, 0, (startPosition + maxNonBlockBytes > Content.Length ? Content.Length - startPosition : maxNonBlockBytes));
retVal = ad.ProcessRequest(cmdType.put, blockURI, blockContent, htHeaders);
startPosition += maxNonBlockBytes;
}
blockURI = string.Format("{0}?comp=blocklist", requestUrl);
StringBuilder sbBlockList = new StringBuilder();
sbBlockList.Append("\n");
foreach (string id in blockIds)
{
sbBlockList.AppendFormat("{0}\n", id);
}
sbBlockList.Append("");
retVal = ad.ProcessRequest(cmdType.put, blockURI, new System.Text.ASCIIEncoding().GetBytes(sbBlockList.ToString()), htHeaders);
}
}
catch (HttpException hex)
{
retVal.StatusCode = (HttpStatusCode)hex.GetHttpCode();
retVal.Succeeded = false;
retVal.Body = hex.GetHtmlErrorMessage();
}
catch (Exception ex)
{
retVal.StatusCode = HttpStatusCode.SeeOther;
retVal.Body = ex.ToString();
retVal.Succeeded = false;
}
finally
{
}
return retVal;
}
///
/// PutBlob creates/updates a blob within Microsoft's Azure Blob Storage
///
/// How many bytes are in the Content
/// Content type of the blob. application/bin is used by default if nothing else is passed in
/// A byte array representing the blob being stored.
/// The name of the container to store the blob in.
/// The name of the blob. Can inlcude paths (/path/blob.txt, for instance)
/// A hashtable containing the Name-Value pairs of MetaData.
///
public azureResults PutBlobWithBlocks(string ContentType, byte[] Content, string containerName, string blobName, Hashtable htMetaData, int BlockLength, int StartBlock)
{
azureResults retVal = new azureResults();
try
{
StringBuilder sb = new StringBuilder();
string sendBody = string.Empty;
string rtnBody = string.Empty;
string requestUrl = string.Format(CultureInfo.CurrentCulture, "{0}{1}/{2}", auth.EndPoint, containerName, blobName);
requestDate = DateTime.UtcNow;
string blockURI = string.Empty;
Hashtable htHeaders = htMetaData;
htHeaders.Add("Content-Type", ContentType);
azureDirect ad = new azureDirect(auth.Account, auth.EndPoint, auth.SharedKey, auth.KeyType);
int blocksCount = (int)Math.Ceiling((double)Content.Length / BlockLength ) ;
string[] blockIds = new string[blocksCount];
int startPosition = StartBlock*BlockLength ;
int blockIdPosition = 0;
for (int i = StartBlock; i < blocksCount; i++)
{
blockIds[blockIdPosition] = Convert.ToBase64String(BitConverter.GetBytes(i));
blockURI = string.Format("{0}?comp=block&blockid={1}", requestUrl, blockIds[blockIdPosition]);
blockIdPosition++;
byte[] blockContent = new byte[BlockLength];
Array.Copy(Content, startPosition, blockContent, 0, (startPosition + BlockLength > Content.Length ? Content.Length - startPosition : BlockLength));
retVal = ad.ProcessRequest(cmdType.put, blockURI, blockContent, htHeaders);
while(retVal.StatusCode != HttpStatusCode.Created)
retVal = ad.ProcessRequest(cmdType.put, blockURI, blockContent, htHeaders);
startPosition += BlockLength;
Console.WriteLine(i);
}
blockURI = string.Format("{0}?comp=blocklist", requestUrl);
StringBuilder sbBlockList = new StringBuilder();
sbBlockList.Append("\n");
foreach (string id in blockIds)
{
sbBlockList.AppendFormat("{0}\n", id);
}
sbBlockList.Append("");
retVal = ad.ProcessRequest(cmdType.put, blockURI, new System.Text.ASCIIEncoding().GetBytes(sbBlockList.ToString()), htHeaders);
}
catch (HttpException hex)
{
retVal.StatusCode = (HttpStatusCode)hex.GetHttpCode();
retVal.Succeeded = false;
retVal.Body = hex.GetHtmlErrorMessage();
}
catch (Exception ex)
{
retVal.StatusCode = HttpStatusCode.SeeOther;
retVal.Body = ex.ToString();
retVal.Succeeded = false;
}
return retVal;
}
public azureResults CopyBlob(string OriginalBlobUrl, string NewBlobUrl, Hashtable htHeaders)
{
if (htHeaders.ContainsKey("x-ms-copy-source"))
htHeaders["x-ms-copy-source"] = new azureCommon().CanonicalizeUrl(OriginalBlobUrl);
else
htHeaders.Add("x-ms-copy-source", new azureCommon().CanonicalizeUrl(OriginalBlobUrl));
azureResults retVal = new azureResults();
azureDirect ad = new azureDirect(auth);
retVal = ad.ProcessRequest(cmdType.put, NewBlobUrl, "", htHeaders);
return retVal;
}
///
/// Metadata is a quick way to get/set the metadata about a container or a blob
///
/// The type of command you want to execute. GET, PUT and DELETE are supported
/// Name of the container. If no blobName is passed in, will apply Metadata
/// command to the Container.
/// The name of the blob in the container, if you want to apply the Metadata command to a blob.
/// A hashtable containing the Name-Value pairs of MetaData.
/// An azureResults showing the results of the request.
public azureResults MetaData(cmdType cmd, string containerName, string blobName, Hashtable htMetaData)
{
azureResults retVal = new azureResults();
Hashtable headers = new Hashtable();
if (htMetaData != null)
if (htMetaData.Count > 0)
foreach (DictionaryEntry item in htMetaData)
{
string metaDataName = item.Key.ToString().ToLower().Replace(" ", "-").Replace("\r", "");
if (!metaDataName.StartsWith("x-ms-meta-"))
metaDataName = "x-ms-meta-" + metaDataName;
headers.Add(metaDataName, item.Value);
}
if (blobName == string.Empty)
{
retVal = Containers(cmd, containerName, headers, "?comp=metadata");
retVal.Succeeded = (retVal.StatusCode == HttpStatusCode.OK);
}
else
{
retVal = Blobs(cmd, containerName, blobName, headers, "?comp=metadata");
}
return retVal;
}
}
}