// WebDownload.cs
//
// Provides wrappers around WebRequest to handle commonly-used tasks:
// - Offline mode
// - ETag stamps
// - Content-encoding issues
// - Timeout setting
//
// Code written for use by Aggie.
namespace Bitworking{
using System;
using System.Net;
using System.IO;
using System.Text;
/// <summary>
/// Holds configuration information for the web download component.
/// </summary>
public struct WebConnectionConfigInfo {
/// <summary>
/// If true, the web download component works in offline mode --
/// all web requests are ignored, and pretended as if successful.
/// </summary>
public bool offline;
/// <summary>
/// Default timeout value for downloads.
/// </summary>
public int timeout_ms;
} // struct WebConnectionConfigInfo
public class WebDownloadConfig {
public int timeout_ms;
public string userAgent;
public string referer;
} // class WebDownloadConfig
public class WebDownload {
static public void DumpWebHeaders( WebHeaderCollection headers ) {
bool dumpHeaders = false;
if ( dumpHeaders ) {
System.Diagnostics.Debug.WriteLine( "--- Dumping headers --- " );
foreach ( string key in headers ) {
System.Diagnostics.Debug.WriteLine( "Header[" + key + "]=" + headers[key] );
}
}
} // DumpWebHeaders
#region Static download functions
static public WebResponse MakeRequestGetResponse( System.Uri uri, int timeout, string userAgent, string etag ) {
WebRequest req = WebRequest.Create( uri );
req.Timeout = timeout;
HttpWebRequest wreq = req as HttpWebRequest;
if ( wreq != null ) {
// If this is an HTTP request, we can also set userAgent and etag
wreq.UserAgent = userAgent;
if ( etag != null && etag != "" )
wreq.Headers.Add( "If-None-Match", etag );
}
WebResponse resp = req.GetResponse();
DumpWebHeaders( resp.Headers );
return resp;
} // MakeRequestGetResponse
static public WebResponse MakeRequestGetResponse( System.Uri uri, WebDownloadConfig context, string etag ) {
WebRequest req = WebRequest.Create( uri );
req.Timeout = context.timeout_ms;
HttpWebRequest wreq = req as HttpWebRequest;
if ( wreq != null ) {
// If this is an HTTP request, we can also set HTTP-specific attributes
wreq.UserAgent = context.userAgent;
wreq.Referer = context.referer;
if ( etag != null && etag != "" )
wreq.Headers.Add( "If-None-Match", etag );
}
WebResponse resp = req.GetResponse();
DumpWebHeaders( resp.Headers );
return resp;
} // MakeRequestGetResponse
static public System.IO.Stream OpenForDownload( Uri what, int timeout ) {
WebResponse resp = MakeRequestGetResponse( what, timeout, "", "" );
return resp.GetResponseStream();
// Note: We return a Stream, which must be Close()-ed. When our
// caller closes the stream, resp is automatically closed as well.
} // OpenForDownload
static public string DownloadToString( string url, int timeout ) {
string etag = "";
return DownloadToString( url, timeout, "", ref etag );
} // DownloadToString( string, int )
static public string DownloadToString( string url, int timeout, string userAgent, ref string etag ) {
string resource;
WebResponse resp = null;
try {
// Make the request and hold on to the response that it brings back
resp = MakeRequestGetResponse( new System.Uri( url ), timeout, userAgent, etag );
// If we got back an ETAG, we hold on to it
string respEtag = resp.Headers.Get( "ETag" );
if ( respEtag != null ) {
etag = respEtag;
}
// TODO: If resp tells us what is the content encoding,
// we use that. If not, we're in a bit of a problem, because
// different streams may have different encoding methods,
// and one needs to make a large matrix of possible cases and
// responses. Instead, what we do here is to ASSUME ISO-8859-1.
// In the future, we may want to revisit this decision (for example,
// we might want to add an explicit test for MIME type text/html and
// check the content encoding type.
Encoding defaultEncoding = Encoding.GetEncoding( "ISO-8859-1" );
resource = GetResponseString( resp, defaultEncoding, true );
}
finally {
if ( resp != null )
resp.Close();
}
return resource;
} // DownloadToString
static public string DownloadToString( string url, WebDownloadConfig context ) {
string etag = "";
return DownloadToString( url, context, ref etag );
}
static public string DownloadToString( string url, WebDownloadConfig context, ref string etag ) {
if ( context == null )
throw new ArgumentNullException( "context", "Download context cannot be null" );
string resource;
WebResponse resp = null;
try {
// Make the request and hold on to the response that it brings back
resp = MakeRequestGetResponse( new System.Uri( url ), context, etag );
// If we got back an ETAG, we hold on to it
string respEtag = resp.Headers.Get( "ETag" );
if ( respEtag != null ) {
etag = respEtag;
}
resource = GetResponseString( resp, Encoding.GetEncoding( "ISO-8859-1" ), true );
}
finally {
if ( resp != null )
resp.Close();
}
return resource;
} // DownloadToString
static public string GetResponseString( WebResponse resp, System.Text.Encoding defaultEncoding, bool closeWhenDone ) {
// Determine the encoding
System.Text.Encoding encoding = defaultEncoding;
string contentEncoding = resp.Headers.Get( "Content-Encoding" );
if ( contentEncoding != null && contentEncoding != "" ) {
try {
encoding = System.Text.Encoding.GetEncoding( contentEncoding );
}
catch ( NotSupportedException ) {
System.Diagnostics.Debug.WriteLine( "Encoding " + contentEncoding + " unrecognized." );
}
}
// Convert the stream we got back into a string
Stream stream = resp.GetResponseStream();
StreamReader reader = null;
string str;
try {
reader = new StreamReader( stream, encoding );
// We need this because the stream might have been already
// read by a debugging routine. Because debugging routines
// might be all over the place, it's best we are prepared to
// roll-back the stream instead of requiring them to do that.
if ( stream.CanSeek )
stream.Seek( 0, SeekOrigin.Begin );
str = reader.ReadToEnd();
}
finally {
if ( closeWhenDone && reader != null )
reader.Close();
}
// Roll-back the stream if required
if ( stream.CanSeek )
stream.Seek( 0, SeekOrigin.Begin );
return str;
} // GetResponseString
#endregion
#region Static proxy functions
static public void EstablishProxy( WebConnectionConfigInfo config, string[] proxyServers ) {
if ( config.offline )
return;
// This is currently a hack. Instead of finding-out what
// connection is currently used and derive the proxy we
// need from that, we simply try each proxy in our list
// until we have a success.
foreach ( string proxy in proxyServers ) {
bool success = false;
// Set global proxy
if ( proxy != "##default##" ) {
if ( proxy == "" ) {
// Explicitly disable default proxy
System.Net.GlobalProxySelection.Select = System.Net.GlobalProxySelection.GetEmptyWebProxy();
}
else {
// Use whatever the config file says
System.Net.WebProxy proxyObject = new System.Net.WebProxy( proxy );
System.Net.GlobalProxySelection.Select = proxyObject;
}
}
// Now attempt to use that proxy
StreamReader reader = null;
try {
// We detect connectivity to the one site that is built to handle load
WebRequest req = WebRequest.Create( "http://www.google.com/" );
WebResponse resp = req.GetResponse();
reader = new StreamReader( resp.GetResponseStream(), Encoding.ASCII );
string resource = reader.ReadToEnd();
success = true;
}
catch ( WebException webEx ) {
HttpWebResponse webResp = webEx.Response as HttpWebResponse;
if ( webResp != null ) {
HttpStatusCode code = webResp.StatusCode;
}
}
finally {
if ( reader != null )
reader.Close();
}
if ( success )
break;
} // foreach proxy
} // EstablishProxy
#endregion
#region Private instance data
private WebConnectionConfigInfo config_;
private string url_;
#endregion
#region Construction
public WebDownload( WebConnectionConfigInfo config, string url ) {
/* TODO: Would it be better if WebConnectionConfigInfo be a class?
if ( config == null )
{
config = new WebConnectionConfigInfo();
config.offline = false;
config.timeout_ms = System.Threading.Timeout.Infinite;
}
*/
config_ = config;
if ( url == null || url == "" )
throw new ArgumentNullException( "url", "URL to download cannot be null or empty" );
url_ = url;
}
#endregion
#region Download (instance) routines
public bool DownloadToFile(
ref string filename, // Old filename (may be null!)
ref string etag,
ref string comments,
string newFilename
) {
StreamWriter writer = null;
bool success = true;
try {
if ( config_.offline ) {
// In offline, we behave as if everything is successfully cached,
// even if it's not so
filename = newFilename;
return true;
}
// If we have both an etag and a valid old file to go with it,
// we can send the etag. Otherwise, insist on getting back a file.
if ( etag != null && etag != "" )
if ( !File.Exists( filename ) )
etag = "";
string resource = DownloadToString( url_, config_.timeout_ms, VersionInfo.UserAgent, ref etag );
filename = newFilename;
File.Delete( filename );
writer = new StreamWriter( File.OpenWrite( filename ) ); // By default, UTF-8
writer.Write( resource );
}
catch ( WebException webEx ) {
HttpWebResponse webResp = webEx.Response as HttpWebResponse;
if ( webResp != null && webResp.StatusCode == HttpStatusCode.NotModified ) {
comments = "Channel has not changed since last read.";
}
else {
success = false;
comments = webEx.Message;
}
}
catch ( Exception ex ) {
success = false;
comments = ex.Message;
}
finally {
if ( writer != null )
writer.Close();
}
// TODO: throw in case of an error?
return success;
} // DownloadToFile
#endregion
} // class WebDownload
} // namespace Bitworking
|