一個小需求，獲取遠程頁面的源碼，主要用于抓數據。原來用的好好的，最近突然不能獲取頁面源碼了，但是仍然可以用瀏覽器正常瀏覽。（文后附源碼下載。^_^）

　　經過分析，原來用的代碼如下：

StreamReader sreader = null ; string result = string .Empty; try { HttpWebRequest httpWebRequest = (HttpWebRequest)WebRequest.Create(Url); // httpWebRequest.Timeout = 20; httpWebRequest.KeepAlive = false ; #endregion HttpWebResponse httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse(); if (httpWebResponse.StatusCode == HttpStatusCode.OK) { sreader = new StreamReader(httpWebResponse.GetResponseStream(), encoding); result = reader.ReadToEnd(); if ( null != httpWebResponse) { httpWebResponse.Close(); } return result; } return result; ; } catch (WebException e) { return null ; } finally { if (sreader != null ) { sreader.Close(); } }

查了下資料，原來需要加參數。

　#region 關鍵參數，否則會取不到內容　Important Parameters,else get nothing. httpWebRequest.UserAgent = "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)"; httpWebRequest.Accept = "*/*"; httpWebRequest.KeepAlive = true; httpWebRequest.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5"); #endregion


修正后的代碼如下：

#region 讀取頁面詳細信息 /**/ /// <summary> /// 讀取頁面詳細信息 /// </summary> /// <param name="Url"> 需要讀取的地址 </param> /// <param name="encoding"> 讀取的編碼方式 </param> /// <returns></returns> public static string GetStringByUrl( string Url, System.Text.Encoding encoding) { if (Url.Equals( " about:blank " )) return null ; ; if ( ! Url.StartsWith( " http:// " ) && ! Url.StartsWith( " https:// " )) { Url = " http:// " + Url; } int dialCount = 0 ; loop: StreamReader sreader = null ; string result = string .Empty; try { HttpWebRequest httpWebRequest = (HttpWebRequest)WebRequest.Create(Url); // httpWebRequest.Timeout = 20; 關鍵參數，否則會取不到內容　Important Parameters,else get nothing. #region 關鍵參數，否則會取不到內容　Important Parameters,else get nothing. httpWebRequest.UserAgent = " User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;) " ; httpWebRequest.Accept = " */* " ; httpWebRequest.KeepAlive = true ; httpWebRequest.Headers.Add( " Accept-Language " , " zh-cn,en-us;q=0.5 " ); #endregion HttpWebResponse httpWebResponse = (HttpWebResponse)httpWebRequest.GetResponse(); if (httpWebResponse.StatusCode == HttpStatusCode.OK) { sreader = new StreamReader(httpWebResponse.GetResponseStream(), encoding); char [] cCont = new char [ 256 ]; int count = sreader.Read(cCont, 0 , 256 ); while (count > 0 ) { // Dumps the 256 characters on a string and displays the string to the console. String str = new String(cCont, 0 , count); result += str; count = sreader.Read(cCont, 0 , 256 ); } } if ( null != httpWebResponse) { httpWebResponse.Close(); } return result; } catch (WebException e) { if (e.Status == WebExceptionStatus.ConnectFailure) { dialCount ++ ; ReDial(); } if (dialCount < 5 ) { goto loop; } return null ; } finally { if (sreader != null ) { sreader.Close(); } } } #endregion public static void ReDial() { int res = 1 ; /**/ /// /while (res != 0) /// /{ /// / CSDNWebTest.RASDisplay ras = new RASDisplay(); /// / ras.Disconnect(); /// / res = ras.Connect("asdl"); /// / System.Threading.Thread.Sleep(TimeSpan.FromSeconds(10)); /// /} }

問題是解決了，后來再想了想，可以用WebClient先把頁面download到本地臨時文件，再讀取文本內容。

代碼如下：


private string GetPageByWebClient( string url) { string result = null ; if (url.Equals( " about:blank " )) return null ; ; if ( ! url.StartsWith( " http:// " ) && ! url.StartsWith( " https:// " )) { url = " http:// " + url; } string filename = RandomKey( 1111 , 9999 ) + " .txt " ; DownloadOneFileByURLWithWebClient(filename, url, " C:// " ); StreamReader sr = new StreamReader( " c:// " + filename, System.Text.Encoding.Default); try { result = sr.ReadToEnd(); return result; } catch { return null ; } finally { if (sr != null ) { sr.Close(); } } } private string RandomKey( int b, int e) { return DateTime.Now.ToString( " yyyyMMdd-HHmmss-fff- " ) + this .getRandomID(b, e); } private int getRandomID( int minValue, int maxValue) { Random ri = new Random( unchecked (( int )DateTime.Now.Ticks)); int k = ri.Next(minValue, maxValue); return k; } private string GuidString { get { return Guid.NewGuid().ToString(); } } 　　　　 /**/ /// Web Client Method ,only For Small picture /// </summary> /// <param name="fileName"></param> /// <param name="url"></param> /// <param name="localPath"></param> public static void DownloadOneFileByURLWithWebClient( string fileName, string url, string localPath) { System.Net.WebClient wc = new System.Net.WebClient(); if (File.Exists(localPath + fileName)) { File.Delete(localPath + fileName); } if (Directory.Exists(localPath) == false ) { Directory.CreateDirectory(localPath); } wc.DownloadFile(url + fileName, localPath + fileName); }


結果不能獲取源碼。錯誤如下：

再想想，還有Webbrowser控件可以用啊。在WinFrom下只要在主線程前加[STAThread]即可。

[STAThread] public void GetURLContentByWebBrowser() { try { // webBrowser1 = new WebBrowser(); string url = txtUrl.Text.Trim(); string result = null ; WebBrowser wb = new WebBrowser(); /**/ /// /if (wb != null){ wb.DocumentCompleted += new WebBrowserDocumentCompletedEventHandler(wb_DocumentCompleted); } if (String.IsNullOrEmpty(url)) return ; if (url.Equals( " about:blank " )) return ; ; if ( ! url.StartsWith( " http:// " ) && ! url.StartsWith( " https:// " )) { url = " http:// " + url; } try { wb.Navigate( new Uri(url)); result = wb.DocumentText; lbResult.Text = result; } catch (System.UriFormatException) { } return ; } catch (Exception ex) { // WriteLog.Writelog("這是獲取頁面全部html代碼時發(fā)生的錯誤：" + url, ex); throw ex; // return ; } }


在WebForm就麻煩些了，出現錯誤， 線程不在單線程單元中，故無法實例化ActiveX控件“8856f961-340a-11d0-a96b-00c04fd705a2”



代碼如下：
private string GetPageStringbyWebBrowser( string url) { if (url.Equals( " about:blank " )) return null ; ; if ( ! url.StartsWith( " http:// " ) && ! url.StartsWith( " https:// " )) { url = " http:// " + url; } WebBrowser myWB = new WebBrowser(); myWB.ScrollBarsEnabled = false ; myWB.Navigate(url); while (myWB.ReadyState != WebBrowserReadyState.Complete) { System.Windows.Forms.Application.DoEvents(); } if (myWB != null ) { System.IO.StreamReader getReader = null ; try { getReader = new System.IO.StreamReader(myWB.DocumentStream, System.Text.Encoding.GetEncoding(myWB.Document.Encoding)); string gethtml = getReader.ReadToEnd(); return gethtml; } catch { return null ; } finally { if (getReader != null ) { getReader.Close(); } myWB.Dispose(); } } return null ; }

后來搜索N小時(N>=5)后，終于找到可行解決方案,在WebPage頁面頭部加入 AspCompat="true"

即<%@ Page Language="C#" AspCompat="true" ******/>

MSDN給出的解釋是：
在 ASP .NET 網頁的 <%@Page> 標記中包含兼容性屬性 aspcompat=true，如 <%@Page aspcompat=true Language=VB%>。使用此屬性將強制網頁以 STA 模式執(zhí)行，從而確保您的組件可以繼續(xù)正確運行。如果試圖使用 STA 組件但沒有指定此標記，運行時將會發(fā)生異常情況。

將此屬性的值設置為 true 時，將允許網頁調用 COM+ 1.0 組件，該組件需要訪問非管理的 ASP 內置對象?？梢酝ㄟ^ ObjectContext 對象進行訪問。

如果將此標記的值設為 true，性能會稍微有些下降。建議只在確實需要時才這樣做。

終于可以了！　不知道有沒有更好的方法？？

附： 源碼下載。

邀月注：

如果不能測試，請注意是否在域（AD)環(huán)境下，如果是！　請注意設置代理和防火墻
請參考：
http://dev.csdn.net/article/83914.shtm

或 http://blog.csdn.net/downmoon/archive/2006/04/14/663337.aspx

或 http://www.cnblogs.com/downmoon/archive/2007/12/29/1019701.html

在webForm中WebRequest/WebClient/WebBrowser獲取遠程頁面源碼的三種方式（downmoon)