C# HTTP GET 请求到 http://sede.educaion.gob.es/

Question

我正在尝试向https://sede.educacion.gob.es/publiventa/catalogo.action?cod=E 执行 GET 请求；使用 cod=E 参数，在浏览器中，网站会在“Materias de educación”下方打开一个菜单，但是当我使用 C# 执行请求时，该菜单没有加载，我需要它。这是我用来将 Html 读取为字符串以便稍后使用 HtmlAgilityPack 解析它的代码。

private string readHtml(string urlAddress)
    {
        HttpWebRequest request = (HttpWebRequest)WebRequest.Create(urlAddress);
        request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0";
        request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
        request.AutomaticDecompression = DecompressionMethods.GZip;
        HttpWebResponse response = (HttpWebResponse)request.GetResponse();

        if (response.StatusCode == HttpStatusCode.OK)
        {
            Stream receiveStream = response.GetResponseStream();
            StreamReader readStream = null;

            if (response.CharacterSet == null)
            {
                readStream = new StreamReader(receiveStream);
            }
            else
            {
                readStream = new StreamReader(receiveStream, Encoding.GetEncoding(response.CharacterSet));
            }

            string data = readStream.ReadToEnd();

            response.Close();
            readStream.Close();
            return data;
        }
        return null;
    }

Answer 1

您发布的 Uri (https://sede.educacion.gob.es/publiventa/catalogo.action?cod=E) 使用 Javascript 开关来显示菜单内容。
当您连接到该 Uri（不单击菜单链接）时，该站点会显示该页面的三个不同版本。

1) 关闭菜单和建议的新版本的页面
2) 关闭菜单和搜索引擎字段的页面
3) 带有打开菜单和选择菜单内容的页面

此开关基于记录当前会话的内部过程。除非您单击菜单链接（连接到事件侦听器），否则 Javascript 过程会以不同状态显示页面。
我看了看；这些脚本很长（一个完整的多用途库），我没有时间全部解析（也许你可以这样做）以找出事件监听器传递的参数。

但是，三态版本切换是不变的。
我的意思是您可以调用该页面三次，保留 Cookie 容器：第三次连接到它时，它将流式传输整个菜单内容及其链接。

如果你请求同一个页面三次，Html页面第三次会 包含所有
Materias de educación 链接

public async void SomeMethodAsync()
{
    string HtmlPage = await GetHttpStream([URI]);
    HtmlPage = await GetHttpStream([URI]);
    HtmlPage = await GetHttpStream([URI]);
}

这或多或少是我用来获取该页面的内容：

CookieContainer CookieJar = new CookieContainer();

public async Task<string> GetHttpStream(Uri HtmlPage)
{
    HttpWebRequest httpRequest;
    string Payload = string.Empty;
    httpRequest = WebRequest.CreateHttp(HtmlPage);

    try
    {
        httpRequest.CookieContainer = CookieJar;
        httpRequest.KeepAlive = true;
        httpRequest.ConnectionGroupName = Guid.NewGuid().ToString();
        httpRequest.AllowAutoRedirect = true;
        httpRequest.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
        httpRequest.ServicePoint.MaxIdleTime = 30000;
        httpRequest.ServicePoint.Expect100Continue = false;
        httpRequest.UserAgent = "Mozilla/5.0 (Windows NT 10; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0";
        httpRequest.Accept = "ext/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
        httpRequest.Headers.Add(HttpRequestHeader.AcceptLanguage, "es-ES,es;q=0.8,en-US;q=0.5,en;q=0.3");
        httpRequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip, deflate;q=0.8");
        httpRequest.Headers.Add(HttpRequestHeader.CacheControl, "no-cache");


        using (HttpWebResponse httpResponse = (HttpWebResponse)await httpRequest.GetResponseAsync())
        {
            Stream ResponseStream = httpResponse.GetResponseStream();

            if (httpResponse.StatusCode == HttpStatusCode.OK)
            {
                try
                {
                    //ResponseStream.Position = 0;
                    Encoding encoding = Encoding.GetEncoding(httpResponse.CharacterSet);

                    using (MemoryStream _memStream = new MemoryStream())
                    {
                        if (httpResponse.ContentEncoding.Contains("gzip"))
                        {
                            using (GZipStream _gzipStream = new GZipStream(ResponseStream, System.IO.Compression.CompressionMode.Decompress))
                            {
                                _gzipStream.CopyTo(_memStream);
                            };
                        }
                        else if (httpResponse.ContentEncoding.Contains("deflate"))
                        {
                            using (DeflateStream _deflStream = new DeflateStream(ResponseStream, System.IO.Compression.CompressionMode.Decompress))
                            {
                                _deflStream.CopyTo(_memStream);
                            };
                        }
                        else
                        {
                            ResponseStream.CopyTo(_memStream);
                        }

                        _memStream.Position = 0;
                        using (StreamReader _reader = new StreamReader(_memStream, encoding))
                        {
                            Payload = _reader.ReadToEnd().Trim();
                        };
                    };
                }
                catch (Exception)
                {
                    Payload = string.Empty;
                }
            }
        }
    }
    catch (WebException exW)
    {
        if (exW.Response != null)
        {
            //Handle WebException
        }
    }
    catch (System.Exception exS)
    {
        //Handle System.Exception
    }

    CookieJar = httpRequest.CookieContainer;
    return Payload;
}