WebScrape using Regex, C#

WebScrape using Regex

Language: Layout:

//Rextester.Program.Main is the entry point for your code. Don't change it.
//Compiler version 4.0.30319.17929 for Microsoft (R) .NET Framework 4.5

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Net;

namespace Rextester
{
    public class Program
    {
        public static void Main(string[] args)
        {
            // 1.
            // URL: http://en.wikipedia.org/wiki/Main_Page
            WebClient w = new WebClient();
            string s = w.DownloadString("http://en.wikipedia.org/wiki/Main_Page");

// 2.
            foreach (LinkItem i in LinkFinder.Find(s))
            {
                Console.WriteLine(i);
            }
        }
    }
    
    public struct LinkItem
    {
        public string Href;
        public string Text;

public override string ToString()
        {
            return Href + "\n\t" + Text;
        }
    }

static class LinkFinder
    {
        public static List<LinkItem> Find(string file)
        {
            List<LinkItem> list = new List<LinkItem>();

// 1.
            // Find all matches in file.
            MatchCollection m1 = Regex.Matches(file, @"(<a.*?>.*?</a>)",
                RegexOptions.Singleline);

// 2.
            // Loop over each match.
            foreach (Match m in m1)
            {
                string value = m.Groups[1].Value;
                LinkItem i = new LinkItem();

// 3.
                // Get href attribute.
                Match m2 = Regex.Match(value, @"href=\""(.*?)\""",
                    RegexOptions.Singleline);
                if (m2.Success)
                {
                    i.Href = m2.Groups[1].Value;
                }

// 4.
                // Remove inner tags from text.
                string t = Regex.Replace(value, @"\s*<.*?>\s*", "",
                    RegexOptions.Singleline);
                i.Text = t;

list.Add(i);
            }
            return list;
        }
    }
}

Show compiler warnings [ + ] Show input
Compilation time: 0,16 sec, absolute running time: 0,98 sec, cpu time: 0,86 sec, average memory usage: 28 Mb, average nr of threads: 9

#mw-head navigation #p-search search /wiki/Wikipedia Wikipedia /wiki/Free_content free /wiki/Encyclopedia encyclopedia /wiki/Wikipedia:Introduction anyone can edit /wiki/Special:Statistics 5,620,330 /wiki/English_language English /wiki/Portal:Arts Arts /wiki/Portal:Biography Biography /wiki/Portal:Geography Geography /wiki/Portal:History History /wiki/Portal:Mathematics Mathematics /wiki/Portal:Science Science /wiki/Portal:Society Society /wiki/Portal:Technology Technology /wiki/Portal:Contents/Portals All portals //en.wikipedia.org/wiki/The_Destroying_Angel_and_Daemons_of_Evil_Interrupting_the_Orgies_of_the_Vicious_and_Intemperate#/media/File:The_Destroying_Angel_and_Daemons_of_Evil_Interrupting_the_Orgies_of_the_Vicious_and_Intemperate,_William_Etty,_1832.jpg //en.wikipedia.org/wiki/The_Destroying_Angel_and_Daemons_of_Evil_Interrupting_the_Orgies_of_the_Vicious_and_Intemperate#/media/File:The_Destroying_Angel_and_Daemons_of_Evil_Interrupting_the_Orgies_of_the_Vicious_and_Intemperate,_William_Etty,_1832.jpg //en.wikipedia.org/wiki/The_Destroying_Angel_and_Daemons_of_Evil_Interrupting_the_Orgies_of_the_Vicious_and_Intemperate#/media/File:The_Destroying_Angel_and_Daemons_of_Evil_Interrupting_the_Orgies_of_the_Vicious_and_Intemperate,_William_Etty,_1832.jpg Expand /wiki/The_Destroying_Angel_and_Daemons_of_Evil_Interrupting_the_Orgies_of_the_Vicious_and_Intemperate The Destroying Angel and Daemons of Evil Interrupting the Orgies of the Vicious and Intemperate /wiki/Oil_painting oil painting /wiki/William_Etty William Etty /wiki/Nude_(art) nude paintings /wiki/Destroying_angel_(Bible) destroying angel /wiki/Daemon_(classical_mythology) daemons /wiki/Joseph_Whitworth Joseph Whitworth /wiki/Manchester_Art_Gallery Manchester Art Gallery /wiki/The_Destroying_Angel_and_Daemons_of_Evil_Interrupting_the_Orgies_of_the_Vicious_and_Intemperate Full article... /wiki/Gloucestershire_Regiment Gloucestershire Regiment /wiki/Neferirkare_Kakai Neferirkare Kakai /wiki/Knuckles%27_Chaotix Knuckles' Chaotix /wiki/Wikipedia:Today%27s_featured_article/April_2018 Archive https://lists.wikimedia.org/mailman/listinfo/daily-article-l By email /wiki/Wikipedia:Featured_articles More featured articles /wiki/File:Obendorf_Barn_-_Wilder_Idaho.jpg /wiki/Gothic-arch_barn Gothic-arch barn /wiki/Sears Sears & Roebuck catalog /wiki/Pedro_Zaragoza Pedro Zaragoza /wiki/Excommunication_(Catholic_Church) excommunication /wiki/Bikini bikinis /wiki/Benidorm Benidorm /wiki/Parvulastra_vivipara Parvulastra vivipara /wiki/Minkhaung_Medaw Minkhaung Medaw /wiki/Prome_Kingdom Prome /wiki/Hanthawaddy_Kingdom Hanthawaddy /wiki/Kingdom_of_Mrauk_U Mrauk U /wiki/George_Town,_Penang George Town /wiki/Penang Penang /wiki/Richard_Hamming Richard Hamming /wiki/Nobel_Prize Nobel Prize /wiki/Nobel_Prize_effect no longer work on small problems /wiki/Neomanila Neomanila /wiki/Contract_killing hitmen /wiki/New_Year%27s_Day New Year's Day /wiki/First_Day_Hikes take a hike /wiki/Wikipedia:Recent_additions Archive /wiki/Wikipedia:Your_first_article Start a new article /wiki/Template_talk:Did_you_know Nominate an article /wiki/File:Miguel_Diaz_Canel.jpg /wiki/Miguel_D%C3%ADaz-Canel Miguel DÃaz-Canel /wiki/Ra%C3%BAl_Castro RaÃºl Castro /wiki/President_of_Cuba President of Cuba /wiki/Horse_racing horse racing /wiki/Tiger_Roll Tiger Roll /wiki/2018_Grand_National Grand National /wiki/Aintree_Racecourse Aintree Racecourse /wiki/2018_bombing_of_Damascus_and_Homs strike /wiki/Douma_chemical_attack chemical weapons attack /wiki/Douma,_Syria Douma /wiki/Algerian_Air_Force Algerian Air Force /wiki/Ilyushin_Il-76 Ilyushin Il-76 /wiki/2018_Algerian_Air_Force_Il-76_crash crashes /wiki/Deaths_in_2018 Recent deaths /wiki/Nabi_Tajima Nabi Tajima /wiki/Verne_Troyer Verne Troyer /wiki/Avicii Avicii /wiki/Agn%C3%A8s-Marie_Valois AgnÃ¨s-Marie Valois /wiki/Portal:Current_events Other recent events /wiki/Wikipedia:In_the_news/Candidates Nominate an article /wiki/April_23 April 23 /wiki/File:New_Coke_can.jpg /wiki/1516 1516 /wiki/Reinheitsgebot Reinheitsgebot /wiki/Duchy_of_Bavaria Bavaria /wiki/1661 1661 /wiki/Charles_II_of_England Charles II /wiki/Westminster_Abbey Westminster Abbey /wiki/1918 1918 /wiki/World_War_I First World War /wiki/Royal_Navy Royal Navy /wiki/Zeebrugge_Raid conducted an unsuccessful raid /wiki/Port_of_Bruges-Zeebrugge port of Bruges-Zeebrugge /wiki/1951 1951 /wiki/William_N._Oatis William N. Oatis /wiki/Espionage espionage /wiki/1985 1985 /wiki/The_Coca-Cola_Company The Coca-Cola Company /wiki/Coca-Cola Coca-Cola /wiki/New_Coke New Coke /wiki/William_Shakespeare William Shakespeare /wiki/Karl_Friedrich_Bahrdt Karl Friedrich Bahrdt /wiki/Kathy_Lynch Kathy Lynch /wiki/April_22 April 22 /wiki/April_23 April 23 /wiki/April_24 April 24 /wiki/Wikipedia:Selected_anniversaries/April Archive https://lists.wikimedia.org/mailman/listinfo/daily-article-l By email /wiki/List_of_historical_anniversaries List of historical anniversaries /wiki/File:QueenRegine.jpg /wiki/List_of_songs_recorded_by_Regine_Velasquez Regine Velasquez has recorded more than 300 songs /wiki/Regine_(album) Regine /wiki/Polygram Polygram /wiki/Listen_Without_Prejudice_(Regine_Velasquez_album) Listen Without Prejudice /wiki/Southeast_Asia Southeast /wiki/East_Asia East Asia /wiki/R2K R2K /wiki/List_of_best-selling_albums_in_the_Philippines best-selling album by a solo female artist /wiki/Covers_(Regine_Velasquez_album) Covers. Vol 1 /wiki/Covers_Volume_2 Covers, Vol. 2 /wiki/Low_Key Low Key /wiki/Fantasy_(Regine_Velasquez_album) Fantasy /wiki/Regine_Velasquez Velasquez /wiki/List_of_songs_recorded_by_Regine_Velasquez Full list... /wiki/Wright_Brothers_Memorial_Trophy Wright Brothers Memorial Trophy /wiki/List_of_accolades_received_by_Zootopia Accolades received byZootopia /wiki/Municipalities_of_Aguascalientes Municipalities of Aguascalientes /wiki/Wikipedia:Today%27s_featured_list/April_2018 Archive /wiki/Wikipedia:Featured_lists More featured lists /wiki/File:Pippin-Atmark-Console-Set.jpg /wiki/Apple_Bandai_Pippin Apple Bandai Pippin /wiki/Apple_Inc. Apple Computer /wiki/Apple_Pippin Apple Pippin /wiki/Bandai Bandai /wiki/Thin_client thin client /wiki/User:Evan-Amos Evan Amos /wiki/Template:POTD/2018-04-22 Madonna of the Rose Garden /wiki/Template:POTD/2018-04-21 Runway /wiki/Template:POTD/2018-04-20 NGC 1427A /wiki/Wikipedia:Picture_of_the_day/April_2018 Archive /wiki/Wikipedia:Featured_pictures More featured pictures /wiki/Wikipedia:Community_portal Community portal /wiki/Wikipedia:Help_desk Help desk /wiki/Wikipedia:Local_Embassy Local embassy /wiki/Wikipedia:Reference_desk Reference desk /wiki/Wikipedia:News Site news /wiki/Wikipedia:Village_pump Village pump /wiki/Wikimedia_Foundation Wikimedia Foundation https://wikimediafoundation.org/wiki/Our_projects projects https://commons.wikimedia.org/wiki/ //commons.wikimedia.org/ Commons https://www.mediawiki.org/wiki/ //mediawiki.org/ MediaWiki https://meta.wikimedia.org/wiki/ //meta.wikimedia.org/ Meta-Wiki https://en.wikibooks.org/wiki/ //en.wikibooks.org/ Wikibooks https://www.wikidata.org/wiki/ //www.wikidata.org/ Wikidata https://en.wikinews.org/wiki/ //en.wikinews.org/ Wikinews https://en.wikiquote.org/wiki/ //en.wikiquote.org/ Wikiquote https://en.wikisource.org/wiki/ //en.wikisource.org/ Wikisource https://species.wikimedia.org/wiki/ //species.wikimedia.org/ Wikispecies https://en.wikiversity.org/wiki/ //en.wikiversity.org/ Wikiversity https://en.wikivoyage.org/wiki/ //en.wikivoyage.org/ Wikivoyage https://en.wiktionary.org/wiki/ //en.wiktionary.org/ Wiktionary /wiki/English_language English /wiki/Special:Statistics 5,620,330 https://de.wikipedia.org/wiki/ Deutsch https://es.wikipedia.org/wiki/ EspaÃ±ol https://fr.wikipedia.org/wiki/ FranÃ§ais https://it.wikipedia.org/wiki/ Italiano https://nl.wikipedia.org/wiki/ Nederlands https://ja.wikipedia.org/wiki/ æ—¥æœ¬èªž https://pl.wikipedia.org/wiki/ Polski https://ru.wikipedia.org/wiki/ Ð ÑƒÑÑÐºÐ¸Ð¹ https://sv.wikipedia.org/wiki/ Svenska https://vi.wikipedia.org/wiki/ Tiáº¿ng Viá»‡t https://ar.wikipedia.org/wiki/ Ø§Ù„Ø¹Ø±Ø¨ÙŠØ© https://id.wikipedia.org/wiki/ Bahasa Indonesia https://ms.wikipedia.org/wiki/ Bahasa Melayu https://ca.wikipedia.org/wiki/ CatalÃ https://cs.wikipedia.org/wiki/ ÄŒeÅ¡tina https://eu.wikipedia.org/wiki/ Euskara https://fa.wikipedia.org/wiki/ ÙØ§Ø±Ø³ÛŒ https://ko.wikipedia.org/wiki/ í•œêµì–´ https://hu.wikipedia.org/wiki/ Magyar https://no.wikipedia.org/wiki/ Norsk https://pt.wikipedia.org/wiki/ PortuguÃªs https://ro.wikipedia.org/wiki/ RomÃ¢nÄƒ https://sr.wikipedia.org/wiki/ Srpski https://sh.wikipedia.org/wiki/ Srpskohrvatski https://fi.wikipedia.org/wiki/ Suomi https://tr.wikipedia.org/wiki/ TÃ¼rkÃ§e https://uk.wikipedia.org/wiki/ Ð£ÐºÑ€Ð°Ñ—Ð½ÑÑŒÐºÐ° https://zh.wikipedia.org/wiki/ ä¸æ–‡ https://bs.wikipedia.org/wiki/ Bosanski https://bg.wikipedia.org/wiki/ Ð‘ÑŠÐ»Ð³Ð°Ñ€ÑÐºÐ¸ https://da.wikipedia.org/wiki/ Dansk https://et.wikipedia.org/wiki/ Eesti https://el.wikipedia.org/wiki/ Î•Î»Î»Î·Î½Î¹ÎºÎ¬ https://simple.wikipedia.org/wiki/ English (simple form) https://eo.wikipedia.org/wiki/ Esperanto https://gl.wikipedia.org/wiki/ Galego https://he.wikipedia.org/wiki/ ×¢×‘×¨×™×ª https://hr.wikipedia.org/wiki/ Hrvatski https://lv.wikipedia.org/wiki/ LatvieÅ¡u https://lt.wikipedia.org/wiki/ LietuviÅ³ https://nn.wikipedia.org/wiki/ Norsk nynorsk https://sk.wikipedia.org/wiki/ SlovenÄina https://sl.wikipedia.org/wiki/ SlovenÅ¡Äina https://th.wikipedia.org/wiki/ à¹„à¸—à¸¢ https://meta.wikimedia.org/wiki/List_of_Wikipedias Complete list of Wikipedias https://en.wikipedia.org/w/index.php?title=Main_Page&oldid=807996266 https://en.wikipedia.org/w/index.php?title=Main_Page&oldid=807996266 /wiki/Special:MyTalk Talk /wiki/Special:MyContributions Contributions /w/index.php?title=Special:CreateAccount&returnto=Main+Page Create account /w/index.php?title=Special:UserLogin&returnto=Main+Page Log in /wiki/Main_Page Main Page /wiki/Talk:Main_Page Talk /wiki/Main_Page Read /w/index.php?title=Main_Page&action=edit View source /w/index.php?title=Main_Page&action=history View history /wiki/Main_Page /wiki/Main_Page Main page /wiki/Portal:Contents Contents /wiki/Portal:Featured_content Featured content /wiki/Portal:Current_events Current events /wiki/Special:Random Random article https://donate.wikimedia.org/wiki/Special:FundraiserRedirector?utm_source=donate&utm_medium=sidebar&utm_campaign=C13_en.wikipedia.org&uselang=en Donate to Wikipedia //shop.wikimedia.org Wikipedia store /wiki/Help:Contents Help /wiki/Wikipedia:About About Wikipedia /wiki/Wikipedia:Community_portal Community portal /wiki/Special:RecentChanges Recent changes //en.wikipedia.org/wiki/Wikipedia:Contact_us Contact page /wiki/Special:WhatLinksHere/Main_Page What links here /wiki/Special:RecentChangesLinked/Main_Page Related changes /wiki/Wikipedia:File_Upload_Wizard Upload file /wiki/Special:SpecialPages Special pages /w/index.php?title=Main_Page&oldid=807996266 Permanent link /w/index.php?title=Main_Page&action=info Page information https://www.wikidata.org/wiki/Special:EntityPage/Q5296 Wikidata item /w/index.php?title=Special:CiteThisPage&page=Main_Page&id=807996266 Cite this page /w/index.php?title=Special:Book&bookcmd=book_creator&referer=Main+Page Create a book /w/index.php?title=Special:ElectronPdf&page=Main+Page&action=show-download-screen Download as PDF /w/index.php?title=Main_Page&printable=yes Printable version https://commons.wikimedia.org/wiki/Main_Page Wikimedia Commons https://www.mediawiki.org/wiki/MediaWiki MediaWiki https://meta.wikimedia.org/wiki/Main_Page Meta-Wiki https://species.wikimedia.org/wiki/Main_Page Wikispecies https://en.wikibooks.org/wiki/Main_Page Wikibooks https://www.wikidata.org/wiki/Wikidata:Main_Page Wikidata https://en.wikinews.org/wiki/Main_Page Wikinews https://en.wikiquote.org/wiki/Main_Page Wikiquote https://en.wikisource.org/wiki/Main_Page Wikisource https://en.wikiversity.org/wiki/Wikiversity:Main_Page Wikiversity https://en.wikivoyage.org/wiki/Main_Page Wikivoyage https://en.wiktionary.org/wiki/Wiktionary:Main_Page Wiktionary https://ar.wikipedia.org/wiki/ Ø§Ù„Ø¹Ø±Ø¨ÙŠØ© https://bg.wikipedia.org/wiki/ Ð‘ÑŠÐ»Ð³Ð°Ñ€ÑÐºÐ¸ https://bs.wikipedia.org/wiki/ Bosanski https://ca.wikipedia.org/wiki/ CatalÃ https://cs.wikipedia.org/wiki/ ÄŒeÅ¡tina https://da.wikipedia.org/wiki/ Dansk https://de.wikipedia.org/wiki/ Deutsch https://et.wikipedia.org/wiki/ Eesti https://el.wikipedia.org/wiki/ Î•Î»Î»Î·Î½Î¹ÎºÎ¬ https://es.wikipedia.org/wiki/ EspaÃ±ol https://eo.wikipedia.org/wiki/ Esperanto https://eu.wikipedia.org/wiki/ Euskara https://fa.wikipedia.org/wiki/ ÙØ§Ø±Ø³ÛŒ https://fr.wikipedia.org/wiki/ FranÃ§ais https://gl.wikipedia.org/wiki/ Galego https://ko.wikipedia.org/wiki/ í•œêµì–´ https://hr.wikipedia.org/wiki/ Hrvatski https://id.wikipedia.org/wiki/ Bahasa Indonesia https://it.wikipedia.org/wiki/ Italiano https://he.wikipedia.org/wiki/ ×¢×‘×¨×™×ª https://ka.wikipedia.org/wiki/ áƒ¥áƒáƒ áƒ—áƒ£áƒšáƒ˜ https://lv.wikipedia.org/wiki/ LatvieÅ¡u https://lt.wikipedia.org/wiki/ LietuviÅ³ https://hu.wikipedia.org/wiki/ Magyar https://ms.wikipedia.org/wiki/ Bahasa Melayu https://nl.wikipedia.org/wiki/ Nederlands https://ja.wikipedia.org/wiki/ æ—¥æœ¬èªž https://no.wikipedia.org/wiki/ Norsk https://nn.wikipedia.org/wiki/ Norsk nynorsk https://pl.wikipedia.org/wiki/ Polski https://pt.wikipedia.org/wiki/ PortuguÃªs https://ro.wikipedia.org/wiki/ RomÃ¢nÄƒ https://ru.wikipedia.org/wiki/ Ð ÑƒÑÑÐºÐ¸Ð¹ https://simple.wikipedia.org/wiki/ Simple English https://sk.wikipedia.org/wiki/ SlovenÄina https://sl.wikipedia.org/wiki/ SlovenÅ¡Äina https://sr.wikipedia.org/wiki/ Ð¡Ñ€Ð¿ÑÐºÐ¸ / srpski https://sh.wikipedia.org/wiki/ Srpskohrvatski / ÑÑ€Ð¿ÑÐºÐ¾Ñ…Ñ€Ð²Ð°Ñ‚ÑÐºÐ¸ https://fi.wikipedia.org/wiki/ Suomi https://sv.wikipedia.org/wiki/ Svenska https://th.wikipedia.org/wiki/ à¹„à¸—à¸¢ https://tr.wikipedia.org/wiki/ TÃ¼rkÃ§e https://uk.wikipedia.org/wiki/ Ð£ÐºÑ€Ð°Ñ—Ð½ÑÑŒÐºÐ° https://vi.wikipedia.org/wiki/ Tiáº¿ng Viá»‡t https://zh.wikipedia.org/wiki/ ä¸æ–‡ //en.wikipedia.org/wiki/Wikipedia:Text_of_Creative_Commons_Attribution-ShareAlike_3.0_Unported_License Creative Commons Attribution-ShareAlike License //creativecommons.org/licenses/by-sa/3.0/ //wikimediafoundation.org/wiki/Terms_of_Use Terms of Use //wikimediafoundation.org/wiki/Privacy_policy Privacy Policy //www.wikimediafoundation.org/ Wikimedia Foundation, Inc. https://wikimediafoundation.org/wiki/Privacy_policy Privacy policy /wiki/Wikipedia:About About Wikipedia /wiki/Wikipedia:General_disclaimer Disclaimers //en.wikipedia.org/wiki/Wikipedia:Contact_us Contact Wikipedia https://www.mediawiki.org/wiki/Special:MyLanguage/How_to_contribute Developers https://wikimediafoundation.org/wiki/Cookie_statement Cookie statement //en.m.wikipedia.org/w/index.php?title=Main_Page&mobileaction=toggle_view_mobile Mobile view https://wikimediafoundation.org/ //www.mediawiki.org/