Lesen und verarbeiten

User-agent: *
Allow: /
Disallow: /*CR-Dokumentation.pdf$
Disallow: /gutscheine/suche?
Disallow: /gutscheine/*?code=*
Disallow: /gutscheine/*&code=*

Sitemap: https://www.spiegel.de/sitemaps/news-de.xml
Sitemap: https://www.spiegel.de/sitemaps/videos/sitemap.xml
Sitemap: https://www.spiegel.de/plus/sitemap.xml
Sitemap: https://www.spiegel.de/sitemap.xml
Sitemap: https://www.spiegel.de/gutscheine/sitemap.xml

using PuppeteerSharp;
using System;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;

using var browser =
    await Puppeteer.LaunchAsync(new LaunchOptions
{
  Headless = true, //Browser nicht anzeigen
  SlowMo = 40,
  ExecutablePath = @"C:\Program Files (x86)\"
    + @"Microsoft\Edge\Application\msedge.exe"
});
var page = await browser.NewPageAsync();
var userAgent = await browser.GetUserAgentAsync();
await page.SetUserAgentAsync($"{userAgent} TestBot");

await page.GoToAsync(
  "https://www.janschreier.de/Fallzahlen1.html");
string[] paragraphs = await page.
  EvaluateExpressionAsync<string[]>(
  @"Array.from(document.querySelectorAll('p'))."
  # @"map(p => p.innerText)");

// Ab hier erfolgt die Evaluation unabhängig
// von PuppeteerSharp
var datumRegex = new Regex(
  @"Datenstand: (\d{1,2}\.\d{2}\.202[1,2])");
var datumText = FindFirstRegexInStringArray(
  paragraphs, datumRegex);

if (datumText == null)
{
  Console.WriteLine("Abbruch! Datum nicht gefunden");
}
else
{
  var neueFaelleRegex = new Regex(
    @"(\d*) weitere Fälle");
  var neueFaelle = FindFirstRegexInStringArray(
    paragraphs, neueFaelleRegex);
  Console.WriteLine($"{datumText} : {neueFaelle ??"
    # " 'neue Fälle nicht gefunden'}");

  var inzidenzRegex = new Regex(@"7-Tages-Inzidenz "
    + @"beträgt (\d{1,4},\d{0,2})\.");
  var inzidenz = FindFirstRegexInStringArray(
    paragraphs, inzidenzRegex);
  Console.WriteLine($"{datumText} : {inzidenz ?? "
    + $"'Inzidenz nicht gefunden'}");
}

static string? FindFirstRegexInStringArray(
    string[] paragraphs, Regex regex)
{
  foreach (string paragraph in paragraphs)
  {
    var regexMatch = regex.Match(paragraph);
    if (regexMatch.Success)
    { return regexMatch.Groups[1].ToString(); }
  }
  return null;
}

using PuppeteerSharp;
using System;
using System.Text.RegularExpressions;
using System.Threading.Tasks;

using var browser = await Puppeteer.LaunchAsync(
    new LaunchOptions
{
  Headless = true, SlowMo = 20,
  ExecutablePath = @"C:\Program Files (x86)\"
    + @"Microsoft\Edge\Application\msedge.exe"
});
var page = await browser.NewPageAsync();
var userAgent = await browser.GetUserAgentAsync();
await page.SetUserAgentAsync($"{userAgent} TestBot");

await page.GoToAsync("https://www.wikipedia.org/");

await page.SetzeSpracheImSuchfeld("es");

var navigationTask = page.WaitForNavigationAsync();
await page.GebeSuchBegriffEinUndStarteSuche(
  "mallorca");
await navigationTask;

await page.WaitForSelectorAsync(".infobox");
string[] tableRowTexts =
  await page.EvaluateExpressionAsync<string[]>(
  @"Array.from(document.querySelector('.infobox').
  querySelectorAll('tr')).map(tr => tr.innerText)");

var populationRegex = new Regex(
  @"Total\s([\d\s]*)hab\.");
var populationText = FindFirstRegexInStringArray(
  tableRowTexts, populationRegex);

if (populationText != null)
{
  Console.WriteLine($"Laut der spanischen "
    + $" Wikipedia hat Mallorca "
    + $"{populationText.Trim()} Einwohner.");
}

static string? FindFirstRegexInStringArray(
    string[] paragraphs, Regex regex)
{
  foreach (string paragraph in paragraphs)
  {
    var regexMatch = regex.Match(paragraph);
    if (regexMatch.Success)
    {
      return regexMatch.Groups[1].ToString();
    }
  }
  return null;
}

public static class WikipediaPageExtensions
{
  public static async Task SetzeSpracheImSuchfeld(
    + this Page page, string language) => await
    page.SelectAsync("#searchLanguage", language);

  public static async Task
      GebeSuchBegriffEinUndStarteSuche(
      this Page page, string searchstring)
  {
    var elementHandle = await page.
      QuerySelectorAsync("#searchInput");
    await elementHandle.TypeAsync(searchstring);
    await elementHandle.PressAsync(
      PuppeteerSharp.Input.Key.Enter);
  }
}