Crawler Teaser Indexer Bundle#

composer require atoolo/crawler-teaser-indexer

composer update

bin/console crawler:scheduler-atoolo-crawler-teaser-indexer -vvv

composer require atoolo/crawler-teaser-indexer

config/packages/atoolo_crawler_master.yaml

parameters:
  # Defines execution schedule
  atoolo.crawler.schedule:
    - '0 3 * * *'

  # Status codes that trigger retries
  atoolo.crawler.retry_status_codes:
    - 408
    - 429
    - 500
    - 502
    - 503
    - 504

  # URL endings that are never followed (applies to all sites)
  atoolo.crawler.deny_endings:
    - .jpg
    - .png
    - .pdf
    - .zip

<?php return [
  "name" => "Indexer für externe Teaser",
  "data" => [
    "sp_crawling_sites" => [
      [
        "sp_id" => "!To Fill in the Pagename in snakecase!",
        "sp_max_teaser" => "100",
        "sp_cleanup_threshold" => "10",
        "sp_parallel_requests" => "3",
        "sp_delay_ms" => "500",
        "sp_user_agent" => "ExampleBot/1.0 (+contact@example.org)",
        "sp_start_urls" => [
          [
            "sp_url" => "!To Fill in the startpage absolute url!",
            "sp_extraction_depth" => "2",
          ],
        ],
        "sp_link_selector" => "#content a[href]",
        "sp_allow_prefixes" => [
          "https://",
        ],
        "sp_title_prefix" => "Pagename - ",
        "sp_title_opengraph" => ["og:title"],
        "sp_title_css" => [
          "!To Fill in the Title CssSelector!",
          "#content h2",
        ],
      ],
    ],
  ],
];

bin/console crawler:scheduler-atoolo-crawler-teaser-indexer -vvv

command=/path/to/app/bin/console messenger:consume scheduler_atoolo-crawler-teaser-indexer <other transports>

supervisorctl reread && supervisorctl update

bin/console debug:scheduler atoolo-crawler-teaser-indexer
ps aux | grep "messenger:consume"

URLCollector ──▶ Parser ──▶ Processor ──▶ Indexer
(crawl + fetch)  (extract)   (clean)       (Solr)

./bin/console cache:clear

parameters:
  # Defines execution schedule
  # - Cron expression (5 fields)
  # - Format:
  # ┌ Minute (0–59)
  # │ ┌ Hour (0–23)
  # │ │ ┌ Day of month (1–31)
  # │ │ │ ┌ Month (1–12)
  # │ │ │ │ ┌ Day of week (0–6, Sun=0)
  # │ │ │ │ │
  # * * * * *

  # Example:
  # schedule (field 1): 0      → minute
  # schedule (field 2): 3      → hour
  # schedule (field 3): */3    → every 3 days
  # schedule (field 4-5): * *  → every month, every weekday
  atoolo.crawler.schedule:
    - '0 3 * * *'
    - '15 3 * * *'

  # Status codes that trigger retries
  atoolo.crawler.retry_status_codes:
    - 408
    - 429
    - 500
    - 501
    - 502
    - 503
    - 504

  # URL endings that are never followed (merged with sp_deny_endings of each site)
  atoolo.crawler.deny_endings:
    # Images & graphics
    - .jpg
    - .jpeg
    - .png
    - .gif
    - .svg
    - .webp
    # Documents
    - .pdf
    - .doc
    - .docx
    # Archives
    - .zip
    - .gz
    # Web assets & data
    - .js
    - .css
    - .json
    - .xml
    # Media & fonts
    - .mp3
    - .mp4
    - .woff
    - .woff2

    # Category IDs written to every teaser of this site (Solr field sp_category)
    "sp_categories" => [1525, 1523, 1524],

    # Category path IDs written to every teaser of this site (Solr field sp_category_path)
    "sp_categories_path" => [1525, 1528, 1524],

  # Required. Unique ID for this website configuration.
  # Used as source of the documents in Solr (sp_source, sp_objecttype).
  "sp_id" => "source_pagename",

  # Respect robots.txt (default: false)
  # Only applied if sp_robots_url is set. If the robots.txt cannot be read,
  # all URLs are allowed.
  "sp_respect_robots_txt" => false,

  # Absolute URL of the robots.txt of the target domain
  "sp_robots_url" => "https://www.example.com/robots.txt",

  # Maximum number of pages fetched while crawling (default: 100)
  # The crawl stops once this number of pages has been fetched.
  # sp_forced_article_urls are not counted.
  "sp_max_teaser" => "100",

  # Old documents of this site are only removed from Solr if MORE than
  # this number of teasers was indexed (default: 50)
  "sp_cleanup_threshold" => "50",

  # Maximum number of retries for a failed request (default: 3)
  # 0 means only one attempt (no retries)
  "sp_max_retry" => "3",

  # Delay before the first retry in milliseconds (default: 500)
  # The delay doubles with every retry (500, 1000, 2000, ...).
  # A Retry-After header of the target system takes precedence.
  "sp_backoff_ms" => "500",

  # Minimum delay between two requests to the same host in milliseconds (default: 150)
  # (increase if the target system blocks requests)
  "sp_delay_ms" => "500",

  # Number of requests sent in parallel (default: 1; recommended 1–3, never above 10)
  "sp_parallel_requests" => "3",

  # Clearly identifiable User-Agent (default: "Atoolo/Crawler-Teaser-Indexer")
  "sp_user_agent" => "ExampleBot/1.0 (+contact@example.org)",

  # Start URLs for the crawler
  # sp_extraction_depth: links are followed from the start page (depth 0)
  # up to this depth; the pages found at the next level are still fetched,
  # but their links are not followed.
  # Example with 2: start page → level 1 → level 2 → level 3 (leaf pages)
  "sp_start_urls" => [
      [
          "sp_url" => "https://www.example.com/microsite/index.php",
          "sp_extraction_depth" => 2
      ],
      [
          "sp_url" => "https://www.example.com/",
          "sp_extraction_depth" => 1
      ]
  ],

  # CSS selector for the links to follow (default: #content a[href])
  "sp_link_selector" => "#content a[href]",

  # Allowed URL prefixes (whitelist)
  # If set, a URL must start with one of them.
  "sp_allow_prefixes" => [
      "https://www.example.com/microsite/"
  ],

  # Explicit exclusions of URL prefixes (blacklist)
  "sp_deny_prefixes" => [
      "https://www.example.com/microsite/meta/",
      "https://www.example.com/microsite/about_us/team.php"
  ],

  # Additional URL endings that are not followed on this site
  # Merged with atoolo.crawler.deny_endings, compared case-insensitively
  # against the URL path.
  "sp_deny_endings" => [
      ".ics",
      ".vcf"
  ],

  # URLs that are always fetched and always pass the content scoring,
  # independent of sp_max_teaser. They are still skipped if no title
  # can be determined.
  "sp_forced_article_urls" => [
      "https://www.example.com/microsite/important-page.php"
  ],

  # Remove query parameters from discovered URLs (default: false)
  # The parameters are removed permanently: the URL without them is
  # fetched and indexed. This avoids fetching the same page several times.
  "sp_strip_query_params_active" => true,

  "sp_strip_query_params" => [
      "utm_source",
      "utm_medium",
      "sort",
      "view",
      "print",
      "fbclid",
      "gclid"
  ],

  # Remove the fragment (#...) from URLs starting with one of these prefixes
  # Example: page.php#contact and page.php#directions are fetched once as page.php
  "sp_strip_fragments" => [
      "https://www.example.com/"
  ]

  # XPath expressions (not CSS selectors) that split one page into
  # several teasers, e.g. an overview page with a list of events.
  # Every matched element becomes its own teaser; all selectors below
  # are then evaluated inside that element only.
  # If a matched element contains another matched element, only the
  # inner one is used.
  # Without a value, no match or an invalid expression, the whole page
  # is one teaser.
  "sp_split_html_document" => [
      "//article",
      "//*[@id='content']//div[contains(@class, 'event')]"
  ],

  # Used to clearly identify the article's source
  "sp_title_prefix" => "Pagename - ",

  # OpenGraph tags are preferred when extracting data
  "sp_title_opengraph" => [
      "og:title"
  ],

  # CSS selectors (skipped if empty)
  "sp_title_css" => [
      "h1",
      ".page-title",
      "#content h2"
  ],

  # Maximum character length including the "…" appended when truncated
  # (default: 999)
  "sp_title_max_chars" => 120

  # Enable intro text extraction (default: false)
  "sp_introText_present" => true,

  # If true, the teaser is skipped if no intro text is found (default: false)
  "sp_introText_required_field" => false,

  # Preferred OpenGraph tags
  "sp_introText_opengraph" => [
      "og:description"
  ],

  # CSS selectors (skipped if empty)
  "sp_introText_css" => [
      "#content p",
      "main p",
      ".description"
  ],

  # Maximum character length including the "…" appended when truncated
  # (default: 999)
  "sp_introText_max_chars" => 200

  # Enable DateTime extraction (default: false)
  "sp_datetime_present" => false,

  # If true, the teaser is skipped if no valid DateTime is found (default: false)
  "sp_datetime_required_field" => false,

  # If only a date is present, append 00:00:00 (default: true)
  # Example:
  # "2026-01-21" → "2026-01-21 00:00:00"
  # Solr requires DateTime, not Date Type
  "sp_datetime_only_date" => true,

  # Preferred OpenGraph tags
  "sp_datetime_opengraph" => [
      "article:published_time"
  ],

  # CSS selectors (skipped if empty)
  # The datetime attribute of the element is preferred over its text,
  # e.g. <time datetime="2026-01-21">21. Januar</time>.
  # The value has to be a date format PHP can parse.
  "sp_datetime_css" => [
      "time[datetime]",
      ".published-date",
      ".article-date"
  ]

  # Enable content scoring (default: false)
  "sp_content_scoring_active" => true,

  # Minimum required score for a teaser to be indexed (default: 4)
  "sp_content_scoring_min_score" => 7,

  # CSS selectors for the main content that is scored
  # The first selector that yields text is used; without a value or
  # without a match, the visible text of the whole page (or split block)
  # is scored.
  "sp_relevance_content_selector" => [
      "#content",
      "main"
  ],

  # Positive signals (increase score)
  "sp_content_scoring_positive" => [
      [
          "sp_score" => 4,
          "sp_match_any" => [
              "online application",
              "book appointment",
              "submit application"
          ]
      ],
      [
          "sp_score" => 1,
          "sp_match_any" => [
              "procedure",
              "legal basis",
              "further information"
          ]
      ]
  ],

  # Negative signals (decrease score)
  "sp_content_scoring_negative" => [
      [
          "sp_score" => -6,
          "sp_match_any" => [
              "newsletter",
              "advertising",
              "press office"
          ]
      ],
      [
          # Pages with less than 150 characters of text
          "sp_score" => -3,
          "sp_condition" => [
              "sp_body_text_length" => 150
          ]
      ]
  ]

<?php

declare(strict_types=1);

namespace App\Crawler;

use Atoolo\CrawlerIndexer\Config\PipelineConfig;
use Atoolo\CrawlerIndexer\Pipeline\Collector\LinkFilterInterface;
use Symfony\Component\DependencyInjection\Attribute\AsDecorator;
use Symfony\Component\DependencyInjection\Attribute\AutowireDecorated;

#[AsDecorator(LinkFilterInterface::class)]
final class NoPrintLinkFilter implements LinkFilterInterface
{
    public function __construct(
        #[AutowireDecorated] private readonly LinkFilterInterface $inner,
    ) {}

    public function filter(array $urls, PipelineConfig $config): array
    {
        return array_values(array_filter(
            $this->inner->filter($urls, $config),
            static fn (string $url): bool => !str_contains($url, '/print/'),
        ));
    }
}

<?php

declare(strict_types=1);

namespace App\Crawler;

use Atoolo\CrawlerIndexer\Config\PipelineConfig;
use Atoolo\CrawlerIndexer\Pipeline\Parser\FieldExtractorInterface;
use Atoolo\CrawlerIndexer\Pipeline\Parser\FieldSource;

final class DataDateExtractor implements FieldExtractorInterface
{
    public function supports(string $field): bool
    {
        return self::FIELD_DATETIME === $field;
    }

    public function extract(string $field, FieldSource $source, PipelineConfig $config): mixed
    {
        $raw = $source->attr('.event', 'data-date');
        if (null === $raw || 'example_events' !== $config->id()) {
            return null;
        }

        return \DateTimeImmutable::createFromFormat('d.m.Y', $raw) ?: null;
    }
}