feat: add guzzle-powered recursive website crawling and masterminds/html5 integration

composer.json

@@ -30,7 +30,9 @@
         "symfony/yaml": "^6.4|^7.0",
         "monolog/monolog": "^3.0",
         "vlucas/phpdotenv": "^5.6",
-        "google-gemini-php/client": "^1.0"
+        "google-gemini-php/client": "^1.0",
+        "guzzlehttp/guzzle": "^7.8",
+        "masterminds/html5": "^2.8"
     },
     "require-dev": {
         "phpunit/phpunit": "^10.5",

demo/index.php

@@ -1,19 +1,11 @@
 <?php
 
-// 1. Setup Autoloading (Mocking Composer)
+// 1. Setup Autoloading
-require_once __DIR__ . '/../src/Scanner.php';
+require_once __DIR__ . '/../vendor/autoload.php';
-require_once __DIR__ . '/../src/Violation.php';
-require_once __DIR__ . '/../src/SourceLocator.php';
-require_once __DIR__ . '/../src/Reporter/HtmlReporter.php';
-require_once __DIR__ . '/../src/Rule/RuleInterface.php';
-require_once __DIR__ . '/../src/Rules/ImgAltText.php';
-require_once __DIR__ . '/../src/Rules/FormLabel.php';
-require_once __DIR__ . '/../src/Rules/HtmlHasLang.php';
-require_once __DIR__ . '/../src/Rules/EmptyLink.php';
 
-use YakNet\AccessibilityConsole\Scanner;
+use YakNet\AccessibilityConsole\Core\Scanner;
-use YakNet\AccessibilityConsole\SourceLocator;
+use YakNet\AccessibilityConsole\Source\SourceLocator;
-use YakNet\AccessibilityConsole\Reporter\HtmlReporter;
+use YakNet\AccessibilityConsole\Reporting\HtmlDashboardReporter;
 use YakNet\AccessibilityConsole\Rules\ImgAltText;
 use YakNet\AccessibilityConsole\Rules\FormLabel;
 
@@ -48,19 +40,18 @@ $violations = $scanner->scan($html);
 
 if (!empty($violations)) {
     // Locate Sources
-    // We point it to the 'views' directory so it knows where to look for the error source
     $locator = new SourceLocator(__DIR__ . '/views');
 
     foreach ($violations as $violation) {
-        $location = $locator->locate($violation->snippet);
+        $location = $locator->locate($violation->htmlSnippet);
         if ($location) {
-            $violation->setSourceLocation($location['file'], $location['line']);
+            $violation->location = $location;
         }
     }
 
-    // Render Report
+    // Render Report (Format as results mapped by page URL)
-    $reporter = new HtmlReporter();
+    $reporter = new HtmlDashboardReporter();
-    $reportHtml = $reporter->render($violations);
+    $reportHtml = $reporter->render(['http://localhost:8080/index.php' => $violations], 'demo/index.php');
 
     // Inject into body
     $html = str_replace('</body>', $reportHtml . '</body>', $html);

src/Console/Command/ScanCommand.php

@@ -26,7 +26,10 @@ class ScanCommand extends Command
             ->addArgument('target', InputArgument::REQUIRED, 'The URL or file path to scan')
             ->addOption('ai', null, InputOption::VALUE_NONE, 'Enable AI-powered fix suggestions')
             ->addOption('report', null, InputOption::VALUE_REQUIRED, 'Save report to file (html or json)')
-            ->addOption('project-path', null, InputOption::VALUE_REQUIRED, 'Base path for source mapping');
+            ->addOption('project-path', null, InputOption::VALUE_REQUIRED, 'Base path for source mapping')
+            ->addOption('crawl', null, InputOption::VALUE_NONE, 'Crawl internal links recursively')
+            ->addOption('depth', null, InputOption::VALUE_REQUIRED, 'Maximum crawling depth', '3')
+            ->addOption('max-pages', null, InputOption::VALUE_REQUIRED, 'Maximum number of pages to crawl', '20');
     }
 
     protected function execute(InputInterface $input, OutputInterface $output): int
@@ -46,73 +49,195 @@ class ScanCommand extends Command
 
         $io->title("Accessibility Console - Scanning: $target");
 
-        // 1. Fetch HTML
+        // 1. Setup Scanner
-        $html = @file_get_contents($target);
-        if ($html === false || $html === '') {
-            $io->error("Could not read target: $target");
-            return Command::FAILURE;
-        }
-
-        // 2. Setup Scanner
         $scanner = new Scanner();
         foreach (StandardRuleSet::all() as $rule) {
             $scanner->addRule($rule);
         }
 
-        // 3. Scan
+        // 2. Fetch and Scan
-        $violations = $scanner->scan($html);
+        $isUrl = str_starts_with($target, 'http://') || str_starts_with($target, 'https://');
+        $crawl = (bool)$input->getOption('crawl') && $isUrl;
 
-        if (empty($violations)) {
+        /** @var array<string, \YakNet\AccessibilityConsole\Core\Violation[]> $results */
+        $results = [];
+
+        if ($crawl) {
+            $depthOption = $input->getOption('depth');
+            $depthLimit = is_numeric($depthOption) ? (int)$depthOption : 3;
+            $maxPagesOption = $input->getOption('max-pages');
+            $maxPages = is_numeric($maxPagesOption) ? (int)$maxPagesOption : 20;
+            
+            $io->comment("Starting recursive crawl from: $target (Max Depth: $depthLimit, Max Pages: $maxPages)");
+
+            $parsedBase = parse_url($target);
+            $baseHost = $parsedBase['host'] ?? '';
+
+            $queue = [
+                ['url' => $target, 'depth' => 1]
+            ];
+            $visited = [];
+
+            $client = new \GuzzleHttp\Client([
+                'timeout' => 10,
+                'allow_redirects' => true,
+                'headers' => [
+                    'User-Agent' => 'YakNet Accessibility Console Crawler/1.0',
+                ]
+            ]);
+
+            while (!empty($queue) && count($visited) < $maxPages) {
+                $current = array_shift($queue);
+                if (!is_array($current)) {
+                    continue;
+                }
+                
+                $url = $current['url'];
+                $depth = (int)$current['depth'];
+
+                // Remove fragment
+                $cleanUrl = strtok($url, '#');
+                if ($cleanUrl === false || isset($visited[$cleanUrl])) {
+                    continue;
+                }
+
+                $visited[$cleanUrl] = true;
+                $io->write("  <fg=gray>Crawling [Depth $depth]:</> <fg=cyan>$cleanUrl</>... ");
+
+                try {
+                    $response = $client->request('GET', $cleanUrl);
+                    $html = (string)$response->getBody();
+                    $io->writeln("<fg=green>OK</>");
+                    
+                    // Scan the page HTML
+                    $pageViolations = $scanner->scan($html);
+                    $results[$cleanUrl] = $pageViolations;
+
+                    // Extract links if depth is less than limit
+                    if ($depth < $depthLimit) {
+                        $html5 = new \Masterminds\HTML5(['disable_html_ns' => true]);
+                        $doc = $html5->loadHTML($html);
+                        $links = $doc->getElementsByTagName('a');
+                        
+                        foreach ($links as $link) {
+                            $href = $link->getAttribute('href');
+                            if (empty($href) || str_starts_with($href, 'javascript:') || str_starts_with($href, 'mailto:') || str_starts_with($href, 'tel:')) {
+                                continue;
+                            }
+
+                            // Resolve relative URI
+                            try {
+                                $baseUri = new \GuzzleHttp\Psr7\Uri($cleanUrl);
+                                $relativeUri = new \GuzzleHttp\Psr7\Uri($href);
+                                $absoluteUri = \GuzzleHttp\Psr7\UriResolver::resolve($baseUri, $relativeUri);
+                                $absoluteUrl = (string)$absoluteUri;
+                            } catch (\Throwable) {
+                                continue;
+                            }
+
+                            // Check host compatibility
+                            $parsedAbsolute = parse_url($absoluteUrl);
+                            $absoluteHost = $parsedAbsolute['host'] ?? '';
+                            
+                            if ($absoluteHost === $baseHost) {
+                                // Add to queue if not already visited or in queue
+                                $cleanAbsolute = strtok($absoluteUrl, '#');
+                                if ($cleanAbsolute !== false && !isset($visited[$cleanAbsolute])) {
+                                    $inQueue = false;
+                                    foreach ($queue as $q) {
+                                        if ($q['url'] === $cleanAbsolute) {
+                                            $inQueue = true;
+                                            break;
+                                        }
+                                    }
+                                    if (!$inQueue) {
+                                        $queue[] = ['url' => $cleanAbsolute, 'depth' => $depth + 1];
+                                    }
+                                }
+                            }
+                        }
+                    }
+                } catch (\Throwable $e) {
+                    $io->writeln("<fg=red>FAIL (" . $e->getMessage() . ")</>");
+                }
+            }
+        } else {
+            // Standard single page/file scan
+            $html = @file_get_contents($target);
+            if ($html === false || $html === '') {
+                $io->error("Could not read target: $target");
+                return Command::FAILURE;
+            }
+            $violations = $scanner->scan($html);
+            $results[$target] = $violations;
+        }
+
+        $totalViolations = 0;
+        foreach ($results as $url => $violations) {
+            $totalViolations += count($violations);
+        }
+
+        if ($totalViolations === 0) {
             $io->success("No violations found!");
             return Command::SUCCESS;
         }
 
-        $io->section("Found " . count($violations) . " violations");
+        $io->section("Found a total of $totalViolations violations across " . count($results) . " pages");
 
-        // 4. Setup Helpers
+        // 3. Setup Helpers
         $locator = new SourceLocator($projectPath);
         $config = new Config();
         $fixer = $enableAi ? new GeminiFixer($config->getGeminiApiKey()) : null;
 
-        foreach ($violations as $violation) {
+        foreach ($results as $url => $violations) {
-            $io->writeln("<fg=white;bg=red;options=bold> {$violation->ruleId} </>\t<fg=red>{$violation->message}</>");
+            if (empty($violations)) {
-            
+                continue;
-            // Show Snippet
-            $io->writeln("  <fg=gray>Snippet:</> <fg=yellow>" . htmlspecialchars($violation->htmlSnippet) . "</>");
-
-            // Source Mapping
-            $location = $locator->locate($violation->htmlSnippet);
-            if ($location) {
-                $io->writeln("  <fg=gray>Source:</>  <fg=cyan>{$location['file']}:{$location['line']}</>");
             }
+            $io->section("Page: $url (" . count($violations) . " violations)");
 
-            // AI Fixer
+            foreach ($violations as $violation) {
-            if ($enableAi && $fixer) {
+                $io->writeln("<fg=white;bg=red;options=bold> {$violation->ruleId} </>\t<fg=red>{$violation->message}</>");
-                $io->write("  <fg=gray>AI Fix Suggestion:</> ");
+                
-                $suggestion = $fixer->suggestFix($violation);
+                // Show Snippet
-                if ($suggestion) {
+                $io->writeln("  <fg=gray>Snippet:</> <fg=yellow>" . htmlspecialchars($violation->htmlSnippet) . "</>");
-                    $violation->fixSuggestion = $suggestion;
+
-                    $io->writeln("<fg=green>" . htmlspecialchars($suggestion) . "</>");
+                // Source Mapping
-                } else {
+                $location = $locator->locate($violation->htmlSnippet);
-                    $io->writeln("<fg=red>Could not generate suggestion (check API key)</>");
+                if ($location) {
+                    $io->writeln("  <fg=gray>Source:</>  <fg=cyan>{$location['file']}:{$location['line']}</>");
                 }
-            }
 
-            $io->writeln("");
+                // AI Fixer
+                if ($enableAi && $fixer) {
+                    $io->write("  <fg=gray>AI Fix Suggestion:</> ");
+                    $suggestion = $fixer->suggestFix($violation);
+                    if ($suggestion) {
+                        $violation->fixSuggestion = $suggestion;
+                        $io->writeln("<fg=green>" . htmlspecialchars($suggestion) . "</>");
+                    } else {
+                        $io->writeln("<fg=red>Could not generate suggestion (check API key)</>");
+                    }
+                }
+
+                $io->writeln("");
+            }
         }
 
-        $io->warning("Scanned " . count($violations) . " errors. Use --ai to get fix suggestions.");
+        $io->warning("Scanned " . $totalViolations . " errors. Use --ai to get fix suggestions.");
 
-        // Export Report
+        // 4. Export Report
         $reportPath = $input->getOption('report');
         if (is_string($reportPath) && $reportPath !== '') {
             $extension = pathinfo($reportPath, PATHINFO_EXTENSION);
             if ($extension === 'json') {
-                $data = array_map(fn($v) => $v->toArray(), $violations);
+                $serialized = [];
-                file_put_contents($reportPath, json_encode($data, JSON_PRETTY_PRINT));
+                foreach ($results as $url => $violations) {
+                    $serialized[$url] = array_map(fn($v) => $v->toArray(), $violations);
+                }
+                file_put_contents($reportPath, json_encode($serialized, JSON_PRETTY_PRINT));
             } else {
                 $reporter = new \YakNet\AccessibilityConsole\Reporting\HtmlDashboardReporter();
-                file_put_contents($reportPath, $reporter->render($violations, $target));
+                file_put_contents($reportPath, $reporter->render($results, $target));
             }
             $io->success("Report saved to: $reportPath");
         }

src/Core/Scanner.php

@@ -28,9 +28,10 @@ class Scanner
             return [];
         }
 
-        $doc = new \DOMDocument();
+        $html5 = new \Masterminds\HTML5([
-        // Suppress warnings for malformed HTML
+            'disable_html_ns' => true,
-        @$doc->loadHTML('<?xml encoding="utf-8" ?>' . $html, LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD);
+        ]);
+        $doc = $html5->loadHTML($html);
 
         return $this->engine->run($doc);
     }