start project

app/blueprints/seo/__init__.py

@@ -0,0 +1,13 @@
+from flask import Blueprint
+from flask_login import login_required
+
+bp = Blueprint("seo", __name__, template_folder="templates")
+
+
+@bp.before_request
+@login_required
+def require_login():
+    pass
+
+
+from app.blueprints.seo import routes  # noqa: E402,F401  registers routes

app/blueprints/seo/queries.py

@@ -0,0 +1,140 @@
+"""Context-builder query functions for the SEO tab (Chapter 09).
+
+Per Ch09's own Output instruction, bot-related widgets query bot_hits
+directly (small, indefinitely-retained, indexed — Ch06) rather than a new
+rollup; only the human-side of the crawled-vs-visited comparison needs
+the new human_path_stats_daily table.
+"""
+from __future__ import annotations
+
+from collections import defaultdict
+from datetime import date
+
+from sqlalchemy import case, func
+
+from app.extensions import db
+from app.models.bot_hit import BotHit
+from app.models.human_path_stats import HumanPathStatsDaily
+from app.utils.dates import day_bounds
+
+# ASSUMPTION (flagged): Ch09 doesn't define "major crawler" for the
+# crawl-frequency chart's series cap.
+TOP_N_BOTS_FOR_CHART = 6
+
+
+def get_bot_summary(from_date: date, to_date: date) -> list[dict]:
+    start, end = day_bounds(from_date, to_date)
+    rows = (
+        db.session.query(
+            BotHit.bot_name,
+            func.count().label("hits"),
+            func.sum(case((BotHit.verified.is_(True), 1), else_=0)).label("verified_hits"),
+            func.max(BotHit.timestamp).label("last_seen"),
+        )
+        .filter(BotHit.timestamp >= start, BotHit.timestamp < end)
+        .group_by(BotHit.bot_name)
+        .order_by(func.count().desc())
+        .all()
+    )
+    return [
+        {
+            "bot_name": r.bot_name,
+            "hits": r.hits,
+            "verified_pct": round(r.verified_hits / r.hits * 100, 1) if r.hits else 0.0,
+            "last_seen": r.last_seen.isoformat() if r.last_seen else None,
+        }
+        for r in rows
+    ]
+
+
+def get_crawl_chart_data(from_date: date, to_date: date, bot_name: str | None) -> dict:
+    start, end = day_bounds(from_date, to_date)
+    base_filters = [BotHit.timestamp >= start, BotHit.timestamp < end]
+
+    if bot_name:
+        allowed_bots = [bot_name]
+    else:
+        top_rows = (
+            db.session.query(BotHit.bot_name, func.count().label("hits"))
+            .filter(*base_filters)
+            .group_by(BotHit.bot_name)
+            .order_by(func.count().desc())
+            .limit(TOP_N_BOTS_FOR_CHART)
+            .all()
+        )
+        allowed_bots = [r.bot_name for r in top_rows]
+
+    if not allowed_bots:
+        return {"series": []}
+
+    rows = (
+        db.session.query(func.date(BotHit.timestamp).label("day"), BotHit.bot_name, func.count().label("count"))
+        .filter(*base_filters, BotHit.bot_name.in_(allowed_bots))
+        .group_by("day", BotHit.bot_name)
+        .order_by("day")
+        .all()
+    )
+    points_by_bot: dict[str, list[dict]] = defaultdict(list)
+    for r in rows:
+        points_by_bot[r.bot_name].append({"t": r.day, "count": r.count})
+
+    return {"series": [{"bot_name": b, "points": points_by_bot.get(b, [])} for b in allowed_bots]}
+
+
+def get_bot_status_codes(from_date: date, to_date: date) -> dict:
+    start, end = day_bounds(from_date, to_date)
+    bucket = case(
+        (BotHit.status_code < 300, "2xx"),
+        (BotHit.status_code < 400, "3xx"),
+        (BotHit.status_code < 500, "4xx"),
+        else_="5xx",
+    )
+    rows = (
+        db.session.query(bucket.label("bucket"), func.count().label("count"))
+        .filter(BotHit.timestamp >= start, BotHit.timestamp < end)
+        .group_by("bucket")
+        .all()
+    )
+    breakdown = {"2xx": 0, "3xx": 0, "4xx": 0, "5xx": 0}
+    for r in rows:
+        breakdown[r.bucket] = r.count
+
+    # Ch09 calls out 404 by name specifically, not just the 4xx bucket.
+    not_found_404 = (
+        db.session.query(func.count())
+        .filter(BotHit.timestamp >= start, BotHit.timestamp < end, BotHit.status_code == 404)
+        .scalar()
+    )
+    return {"breakdown": breakdown, "not_found_404": not_found_404 or 0}
+
+
+def get_crawled_vs_visited(from_date: date, to_date: date, page: int, per_page: int) -> tuple[list[list], int]:
+    """Diffed table: one row per path, bot hits vs. human hits."""
+    start, end = day_bounds(from_date, to_date)
+    bot_rows = (
+        db.session.query(BotHit.path, func.count().label("hits"))
+        .filter(BotHit.timestamp >= start, BotHit.timestamp < end)
+        .group_by(BotHit.path)
+        .all()
+    )
+    human_rows = (
+        db.session.query(HumanPathStatsDaily.path, func.sum(HumanPathStatsDaily.count).label("hits"))
+        .filter(HumanPathStatsDaily.date >= from_date, HumanPathStatsDaily.date <= to_date)
+        .group_by(HumanPathStatsDaily.path)
+        .all()
+    )
+    bot_counts = {r.path: r.hits for r in bot_rows}
+    human_counts = {r.path: r.hits for r in human_rows}
+
+    combined = []
+    for path in set(bot_counts) | set(human_counts):
+        b, h = bot_counts.get(path, 0), human_counts.get(path, 0)
+        total = b + h
+        combined.append([path, b, h, round(b / total * 100, 1) if total else 0.0])
+
+    # ASSUMPTION (flagged): sorted by bot hits desc — Ch09 doesn't specify.
+    combined.sort(key=lambda row: row[1], reverse=True)
+
+    total_count = len(combined)
+    offset = (page - 1) * per_page
+    return combined[offset : offset + per_page], total_count

app/blueprints/seo/routes.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from flask import jsonify, request
+
+from app.blueprints.seo import bp
+from app.blueprints.seo.queries import (
+    get_bot_status_codes,
+    get_bot_summary,
+    get_crawl_chart_data,
+    get_crawled_vs_visited,
+)
+from app.utils.dates import parse_date_range
+from app.utils.htmx import render_htmx_aware
+from app.utils.pagination import parse_pagination
+
+
+@bp.route("/seo")
+def seo():
+    from_date, to_date = parse_date_range(request)
+    return render_htmx_aware(
+        request, full_template="seo/index.html", partial_template="seo/_content.html",
+        from_date=from_date, to_date=to_date,
+    )
+
+
+@bp.get("/api/seo/bot-summary")
+def api_bot_summary():
+    from_date, to_date = parse_date_range(request)
+    return jsonify(data=get_bot_summary(from_date, to_date), meta={"from": from_date.isoformat(), "to": to_date.isoformat()})
+
+
+@bp.get("/api/seo/crawl-chart-data")
+def api_crawl_chart_data():
+    from_date, to_date = parse_date_range(request)
+    bot_name = request.args.get("bot")
+    return jsonify(
+        data=get_crawl_chart_data(from_date, to_date, bot_name),
+        meta={"from": from_date.isoformat(), "to": to_date.isoformat(), "bot": bot_name},
+    )
+
+
+@bp.get("/api/seo/bot-status-codes")
+def api_bot_status_codes():
+    from_date, to_date = parse_date_range(request)
+    return jsonify(data=get_bot_status_codes(from_date, to_date), meta={"from": from_date.isoformat(), "to": to_date.isoformat()})
+
+
+@bp.get("/api/seo/crawled-vs-visited")
+def api_crawled_vs_visited():
+    from_date, to_date = parse_date_range(request)
+    page, per_page = parse_pagination(request)
+    rows, total = get_crawled_vs_visited(from_date, to_date, page, per_page)
+    return jsonify(
+        data={"rows": rows, "total": total},
+        meta={"from": from_date.isoformat(), "to": to_date.isoformat(), "page": page, "per_page": per_page},
+    )

app/blueprints/seo/templates/seo/_content.html

@@ -0,0 +1,102 @@
+<div id="seo-content"
+     hx-get="{{ url_for('seo.seo') }}"
+     hx-trigger="change from:#seo-date-range-form"
+     hx-include="#seo-date-range-form"
+     hx-target="#seo-content"
+     hx-swap="outerHTML"
+     data-from="{{ from_date.isoformat() }}"
+     data-to="{{ to_date.isoformat() }}">
+
+  <div class="flex flex-wrap items-end justify-between gap-4 mb-6">
+    <div>
+      <h1 class="font-display font-bold text-xl">SEO &amp; Bot Behavior</h1>
+      <p class="text-sm text-muted dark:text-muted-dark">How search engines are crawling your site</p>
+    </div>
+    <form id="seo-date-range-form" class="flex gap-3 items-end">
+      <label class="text-sm text-muted dark:text-muted-dark">From
+        <input type="date" name="from" value="{{ from_date.isoformat() }}"
+               class="block border border-line dark:border-line-dark rounded-md px-2 py-1 mt-1 bg-surface dark:bg-surface-dark text-ink dark:text-ink-dark font-data text-sm">
+      </label>
+      <label class="text-sm text-muted dark:text-muted-dark">To
+        <input type="date" name="to" value="{{ to_date.isoformat() }}"
+               class="block border border-line dark:border-line-dark rounded-md px-2 py-1 mt-1 bg-surface dark:bg-surface-dark text-ink dark:text-ink-dark font-data text-sm">
+      </label>
+    </form>
+  </div>
+
+  <div id="bot-summary-cards" class="grid grid-cols-2 sm:grid-cols-3 gap-3 mb-6"
+       data-endpoint="{{ url_for('seo.api_bot_summary') }}"></div>
+
+  <div class="grid grid-cols-1 lg:grid-cols-2 gap-4 mb-6">
+    <div class="border border-line dark:border-line-dark rounded-xl p-4 bg-surface dark:bg-surface-dark h-72">
+      <h3 class="text-xs font-medium uppercase tracking-wide text-muted dark:text-muted-dark mb-2">Crawl frequency</h3>
+      <div class="h-56"><canvas id="crawl-frequency-chart" data-endpoint="{{ url_for('seo.api_crawl_chart_data') }}"></canvas></div>
+    </div>
+    <div class="border border-line dark:border-line-dark rounded-xl p-4 bg-surface dark:bg-surface-dark h-72">
+      <h3 class="text-xs font-medium uppercase tracking-wide text-muted dark:text-muted-dark mb-2">Status codes served to bots</h3>
+      <div class="h-44"><canvas id="bot-status-codes-chart" data-endpoint="{{ url_for('seo.api_bot_status_codes') }}"></canvas></div>
+      <p id="bot-404-note" class="text-xs text-muted dark:text-muted-dark mt-2"></p>
+    </div>
+  </div>
+
+  <div>
+    <h3 class="text-xs font-medium uppercase tracking-wide text-muted dark:text-muted-dark mb-2">Most-crawled vs. most-visited URLs</h3>
+    <div id="crawled-vs-visited-grid" data-endpoint="{{ url_for('seo.api_crawled_vs_visited') }}"></div>
+  </div>
+
+  <script>
+    (function initSeoWidgets() {
+      const root = document.getElementById('seo-content');
+      const from = root.dataset.from, to = root.dataset.to;
+      const withRange = (url) => `${url}?from=${from}&to=${to}`;
+
+      fetch(withRange(document.getElementById('bot-summary-cards').dataset.endpoint))
+        .then((r) => r.json())
+        .then(({ data }) => {
+          document.getElementById('bot-summary-cards').innerHTML = data.length ? data.map((bot) => `
+            <div class="bg-surface dark:bg-surface-dark rounded-lg border border-line dark:border-line-dark border-t-2 ${bot.verified_pct < 100 ? 'border-t-warn dark:border-t-warn-dark' : 'border-t-ok dark:border-t-ok-dark'} p-3">
+              <p class="text-xs text-muted dark:text-muted-dark uppercase tracking-wide">${bot.bot_name}</p>
+              <p class="font-data text-xl font-medium mt-0.5">${bot.hits.toLocaleString()} <span class="text-sm text-muted dark:text-muted-dark font-sans">hits</span></p>
+              <p class="text-xs mt-1 ${bot.verified_pct < 100 ? 'text-warn dark:text-warn-dark' : 'text-muted dark:text-muted-dark'}">
+                ${bot.verified_pct}% verified${bot.verified_pct < 100 ? ' — some spoofed' : ''}
+              </p>
+              <p class="text-xs text-muted dark:text-muted-dark font-data mt-0.5">Last seen: ${bot.last_seen ?? '—'}</p>
+            </div>`).join('') : `<p class="text-sm text-muted dark:text-muted-dark col-span-full">No bot activity in this range.</p>`;
+        });
+
+      const crawlEl = document.getElementById('crawl-frequency-chart');
+      fetch(withRange(crawlEl.dataset.endpoint))
+        .then((r) => r.json())
+        .then(({ data }) => window.initChart('crawl-frequency-chart', {
+          type: 'line',
+          data: {
+            labels: [...new Set(data.series.flatMap((s) => s.points.map((p) => p.t)))].sort(),
+            datasets: data.series.map((s, i) => ({
+              label: s.bot_name,
+              data: s.points.map((p) => ({ x: p.t, y: p.count })),
+              tension: 0.3,
+              borderColor: window.KAVOSH_CHART_PALETTE[i % window.KAVOSH_CHART_PALETTE.length],
+            })),
+          },
+        }));
+
+      const statusEl = document.getElementById('bot-status-codes-chart');
+      fetch(withRange(statusEl.dataset.endpoint))
+        .then((r) => r.json())
+        .then(({ data }) => {
+          window.initChart('bot-status-codes-chart', {
+            type: 'bar',
+            data: { labels: Object.keys(data.breakdown), datasets: [{ label: 'Bot Requests', data: Object.values(data.breakdown), backgroundColor: '#0E7C86' }] },
+          });
+          document.getElementById('bot-404-note').textContent = `${data.not_found_404} 404s served to bots in range — wasted crawl budget.`;
+        });
+
+      window.initGrid(
+        'crawled-vs-visited-grid',
+        document.getElementById('crawled-vs-visited-grid').dataset.endpoint,
+        [{ name: 'Path' }, { name: 'Bot Hits' }, { name: 'Human Hits' }, { name: 'Bot Share %' }],
+        { from, to },
+      );
+    })();
+  </script>
+</div>

app/blueprints/seo/templates/seo/index.html

@@ -0,0 +1,5 @@
+{% extends "base.html" %}
+{% block title %}SEO & Bots — Kavosh{% endblock %}
+{% block content %}
+  {% include "seo/_content.html" %}
+{% endblock %}