start project

app/blueprints/security/__init__.py

@@ -0,0 +1,13 @@
+from flask import Blueprint
+from flask_login import login_required
+
+bp = Blueprint("security", __name__, template_folder="templates")
+
+
+@bp.before_request
+@login_required
+def require_login():
+    pass
+
+
+from app.blueprints.security import routes  # noqa: E402,F401  registers routes

app/blueprints/security/queries.py

@@ -0,0 +1,154 @@
+"""Context-builder query functions for the Security tab (Chapter 10),
+with the IP investigation panel's traffic breakdown upgraded to full-
+traffic data (bounded per-IP rollup, added as an explicit follow-up to
+Chapter 10's scope gap).
+"""
+from __future__ import annotations
+
+from collections import defaultdict
+from datetime import date
+
+from sqlalchemy import func
+
+from app.extensions import db
+from app.models.blocklist_suggestion import BlocklistSuggestion
+from app.models.bot_hit import BotHit
+from app.models.ip_registry import IPRegistry
+from app.models.ip_traffic_stats import IpPathStatsDaily, IpStatusStatsDaily
+from app.models.suspicious_event import SuspiciousEvent
+from app.services.severity_scoring import SeverityInputs, compute_effective_severity
+from app.utils.dates import day_bounds
+
+IP_HISTORY_EVENT_LIMIT = 50
+IP_HISTORY_PATH_LIMIT = 20
+
+
+def get_suspicious_events(
+    from_date: date, to_date: date, severity: str | None, rule_type: str | None, page: int, per_page: int,
+) -> tuple[list[list], int]:
+    """suspicious_events is small/indexed/indefinitely-retained (Ch06) —
+    same precedent as Ch09's bot_hits queries, so loading + escalating in
+    Python doesn't violate Ch03 rule 6 (that targets raw per-request rows).
+    """
+    start, end = day_bounds(from_date, to_date)
+    query = db.session.query(SuspiciousEvent).filter(
+        SuspiciousEvent.timestamp >= start, SuspiciousEvent.timestamp < end
+    )
+    if rule_type:
+        query = query.filter(SuspiciousEvent.rule_matched.like(f"{rule_type}:%"))
+    events = query.order_by(SuspiciousEvent.timestamp.desc()).all()
+
+    by_ip: dict[str, list[SuspiciousEvent]] = defaultdict(list)
+    for e in events:
+        by_ip[e.ip].append(e)
+
+    enriched = []
+    for e in events:
+        ip_events = by_ip[e.ip]
+        timestamps = sorted(ev.timestamp for ev in ip_events)
+        avg_interval = (
+            (timestamps[-1] - timestamps[0]).total_seconds() / (len(timestamps) - 1)
+            if len(timestamps) > 1 else None
+        )
+        effective = compute_effective_severity(
+            SeverityInputs(base_severity=e.severity, ip_event_count=len(ip_events), avg_interval_seconds=avg_interval)
+        )
+        if severity and effective != severity:
+            continue
+        enriched.append([e.timestamp.isoformat(), e.ip, e.path, e.rule_matched, effective])
+
+    total = len(enriched)
+    offset = (page - 1) * per_page
+    return enriched[offset : offset + per_page], total
+
+
+def get_sensitive_path_summary(from_date: date, to_date: date) -> list[dict]:
+    """Grouped by request path; filtered to Ch07's sensitive_path rule
+    category. One ranked list, not sub-grouped into config/admin/VCS —
+    Ch07's dictionary has no such taxonomy to reuse.
+    """
+    start, end = day_bounds(from_date, to_date)
+    rows = (
+        db.session.query(
+            SuspiciousEvent.path,
+            func.count().label("hit_count"),
+            func.count(func.distinct(SuspiciousEvent.ip)).label("distinct_ip_count"),
+        )
+        .filter(
+            SuspiciousEvent.timestamp >= start, SuspiciousEvent.timestamp < end,
+            SuspiciousEvent.rule_matched.like("sensitive_path:%"),
+        )
+        .group_by(SuspiciousEvent.path)
+        .order_by(func.count().desc())
+        .all()
+    )
+    return [{"path": r.path, "hit_count": r.hit_count, "distinct_ip_count": r.distinct_ip_count} for r in rows]
+
+
+def get_ip_history(ip: str) -> dict | None:
+    """Pulled from ip_registry (identity + true total_requests), plus
+    bot_hits/suspicious_events (flagged activity), plus the bounded
+    per-IP traffic rollup (top_paths / status_code_distribution — true
+    full-traffic breakdown, added as a follow-up to Ch10's original scope
+    gap). Retention caveat: the per-IP rollup covers roughly the last 30
+    days (see aggregator.py / flask cleanup).
+    """
+    registry = db.session.get(IPRegistry, ip)
+    if registry is None:
+        return None
+
+    path_rows = (
+        db.session.query(IpPathStatsDaily.path, func.sum(IpPathStatsDaily.count).label("count"))
+        .filter(IpPathStatsDaily.ip == ip)
+        .group_by(IpPathStatsDaily.path)
+        .order_by(func.sum(IpPathStatsDaily.count).desc())
+        .limit(IP_HISTORY_PATH_LIMIT)
+        .all()
+    )
+    status_rows = (
+        db.session.query(IpStatusStatsDaily.status_bucket, func.sum(IpStatusStatsDaily.count).label("count"))
+        .filter(IpStatusStatsDaily.ip == ip)
+        .group_by(IpStatusStatsDaily.status_bucket)
+        .all()
+    )
+
+    bot_rows = (
+        db.session.query(BotHit).filter(BotHit.ip == ip)
+        .order_by(BotHit.timestamp.desc()).limit(IP_HISTORY_EVENT_LIMIT).all()
+    )
+    suspicious_rows = (
+        db.session.query(SuspiciousEvent).filter(SuspiciousEvent.ip == ip)
+        .order_by(SuspiciousEvent.timestamp.desc()).limit(IP_HISTORY_EVENT_LIMIT).all()
+    )
+    spoofed_bot_names = sorted({b.bot_name for b in bot_rows if not b.verified})
+
+    return {
+        "ip": ip,
+        "first_seen": registry.first_seen.isoformat(),
+        "last_seen": registry.last_seen.isoformat(),
+        "total_requests": registry.total_requests,
+        "reputation_score": registry.reputation_score,
+        "is_flagged": registry.is_flagged,
+        "spoofed_bot_names": spoofed_bot_names,
+        "top_paths": [[r.path, r.count] for r in path_rows],
+        "status_code_distribution": {r.status_bucket: r.count for r in status_rows},
+        "traffic_window_note": "Path/status breakdown reflects roughly the last 30 days (bounded retention).",
+        "recent_suspicious_events": [
+            {"timestamp": s.timestamp.isoformat(), "path": s.path, "rule_matched": s.rule_matched, "severity": s.severity}
+            for s in suspicious_rows
+        ],
+    }
+
+
+def format_blocklist(suggestions: list[BlocklistSuggestion], fmt: str) -> str:
+    """Ch10: '.htaccess Deny/iptables/fail2ban-style'. 'plain' (a bare IP
+    list) is the most portable interpretation of "fail2ban-style input"
+    without assuming a specific fail2ban jail configuration Ch10 doesn't
+    specify.
+    """
+    ips = [s.ip for s in suggestions]
+    if fmt == "htaccess":
+        return "".join(f"Deny from {ip}\n" for ip in ips)
+    if fmt == "iptables":
+        return "".join(f"iptables -A INPUT -s {ip} -j DROP\n" for ip in ips)
+    return "".join(f"{ip}\n" for ip in ips)

app/blueprints/security/routes.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from flask import Response, jsonify, render_template, request
+
+from app.blueprints.security import bp
+from app.blueprints.security.queries import (
+    format_blocklist, get_ip_history, get_sensitive_path_summary, get_suspicious_events,
+)
+from app.extensions import db
+from app.models.blocklist_suggestion import BlocklistSuggestion
+from app.utils.dates import parse_date_range
+from app.utils.htmx import render_htmx_aware
+from app.utils.pagination import parse_pagination
+
+
+@bp.route("/security")
+def security():
+    from_date, to_date = parse_date_range(request)
+    severity = request.args.get("severity") or ""
+    rule_type = request.args.get("rule_type") or ""
+    return render_htmx_aware(
+        request, full_template="security/index.html", partial_template="security/_content.html",
+        from_date=from_date, to_date=to_date, severity=severity, rule_type=rule_type,
+    )
+
+
+@bp.get("/api/security/events")
+def api_security_events():
+    from_date, to_date = parse_date_range(request)
+    page, per_page = parse_pagination(request)
+    severity = request.args.get("severity") or None
+    rule_type = request.args.get("rule_type") or None
+    rows, total = get_suspicious_events(from_date, to_date, severity, rule_type, page, per_page)
+    return jsonify(
+        data={"rows": rows, "total": total},
+        meta={"from": from_date.isoformat(), "to": to_date.isoformat(), "page": page, "per_page": per_page},
+    )
+
+
+@bp.get("/api/security/sensitive-paths")
+def api_sensitive_paths():
+    from_date, to_date = parse_date_range(request)
+    return jsonify(data=get_sensitive_path_summary(from_date, to_date), meta={"from": from_date.isoformat(), "to": to_date.isoformat()})
+
+
+@bp.get("/api/security/ip/<ip>")
+def api_ip_history(ip: str):
+    history = get_ip_history(ip)
+    if history is None:
+        return render_template("security/_ip_not_found.html", ip=ip), 404
+    return render_template("security/_ip_history.html", ip_data=history)
+
+
+@bp.get("/api/security/export-blocklist")
+def api_export_blocklist():
+    fmt = request.args.get("format", "plain")
+    include_all = request.args.get("all", "false").lower() == "true"
+    query = BlocklistSuggestion.query
+    if not include_all:
+        query = query.filter_by(exported=False)
+    suggestions = query.order_by(BlocklistSuggestion.created_at).all()
+
+    body = format_blocklist(suggestions, fmt)
+    for s in suggestions:
+        s.exported = True
+    db.session.commit()
+
+    return Response(
+        body, mimetype="text/plain",
+        headers={"Content-Disposition": "attachment; filename=kavosh-blocklist.txt"},
+    )

app/blueprints/security/templates/security/_content.html

@@ -0,0 +1,125 @@
+<div id="security-content"
+     hx-get="{{ url_for('security.security') }}"
+     hx-trigger="change from:#security-filter-form"
+     hx-include="#security-filter-form"
+     hx-target="#security-content"
+     hx-swap="outerHTML"
+     data-from="{{ from_date.isoformat() }}"
+     data-to="{{ to_date.isoformat() }}"
+     data-severity="{{ severity }}"
+     data-rule-type="{{ rule_type }}">
+
+  <div class="flex flex-wrap items-end justify-between gap-4 mb-6">
+    <div>
+      <h1 class="font-display font-bold text-xl">Suspicious Requests &amp; IP History</h1>
+      <p class="text-sm text-muted dark:text-muted-dark">Who's poking at your site, and how hard</p>
+    </div>
+    <form id="security-filter-form" class="flex flex-wrap gap-3 items-end">
+      <label class="text-sm text-muted dark:text-muted-dark">From
+        <input type="date" name="from" value="{{ from_date.isoformat() }}" class="block border border-line dark:border-line-dark rounded-md px-2 py-1 mt-1 bg-surface dark:bg-surface-dark text-ink dark:text-ink-dark font-data text-sm">
+      </label>
+      <label class="text-sm text-muted dark:text-muted-dark">To
+        <input type="date" name="to" value="{{ to_date.isoformat() }}" class="block border border-line dark:border-line-dark rounded-md px-2 py-1 mt-1 bg-surface dark:bg-surface-dark text-ink dark:text-ink-dark font-data text-sm">
+      </label>
+      <label class="text-sm text-muted dark:text-muted-dark">Severity
+        <select name="severity" class="block border border-line dark:border-line-dark rounded-md px-2 py-1 mt-1 bg-surface dark:bg-surface-dark text-ink dark:text-ink-dark text-sm">
+          <option value="" {{ 'selected' if not severity }}>All</option>
+          <option value="low" {{ 'selected' if severity == 'low' }}>Low</option>
+          <option value="medium" {{ 'selected' if severity == 'medium' }}>Medium</option>
+          <option value="high" {{ 'selected' if severity == 'high' }}>High</option>
+        </select>
+      </label>
+      <label class="text-sm text-muted dark:text-muted-dark">Rule Type
+        <select name="rule_type" class="block border border-line dark:border-line-dark rounded-md px-2 py-1 mt-1 bg-surface dark:bg-surface-dark text-ink dark:text-ink-dark text-sm">
+          <option value="" {{ 'selected' if not rule_type }}>All</option>
+          <option value="sensitive_path" {{ 'selected' if rule_type == 'sensitive_path' }}>Sensitive Path</option>
+          <option value="injection" {{ 'selected' if rule_type == 'injection' }}>Injection</option>
+          <option value="scanner_ua" {{ 'selected' if rule_type == 'scanner_ua' }}>Scanner UA</option>
+          <option value="spoofed_bot" {{ 'selected' if rule_type == 'spoofed_bot' }}>Spoofed Bot</option>
+        </select>
+      </label>
+    </form>
+  </div>
+
+  <div class="mb-6">
+    <h3 class="text-xs font-medium uppercase tracking-wide text-muted dark:text-muted-dark mb-2">Suspicious events</h3>
+    <div id="suspicious-events-grid" data-endpoint="{{ url_for('security.api_security_events') }}"></div>
+  </div>
+
+  <div class="mb-6">
+    <h3 class="text-xs font-medium uppercase tracking-wide text-muted dark:text-muted-dark mb-2">Sensitive-path probes</h3>
+    <div id="sensitive-paths-panel" class="border border-line dark:border-line-dark rounded-xl bg-surface dark:bg-surface-dark overflow-hidden"
+         data-endpoint="{{ url_for('security.api_sensitive_paths') }}"></div>
+  </div>
+
+  <div class="mb-6 border border-line dark:border-line-dark rounded-xl p-4 bg-surface dark:bg-surface-dark">
+    <h3 class="font-display font-semibold text-sm mb-3">Export blocklist</h3>
+    <div class="flex flex-wrap gap-2 items-center">
+      <a href="{{ url_for('security.api_export_blocklist') }}"
+         class="bg-accent dark:bg-accent-dark text-white dark:text-paper-dark rounded-md px-3 py-1.5 text-sm font-medium hover:opacity-90 transition-opacity">Download new (.txt)</a>
+      <a href="{{ url_for('security.api_export_blocklist', all='true') }}"
+         class="border border-line dark:border-line-dark rounded-md px-3 py-1.5 text-sm hover:bg-paper dark:hover:bg-paper-dark transition-colors">Re-export all</a>
+      <select id="blocklist-format" class="border border-line dark:border-line-dark rounded-md px-2 py-1.5 text-sm bg-surface dark:bg-surface-dark text-ink dark:text-ink-dark" onchange="updateBlocklistLinks(this.value)">
+        <option value="plain">Plain IP list</option>
+        <option value="htaccess">.htaccess Deny</option>
+        <option value="iptables">iptables</option>
+      </select>
+    </div>
+  </div>
+
+  <div id="ip-history-modal" class="fixed inset-0 bg-ink/40 dark:bg-ink-dark/60 items-center justify-center empty:hidden flex z-50"></div>
+
+  <script>
+    (function initSecurityWidgets() {
+      const root = document.getElementById('security-content');
+      const from = root.dataset.from, to = root.dataset.to;
+      const severity = root.dataset.severity, ruleType = root.dataset.ruleType;
+
+      window.initGrid(
+        'suspicious-events-grid',
+        document.getElementById('suspicious-events-grid').dataset.endpoint,
+        [
+          { name: 'Timestamp' },
+          {
+            name: 'IP',
+            formatter: (cell) => window.gridHtml(
+              `<button class="text-accent dark:text-accent-dark underline" hx-get="/api/security/ip/${cell}" hx-target="#ip-history-modal" hx-swap="innerHTML">${cell}</button>`
+            ),
+          },
+          { name: 'Path' }, { name: 'Rule Matched' },
+          {
+            name: 'Severity',
+            formatter: (cell) => {
+              const tone = { low: 'text-muted dark:text-muted-dark', medium: 'text-warn dark:text-warn-dark', high: 'text-danger dark:text-danger-dark' }[cell] || '';
+              return window.gridHtml(`<span class="font-medium ${tone}">${cell}</span>`);
+            },
+          },
+        ],
+        { from, to, severity, rule_type: ruleType },
+      );
+
+      const pathsEl = document.getElementById('sensitive-paths-panel');
+      fetch(`${pathsEl.dataset.endpoint}?from=${from}&to=${to}`)
+        .then((r) => r.json())
+        .then(({ data }) => {
+          pathsEl.innerHTML = data.length
+            ? `<table class="w-full text-sm font-data">
+                 <thead><tr class="text-left text-muted dark:text-muted-dark text-xs uppercase tracking-wide bg-surface-raised dark:bg-surface-raised-dark font-sans">
+                   <th class="px-3 py-2">Path</th><th class="px-3 py-2">Hits</th><th class="px-3 py-2">Distinct IPs</th>
+                 </tr></thead>
+                 <tbody class="divide-y divide-line dark:divide-line-dark">${
+                   data.map((r) => `<tr><td class="px-3 py-2">${r.path}</td><td class="px-3 py-2">${r.hit_count}</td><td class="px-3 py-2">${r.distinct_ip_count}</td></tr>`).join('')
+                 }</tbody></table>`
+            : `<p class="text-sm text-muted dark:text-muted-dark p-4">No sensitive-path probes in range.</p>`;
+        });
+
+      window.updateBlocklistLinks = (fmt) => {
+        document.querySelectorAll('a[href*="export-blocklist"]').forEach((a) => {
+          const url = new URL(a.href, window.location.origin);
+          url.searchParams.set('format', fmt);
+          a.href = url.toString();
+        });
+      };
+    })();
+  </script>
+</div>

app/blueprints/security/templates/security/_ip_history.html

@@ -0,0 +1,26 @@
+<div class="bg-surface dark:bg-surface-dark rounded-xl p-6 max-w-lg w-full relative border border-line dark:border-line-dark">
+  <button class="absolute top-3 right-3 text-muted dark:text-muted-dark hover:text-ink dark:hover:text-ink-dark" onclick="document.getElementById('ip-history-modal').innerHTML=''">
+    <svg class="w-4 h-4"><use href="/static/dist/icons.svg#x"/></svg>
+  </button>
+  <h3 class="font-display font-semibold text-lg mb-3 font-data">{{ ip_data.ip }}</h3>
+  <dl class="text-sm grid grid-cols-2 gap-y-1.5 mb-4 font-data">
+    <dt class="text-muted dark:text-muted-dark font-sans">First seen</dt><dd>{{ ip_data.first_seen }}</dd>
+    <dt class="text-muted dark:text-muted-dark font-sans">Last seen</dt><dd>{{ ip_data.last_seen }}</dd>
+    <dt class="text-muted dark:text-muted-dark font-sans">Total requests</dt><dd>{{ ip_data.total_requests }}</dd>
+    <dt class="text-muted dark:text-muted-dark font-sans">Reputation score</dt><dd>{{ ip_data.reputation_score }}</dd>
+    <dt class="text-muted dark:text-muted-dark font-sans">Flagged</dt>
+    <dd class="{{ 'text-danger dark:text-danger-dark font-medium' if ip_data.is_flagged else '' }}">{{ 'Yes' if ip_data.is_flagged else 'No' }}</dd>
+    {% if ip_data.spoofed_bot_names %}
+    <dt class="text-muted dark:text-muted-dark font-sans">Spoofed bot claims</dt><dd class="text-danger dark:text-danger-dark">{{ ip_data.spoofed_bot_names | join(', ') }}</dd>
+    {% endif %}
+  </dl>
+  <p class="text-xs text-muted dark:text-muted-dark mb-3">{{ ip_data.traffic_window_note }}</p>
+  <h4 class="font-medium text-sm mb-1">Top paths</h4>
+  <ul class="text-sm mb-3 font-data text-ink dark:text-ink-dark space-y-0.5">{% for path, count in ip_data.top_paths %}<li>{{ path }} <span class="text-muted dark:text-muted-dark">— {{ count }}</span></li>{% endfor %}</ul>
+  <h4 class="font-medium text-sm mb-1">Status codes</h4>
+  <ul class="text-sm mb-3 font-data text-ink dark:text-ink-dark space-y-0.5">{% for code, count in ip_data.status_code_distribution.items() %}<li>{{ code }} <span class="text-muted dark:text-muted-dark">— {{ count }}</span></li>{% endfor %}</ul>
+  {% if ip_data.recent_suspicious_events %}
+  <h4 class="font-medium text-sm mb-1">Recent flagged events</h4>
+  <ul class="text-sm font-data text-ink dark:text-ink-dark space-y-0.5">{% for e in ip_data.recent_suspicious_events %}<li>{{ e.timestamp }} — {{ e.path }} <span class="text-muted dark:text-muted-dark">({{ e.rule_matched }}, {{ e.severity }})</span></li>{% endfor %}</ul>
+  {% endif %}
+</div>

app/blueprints/security/templates/security/_ip_not_found.html

@@ -0,0 +1,4 @@
+<div class="bg-surface dark:bg-surface-dark rounded-xl p-6 max-w-sm w-full border border-line dark:border-line-dark">
+  <p class="text-sm text-ink dark:text-ink-dark">No history found for <span class="font-data">{{ ip }}</span> — it hasn't been seen yet.</p>
+  <button onclick="document.getElementById('ip-history-modal').innerHTML=''" class="mt-3 text-sm text-accent dark:text-accent-dark hover:underline">Close</button>
+</div>

app/blueprints/security/templates/security/index.html

@@ -0,0 +1,5 @@
+{% extends "base.html" %}
+{% block title %}Security — Kavosh{% endblock %}
+{% block content %}
+  {% include "security/_content.html" %}
+{% endblock %}