start project
This commit is contained in:
Hemmat
@@ -0,0 +1,7 @@
|
||||
"""Deliberately minimal — Chapter 02's tree lists a standalone `api`
|
||||
blueprint, but Chapter 04/11 route each section's JSON endpoints through
|
||||
its own blueprint instead. Left unpopulated per the project owner's
|
||||
instruction to defer this branch."""
|
||||
from flask import Blueprint
|
||||
|
||||
bp = Blueprint("api", __name__)
|
||||
@@ -0,0 +1,5 @@
|
||||
from flask import Blueprint
|
||||
|
||||
bp = Blueprint("auth", __name__, template_folder="templates")
|
||||
|
||||
from app.blueprints.auth import routes # noqa: E402,F401 registers routes
|
||||
@@ -0,0 +1,18 @@
|
||||
"""Login form (Chapter 12). CSRF handled automatically via form.hidden_tag()
|
||||
(Flask-WTF, Ch12's CSRF requirement)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from flask_wtf import FlaskForm
|
||||
from wtforms import PasswordField, StringField, SubmitField
|
||||
from wtforms.validators import DataRequired
|
||||
|
||||
|
||||
class LoginForm(FlaskForm):
|
||||
# NOTE: no Email() validator — that validator requires the extra
|
||||
# `email-validator` package. Login checks the value against a stored
|
||||
# user row anyway, so a malformed email simply fails to match rather
|
||||
# than needing format validation up front; kept simple to avoid a new
|
||||
# dependency.
|
||||
email = StringField("Email", validators=[DataRequired()])
|
||||
password = PasswordField("Password", validators=[DataRequired()])
|
||||
submit = SubmitField("Log in")
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Single-admin auth routes (Chapter 12 / Chapter 11's route table)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from flask import flash, redirect, render_template, request, url_for
|
||||
from flask_login import current_user, login_required, login_user, logout_user
|
||||
|
||||
from app.blueprints.auth import bp
|
||||
from app.blueprints.auth.forms import LoginForm
|
||||
from app.extensions import db, limiter, login_manager
|
||||
from app.models.user import User
|
||||
|
||||
|
||||
@login_manager.user_loader
|
||||
def load_user(user_id: str) -> User | None:
|
||||
return db.session.get(User, int(user_id))
|
||||
|
||||
|
||||
@bp.route("/login", methods=["GET", "POST"])
|
||||
@limiter.limit("10 per minute") # Ch12: rate limiting on /login at minimum
|
||||
def login():
|
||||
if current_user.is_authenticated:
|
||||
return redirect(url_for("overview.overview"))
|
||||
|
||||
form = LoginForm()
|
||||
if form.validate_on_submit():
|
||||
user = User.query.filter_by(email=form.email.data.strip().lower()).first()
|
||||
if user is not None and user.check_password(form.password.data):
|
||||
login_user(user)
|
||||
next_url = request.args.get("next") or url_for("overview.overview")
|
||||
return redirect(next_url)
|
||||
flash("Invalid email or password.")
|
||||
|
||||
return render_template("auth/login.html", form=form)
|
||||
|
||||
|
||||
@bp.post("/logout")
|
||||
@login_required
|
||||
def logout():
|
||||
# NOTE (flagged): Ch11's route table lists "/login, /logout" under a
|
||||
# shared "GET/POST" column. Logout is POST-only here — a state-changing
|
||||
# action behind a plain GET is a CSRF-adjacent anti-pattern Ch12's own
|
||||
# CSRF requirement argues against; login stays GET (show form) + POST
|
||||
# (submit), matching the table as-is.
|
||||
logout_user()
|
||||
return redirect(url_for("auth.login"))
|
||||
@@ -0,0 +1,27 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Log in — Kavosh{% endblock %}
|
||||
{% block content %}
|
||||
<div class="max-w-sm mx-auto mt-16 sm:mt-24">
|
||||
<div class="text-center mb-6">
|
||||
<h1 class="font-display font-bold text-2xl tracking-tight">Kavosh</h1>
|
||||
<p class="text-sm text-muted dark:text-muted-dark mt-1">Sign in to view your site's traffic</p>
|
||||
</div>
|
||||
<div class="bg-surface dark:bg-surface-dark border border-line dark:border-line-dark rounded-xl p-6 shadow-sm">
|
||||
{% for message in get_flashed_messages() %}
|
||||
<p class="text-danger dark:text-danger-dark text-sm mb-3 bg-danger/10 dark:bg-danger-dark/10 rounded-md px-3 py-2">{{ message }}</p>
|
||||
{% endfor %}
|
||||
<form method="post">
|
||||
{{ form.hidden_tag() }}
|
||||
<div class="mb-4">
|
||||
{{ form.email.label(class="block text-sm font-medium text-muted dark:text-muted-dark mb-1") }}
|
||||
{{ form.email(class="w-full border border-line dark:border-line-dark rounded-md px-3 py-2 bg-paper dark:bg-paper-dark text-ink dark:text-ink-dark focus:outline-none focus:ring-2 focus:ring-accent dark:focus:ring-accent-dark", autofocus=true) }}
|
||||
</div>
|
||||
<div class="mb-5">
|
||||
{{ form.password.label(class="block text-sm font-medium text-muted dark:text-muted-dark mb-1") }}
|
||||
{{ form.password(class="w-full border border-line dark:border-line-dark rounded-md px-3 py-2 bg-paper dark:bg-paper-dark text-ink dark:text-ink-dark focus:outline-none focus:ring-2 focus:ring-accent dark:focus:ring-accent-dark") }}
|
||||
</div>
|
||||
{{ form.submit(class="w-full bg-accent dark:bg-accent-dark text-white dark:text-paper-dark font-medium rounded-md px-4 py-2 hover:opacity-90 transition-opacity cursor-pointer") }}
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,13 @@
|
||||
from flask import Blueprint
|
||||
from flask_login import login_required
|
||||
|
||||
bp = Blueprint("overview", __name__, template_folder="templates")
|
||||
|
||||
|
||||
@bp.before_request
|
||||
@login_required
|
||||
def require_login():
|
||||
pass
|
||||
|
||||
|
||||
from app.blueprints.overview import routes # noqa: E402,F401 registers routes
|
||||
@@ -0,0 +1,167 @@
|
||||
"""Context-builder query functions for the Overview tab (Chapter 08).
|
||||
|
||||
Every function here reads request_stats_hourly / request_stats_daily /
|
||||
referrer_stats_daily / browser_stats_daily — never log_entries — per
|
||||
Ch03 rule 6 / Ch08's own data-source rule.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
from sqlalchemy import case, func
|
||||
|
||||
from app.extensions import db
|
||||
from app.models.browser_stats import BrowserStatsDaily
|
||||
from app.models.referrer_stats import ReferrerStatsDaily
|
||||
from app.models.request_stats import RequestStatsDaily, RequestStatsHourly
|
||||
from app.utils.dates import day_bounds
|
||||
|
||||
# ASSUMPTION (flagged in Ch08): Ch08 doesn't give a numeric threshold for
|
||||
# "hourly vs daily depending on range width" — picked 3 days.
|
||||
HOURLY_GRANULARITY_THRESHOLD_DAYS = 3
|
||||
|
||||
|
||||
def get_kpis(from_date: date, to_date: date) -> dict:
|
||||
"""All six Ch08 KPI-card fields, plus peak-day (folded in — Ch11 has
|
||||
no dedicated route for it and Ch08 calls for only a 'simple max-lookup').
|
||||
"""
|
||||
row = (
|
||||
db.session.query(
|
||||
func.coalesce(func.sum(RequestStatsDaily.count), 0).label("total_requests"),
|
||||
func.coalesce(func.sum(RequestStatsDaily.unique_ips), 0).label("unique_ips_sum"),
|
||||
func.coalesce(func.sum(RequestStatsDaily.bytes_sum), 0).label("total_bandwidth"),
|
||||
func.coalesce(func.sum(RequestStatsDaily.error_count), 0).label("total_errors"),
|
||||
)
|
||||
.filter(RequestStatsDaily.date >= from_date, RequestStatsDaily.date <= to_date)
|
||||
.one()
|
||||
)
|
||||
num_days = (to_date - from_date).days + 1
|
||||
avg_response_size = (row.total_bandwidth / row.total_requests) if row.total_requests else 0.0
|
||||
error_rate_pct = (row.total_errors / row.total_requests * 100) if row.total_requests else 0.0
|
||||
avg_requests_per_day = row.total_requests / num_days if num_days else 0.0
|
||||
|
||||
peak_row = (
|
||||
db.session.query(RequestStatsDaily.date, RequestStatsDaily.count)
|
||||
.filter(RequestStatsDaily.date >= from_date, RequestStatsDaily.date <= to_date)
|
||||
.order_by(RequestStatsDaily.count.desc())
|
||||
.first()
|
||||
)
|
||||
|
||||
return {
|
||||
"total_requests": row.total_requests,
|
||||
# APPROXIMATION (flagged in Ch08): sum of daily unique_ips over-counts
|
||||
# repeat visitors across days.
|
||||
"unique_ips": row.unique_ips_sum,
|
||||
"total_bandwidth_bytes": row.total_bandwidth,
|
||||
"avg_response_size_bytes": round(avg_response_size, 1),
|
||||
"error_rate_pct": round(error_rate_pct, 2),
|
||||
"avg_requests_per_day": round(avg_requests_per_day, 1),
|
||||
"peak_day": {"date": peak_row.date.isoformat(), "count": peak_row.count} if peak_row else None,
|
||||
}
|
||||
|
||||
|
||||
def get_traffic_chart_series(from_date: date, to_date: date) -> dict:
|
||||
span_days = (to_date - from_date).days + 1
|
||||
if span_days <= HOURLY_GRANULARITY_THRESHOLD_DAYS:
|
||||
start, end = day_bounds(from_date, to_date)
|
||||
rows = (
|
||||
db.session.query(RequestStatsHourly.date_hour, func.sum(RequestStatsHourly.count).label("count"))
|
||||
.filter(RequestStatsHourly.date_hour >= start, RequestStatsHourly.date_hour < end)
|
||||
.group_by(RequestStatsHourly.date_hour)
|
||||
.order_by(RequestStatsHourly.date_hour)
|
||||
.all()
|
||||
)
|
||||
return {"granularity": "hourly", "series": [{"t": r.date_hour.isoformat(), "count": r.count} for r in rows]}
|
||||
|
||||
rows = (
|
||||
db.session.query(RequestStatsDaily.date, RequestStatsDaily.count)
|
||||
.filter(RequestStatsDaily.date >= from_date, RequestStatsDaily.date <= to_date)
|
||||
.order_by(RequestStatsDaily.date)
|
||||
.all()
|
||||
)
|
||||
return {"granularity": "daily", "series": [{"t": r.date.isoformat(), "count": r.count} for r in rows]}
|
||||
|
||||
|
||||
def get_status_code_breakdown(from_date: date, to_date: date) -> dict:
|
||||
start, end = day_bounds(from_date, to_date)
|
||||
bucket = case(
|
||||
(RequestStatsHourly.status_code < 300, "2xx"),
|
||||
(RequestStatsHourly.status_code < 400, "3xx"),
|
||||
(RequestStatsHourly.status_code < 500, "4xx"),
|
||||
else_="5xx",
|
||||
)
|
||||
rows = (
|
||||
db.session.query(bucket.label("bucket"), func.sum(RequestStatsHourly.count).label("count"))
|
||||
.filter(RequestStatsHourly.date_hour >= start, RequestStatsHourly.date_hour < end)
|
||||
.group_by("bucket")
|
||||
.all()
|
||||
)
|
||||
breakdown = {"2xx": 0, "3xx": 0, "4xx": 0, "5xx": 0}
|
||||
for r in rows:
|
||||
breakdown[r.bucket] = r.count
|
||||
return breakdown
|
||||
|
||||
|
||||
def get_top_urls(from_date: date, to_date: date, page: int, per_page: int) -> tuple[list[list], int]:
|
||||
"""Top URLs by hits. NOTE (flagged in Ch08): only available within the
|
||||
~90-day hourly retention window (Ch06) — request_stats_daily has no
|
||||
path column, so a wider range returns nothing here.
|
||||
"""
|
||||
start, end = day_bounds(from_date, to_date)
|
||||
hits = func.sum(RequestStatsHourly.count)
|
||||
errors = func.sum(case((RequestStatsHourly.status_code >= 400, RequestStatsHourly.count), else_=0))
|
||||
bytes_sum = func.sum(RequestStatsHourly.bytes_sent_sum)
|
||||
|
||||
base_query = (
|
||||
db.session.query(RequestStatsHourly.path, hits.label("hits"), bytes_sum.label("bytes_sum"), errors.label("errors"))
|
||||
.filter(RequestStatsHourly.date_hour >= start, RequestStatsHourly.date_hour < end)
|
||||
.group_by(RequestStatsHourly.path)
|
||||
)
|
||||
total = base_query.count()
|
||||
rows = base_query.order_by(hits.desc()).offset((page - 1) * per_page).limit(per_page).all()
|
||||
|
||||
results = [
|
||||
[
|
||||
r.path,
|
||||
r.hits,
|
||||
round(r.bytes_sum / r.hits, 1) if r.hits else 0.0,
|
||||
round(r.errors / r.hits * 100, 2) if r.hits else 0.0,
|
||||
]
|
||||
for r in rows
|
||||
]
|
||||
return results, total
|
||||
|
||||
|
||||
def get_top_referrers(from_date: date, to_date: date, page: int, per_page: int) -> tuple[list[list], int]:
|
||||
"""Domain-bucketed referrers (Method A, Ch08 follow-up)."""
|
||||
hits = func.sum(ReferrerStatsDaily.count)
|
||||
base_query = (
|
||||
db.session.query(ReferrerStatsDaily.referrer_domain, hits.label("hits"))
|
||||
.filter(ReferrerStatsDaily.date >= from_date, ReferrerStatsDaily.date <= to_date)
|
||||
.group_by(ReferrerStatsDaily.referrer_domain)
|
||||
)
|
||||
total = base_query.count()
|
||||
rows = base_query.order_by(hits.desc()).offset((page - 1) * per_page).limit(per_page).all()
|
||||
return [[r.referrer_domain, r.hits] for r in rows], total
|
||||
|
||||
|
||||
def get_browser_breakdown(from_date: date, to_date: date) -> dict:
|
||||
"""Human-only (bots excluded at rollup-write time, Ch08)."""
|
||||
browser_rows = (
|
||||
db.session.query(BrowserStatsDaily.browser, func.sum(BrowserStatsDaily.count).label("count"))
|
||||
.filter(BrowserStatsDaily.date >= from_date, BrowserStatsDaily.date <= to_date)
|
||||
.group_by(BrowserStatsDaily.browser)
|
||||
.order_by(func.sum(BrowserStatsDaily.count).desc())
|
||||
.all()
|
||||
)
|
||||
os_rows = (
|
||||
db.session.query(BrowserStatsDaily.os, func.sum(BrowserStatsDaily.count).label("count"))
|
||||
.filter(BrowserStatsDaily.date >= from_date, BrowserStatsDaily.date <= to_date)
|
||||
.group_by(BrowserStatsDaily.os)
|
||||
.order_by(func.sum(BrowserStatsDaily.count).desc())
|
||||
.all()
|
||||
)
|
||||
return {
|
||||
"by_browser": [{"name": r.browser, "count": r.count} for r in browser_rows],
|
||||
"by_os": [{"name": r.os, "count": r.count} for r in os_rows],
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Overview blueprint routes (Chapter 08 / Chapter 11's route table)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from flask import current_app, jsonify, request
|
||||
|
||||
from app.blueprints.overview import bp
|
||||
from app.blueprints.overview.queries import (
|
||||
get_browser_breakdown,
|
||||
get_kpis,
|
||||
get_status_code_breakdown,
|
||||
get_top_referrers,
|
||||
get_top_urls,
|
||||
get_traffic_chart_series,
|
||||
)
|
||||
from app.services.background import resume_incomplete_files
|
||||
from app.utils.dates import parse_date_range
|
||||
from app.utils.htmx import render_htmx_aware
|
||||
from app.utils.pagination import parse_pagination
|
||||
|
||||
|
||||
@bp.route("/overview")
|
||||
def overview():
|
||||
"""Full page on first load, HTMX partial on tab switch / range change (Ch04).
|
||||
|
||||
Also opportunistically resumes any log_files stuck in queued/processing
|
||||
(Ch12 simplification: the replacement for cron's "there's always a
|
||||
next tick" guarantee — see app/services/background.py).
|
||||
"""
|
||||
resume_incomplete_files(current_app._get_current_object())
|
||||
from_date, to_date = parse_date_range(request)
|
||||
return render_htmx_aware(
|
||||
request,
|
||||
full_template="overview/index.html",
|
||||
partial_template="overview/_content.html",
|
||||
from_date=from_date,
|
||||
to_date=to_date,
|
||||
)
|
||||
|
||||
|
||||
@bp.get("/api/overview/kpis")
|
||||
def api_kpis():
|
||||
from_date, to_date = parse_date_range(request)
|
||||
return jsonify(data=get_kpis(from_date, to_date), meta={"from": from_date.isoformat(), "to": to_date.isoformat()})
|
||||
|
||||
|
||||
@bp.get("/api/overview/traffic-chart")
|
||||
def api_traffic_chart():
|
||||
from_date, to_date = parse_date_range(request)
|
||||
return jsonify(
|
||||
data=get_traffic_chart_series(from_date, to_date),
|
||||
meta={"from": from_date.isoformat(), "to": to_date.isoformat()},
|
||||
)
|
||||
|
||||
|
||||
@bp.get("/api/overview/status-codes")
|
||||
def api_status_codes():
|
||||
from_date, to_date = parse_date_range(request)
|
||||
return jsonify(
|
||||
data=get_status_code_breakdown(from_date, to_date),
|
||||
meta={"from": from_date.isoformat(), "to": to_date.isoformat()},
|
||||
)
|
||||
|
||||
|
||||
@bp.get("/api/overview/top-urls")
|
||||
def api_top_urls():
|
||||
from_date, to_date = parse_date_range(request)
|
||||
page, per_page = parse_pagination(request)
|
||||
rows, total = get_top_urls(from_date, to_date, page, per_page)
|
||||
return jsonify(
|
||||
data={"rows": rows, "total": total},
|
||||
meta={"from": from_date.isoformat(), "to": to_date.isoformat(), "page": page, "per_page": per_page},
|
||||
)
|
||||
|
||||
|
||||
@bp.get("/api/overview/top-referrers")
|
||||
def api_top_referrers():
|
||||
from_date, to_date = parse_date_range(request)
|
||||
page, per_page = parse_pagination(request)
|
||||
rows, total = get_top_referrers(from_date, to_date, page, per_page)
|
||||
return jsonify(
|
||||
data={"rows": rows, "total": total},
|
||||
meta={"from": from_date.isoformat(), "to": to_date.isoformat(), "page": page, "per_page": per_page},
|
||||
)
|
||||
|
||||
|
||||
@bp.get("/api/overview/browser-breakdown")
|
||||
def api_browser_breakdown():
|
||||
"""Chapter 11 addition (Method A, Ch08 follow-up) — not in the
|
||||
original route table; see docs/api-contract-final.md."""
|
||||
from_date, to_date = parse_date_range(request)
|
||||
return jsonify(
|
||||
data=get_browser_breakdown(from_date, to_date),
|
||||
meta={"from": from_date.isoformat(), "to": to_date.isoformat()},
|
||||
)
|
||||
@@ -0,0 +1,286 @@
|
||||
<div id="overview-content"
|
||||
hx-get="{{ url_for('overview.overview') }}"
|
||||
hx-trigger="change from:#date-range-form"
|
||||
hx-include="#date-range-form"
|
||||
hx-target="#overview-content"
|
||||
hx-swap="outerHTML"
|
||||
data-from="{{ from_date.isoformat() }}"
|
||||
data-to="{{ to_date.isoformat() }}">
|
||||
|
||||
<div class="flex flex-wrap items-end justify-between gap-4 mb-6">
|
||||
<div>
|
||||
<h1 class="font-display font-bold text-xl">Overview</h1>
|
||||
<p class="text-sm text-muted dark:text-muted-dark">What happened on your site recently</p>
|
||||
</div>
|
||||
<form id="date-range-form" class="flex gap-3 items-end">
|
||||
<label class="text-sm text-muted dark:text-muted-dark">From
|
||||
<input type="date" name="from" value="{{ from_date.isoformat() }}"
|
||||
class="block border border-line dark:border-line-dark rounded-md px-2 py-1 mt-1 bg-surface dark:bg-surface-dark text-ink dark:text-ink-dark font-data text-sm">
|
||||
</label>
|
||||
<label class="text-sm text-muted dark:text-muted-dark">To
|
||||
<input type="date" name="to" value="{{ to_date.isoformat() }}"
|
||||
class="block border border-line dark:border-line-dark rounded-md px-2 py-1 mt-1 bg-surface dark:bg-surface-dark text-ink dark:text-ink-dark font-data text-sm">
|
||||
</label>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<!-- Upload panel — real byte-transfer progress via XHR (uploads.js), then
|
||||
a real parse-progress bar (Ch12) once the file is queued. No cron
|
||||
required: processing starts automatically in the background. -->
|
||||
<div class="mb-6 border border-line dark:border-line-dark rounded-xl p-4 bg-surface dark:bg-surface-dark">
|
||||
<h2 class="font-display font-semibold text-sm mb-3">Upload a log file</h2>
|
||||
<form id="upload-form" action="{{ url_for('uploads.upload_log_file') }}"
|
||||
class="flex flex-wrap gap-3 items-center">
|
||||
<input type="file" name="logfile" accept=".log,.txt,.gz" required
|
||||
class="text-sm text-muted dark:text-muted-dark file:mr-3 file:py-1.5 file:px-3 file:rounded-md file:border-0 file:bg-accent/10 file:text-accent dark:file:bg-accent-dark/15 dark:file:text-accent-dark file:text-sm file:font-medium hover:file:bg-accent/20 dark:hover:file:bg-accent-dark/25 file:cursor-pointer cursor-pointer">
|
||||
<select name="server_type" class="border border-line dark:border-line-dark rounded-md px-2 py-1.5 text-sm bg-surface dark:bg-surface-dark text-ink dark:text-ink-dark">
|
||||
<option value="apache">Apache</option>
|
||||
<option value="litespeed">LiteSpeed</option>
|
||||
</select>
|
||||
<input type="text" name="format_string" placeholder="LogFormat (optional — defaults to Combined)"
|
||||
class="border border-line dark:border-line-dark rounded-md px-2 py-1.5 text-sm flex-1 min-w-[220px] bg-surface dark:bg-surface-dark text-ink dark:text-ink-dark placeholder:text-muted dark:placeholder:text-muted-dark">
|
||||
<button type="submit" class="bg-accent dark:bg-accent-dark text-white dark:text-paper-dark text-sm font-medium rounded-md px-4 py-1.5 hover:opacity-90 transition-opacity">
|
||||
Upload
|
||||
</button>
|
||||
</form>
|
||||
|
||||
<div id="upload-progress-wrap" class="hidden mt-3">
|
||||
<div class="flex items-center justify-between text-sm mb-1">
|
||||
<span id="upload-progress-label" class="text-muted dark:text-muted-dark">Uploading…</span>
|
||||
</div>
|
||||
<div class="w-full h-2 rounded-full bg-line dark:bg-line-dark overflow-hidden">
|
||||
<div id="upload-progress-bar" class="h-full rounded-full bg-accent dark:bg-accent-dark transition-all duration-150" style="width: 0%"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="upload-result" class="mt-3"></div>
|
||||
</div>
|
||||
|
||||
<!-- Uploaded files — select and delete (project-owner follow-up
|
||||
request). Independent of the date-range picker above: this lists
|
||||
uploads by when they arrived, not by which log dates they cover
|
||||
(a single file can span many dates). Selection is intentionally
|
||||
scoped to the currently-rendered page/sort/search view — it
|
||||
doesn't try to persist across a Grid.js re-render, which keeps
|
||||
"what's selected" always visually honest at the cost of losing
|
||||
selection if you page/sort/search mid-selection. -->
|
||||
<div class="mb-6 border border-line dark:border-line-dark rounded-xl bg-surface dark:bg-surface-dark overflow-hidden">
|
||||
<div class="flex items-center justify-between p-4 pb-3">
|
||||
<h2 class="font-display font-semibold text-sm">Uploaded files</h2>
|
||||
<button type="button" id="delete-selected-btn" disabled
|
||||
class="text-sm font-medium rounded-md px-3 py-1.5 border border-danger/40 dark:border-danger-dark/40 text-danger dark:text-danger-dark opacity-40 cursor-not-allowed disabled:opacity-40 enabled:opacity-100 enabled:hover:bg-danger/10 dark:enabled:hover:bg-danger-dark/10 transition-opacity">
|
||||
Delete selected
|
||||
</button>
|
||||
</div>
|
||||
<div id="uploaded-files-grid" class="px-4 pb-4" data-endpoint="{{ url_for('uploads.list_uploads') }}"></div>
|
||||
</div>
|
||||
|
||||
<!-- KPI readout — the one deliberate signature treatment: values set in
|
||||
tabular mono, like numbers straight off the log line, with a thin
|
||||
top rule that switches color when a metric needs attention
|
||||
(structure carries information, not just decoration). -->
|
||||
<div id="kpi-cards" class="grid grid-cols-2 sm:grid-cols-3 gap-3 mb-6" data-endpoint="{{ url_for('overview.api_kpis') }}"></div>
|
||||
|
||||
<div class="grid grid-cols-1 lg:grid-cols-2 gap-4 mb-6">
|
||||
<div class="border border-line dark:border-line-dark rounded-xl p-4 bg-surface dark:bg-surface-dark h-72">
|
||||
<h3 class="text-xs font-medium uppercase tracking-wide text-muted dark:text-muted-dark mb-2">Traffic over time</h3>
|
||||
<div class="h-56"><canvas id="traffic-chart" data-endpoint="{{ url_for('overview.api_traffic_chart') }}"></canvas></div>
|
||||
</div>
|
||||
<div class="border border-line dark:border-line-dark rounded-xl p-4 bg-surface dark:bg-surface-dark h-72">
|
||||
<h3 class="text-xs font-medium uppercase tracking-wide text-muted dark:text-muted-dark mb-2">Status codes</h3>
|
||||
<div class="h-56"><canvas id="status-code-chart" data-endpoint="{{ url_for('overview.api_status_codes') }}"></canvas></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-cols-1 lg:grid-cols-2 gap-4">
|
||||
<div>
|
||||
<h3 class="text-xs font-medium uppercase tracking-wide text-muted dark:text-muted-dark mb-2">Top URLs</h3>
|
||||
<div id="top-urls-grid" data-endpoint="{{ url_for('overview.api_top_urls') }}"></div>
|
||||
</div>
|
||||
<div>
|
||||
<h3 class="text-xs font-medium uppercase tracking-wide text-muted dark:text-muted-dark mb-2">Top referrers</h3>
|
||||
<div id="top-referrers-grid" data-endpoint="{{ url_for('overview.api_top_referrers') }}"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-cols-1 lg:grid-cols-2 gap-4 mt-6">
|
||||
<div class="border border-line dark:border-line-dark rounded-xl p-4 bg-surface dark:bg-surface-dark h-64">
|
||||
<h3 class="text-xs font-medium uppercase tracking-wide text-muted dark:text-muted-dark mb-2">Browsers</h3>
|
||||
<div class="h-48"><canvas id="browser-breakdown-chart" data-endpoint="{{ url_for('overview.api_browser_breakdown') }}"></canvas></div>
|
||||
</div>
|
||||
<div class="border border-line dark:border-line-dark rounded-xl p-4 bg-surface dark:bg-surface-dark h-64">
|
||||
<h3 class="text-xs font-medium uppercase tracking-wide text-muted dark:text-muted-dark mb-2">Operating systems</h3>
|
||||
<div class="h-48"><canvas id="os-breakdown-chart"></canvas></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function initOverviewWidgets() {
|
||||
const root = document.getElementById('overview-content');
|
||||
const from = root.dataset.from, to = root.dataset.to;
|
||||
const withRange = (url) => `${url}?from=${from}&to=${to}`;
|
||||
|
||||
window.initUploadForm('upload-form');
|
||||
initUploadedFilesGrid();
|
||||
|
||||
fetch(withRange(document.getElementById('kpi-cards').dataset.endpoint))
|
||||
.then((r) => r.json())
|
||||
.then(({ data }) => renderKpiCards(data));
|
||||
|
||||
const trafficEl = document.getElementById('traffic-chart');
|
||||
fetch(withRange(trafficEl.dataset.endpoint))
|
||||
.then((r) => r.json())
|
||||
.then(({ data }) => window.initChart('traffic-chart', {
|
||||
type: 'line',
|
||||
data: {
|
||||
labels: data.series.map((p) => p.t),
|
||||
datasets: [{
|
||||
label: 'Requests', data: data.series.map((p) => p.count), tension: 0.3,
|
||||
borderColor: '#0E7C86', backgroundColor: '#0E7C8622', fill: true,
|
||||
}],
|
||||
},
|
||||
}));
|
||||
|
||||
const statusEl = document.getElementById('status-code-chart');
|
||||
fetch(withRange(statusEl.dataset.endpoint))
|
||||
.then((r) => r.json())
|
||||
.then(({ data }) => window.initChart('status-code-chart', {
|
||||
type: 'doughnut',
|
||||
data: { labels: Object.keys(data), datasets: [{ data: Object.values(data), backgroundColor: window.KAVOSH_CHART_PALETTE }] },
|
||||
}));
|
||||
|
||||
window.initGrid(
|
||||
'top-urls-grid',
|
||||
document.getElementById('top-urls-grid').dataset.endpoint,
|
||||
[{ name: 'Path' }, { name: 'Hits' }, { name: 'Avg Size (B)' }, { name: 'Error Rate %' }],
|
||||
{ from, to },
|
||||
);
|
||||
|
||||
window.initGrid(
|
||||
'top-referrers-grid',
|
||||
document.getElementById('top-referrers-grid').dataset.endpoint,
|
||||
[{ name: 'Referrer Domain' }, { name: 'Hits' }],
|
||||
{ from, to },
|
||||
);
|
||||
|
||||
const browserEl = document.getElementById('browser-breakdown-chart');
|
||||
fetch(withRange(browserEl.dataset.endpoint))
|
||||
.then((r) => r.json())
|
||||
.then(({ data }) => {
|
||||
window.initChart('browser-breakdown-chart', {
|
||||
type: 'bar',
|
||||
data: { labels: data.by_browser.map((r) => r.name), datasets: [{ label: 'Browser', data: data.by_browser.map((r) => r.count), backgroundColor: '#0E7C86' }] },
|
||||
});
|
||||
window.initChart('os-breakdown-chart', {
|
||||
type: 'bar',
|
||||
data: { labels: data.by_os.map((r) => r.name), datasets: [{ label: 'OS', data: data.by_os.map((r) => r.count), backgroundColor: '#B8860B' }] },
|
||||
});
|
||||
});
|
||||
|
||||
function renderKpiCards(kpi) {
|
||||
const errorTone = kpi.error_rate_pct >= 5 ? 'danger' : kpi.error_rate_pct >= 1 ? 'warn' : 'ok';
|
||||
const cards = [
|
||||
['Total requests', kpi.total_requests.toLocaleString(), 'accent'],
|
||||
['Unique IPs (approx.)', kpi.unique_ips.toLocaleString(), 'accent'],
|
||||
['Bandwidth', `${(kpi.total_bandwidth_bytes / 1e6).toFixed(1)} MB`, 'accent'],
|
||||
['Avg response size', `${kpi.avg_response_size_bytes} B`, 'accent'],
|
||||
['Error rate', `${kpi.error_rate_pct}%`, errorTone],
|
||||
['Avg requests / day', kpi.avg_requests_per_day.toLocaleString(), 'accent'],
|
||||
];
|
||||
const toneBorder = {
|
||||
accent: 'border-t-accent dark:border-t-accent-dark',
|
||||
ok: 'border-t-ok dark:border-t-ok-dark',
|
||||
warn: 'border-t-warn dark:border-t-warn-dark',
|
||||
danger: 'border-t-danger dark:border-t-danger-dark',
|
||||
};
|
||||
const el = document.getElementById('kpi-cards');
|
||||
el.innerHTML = cards.map(([label, value, tone]) => `
|
||||
<div class="bg-surface dark:bg-surface-dark rounded-lg border border-line dark:border-line-dark border-t-2 ${toneBorder[tone]} p-3">
|
||||
<p class="text-xs text-muted dark:text-muted-dark uppercase tracking-wide">${label}</p>
|
||||
<p class="font-data text-2xl font-medium mt-0.5">${value}</p>
|
||||
</div>`).join('');
|
||||
if (kpi.peak_day) {
|
||||
el.innerHTML += `
|
||||
<div class="bg-surface dark:bg-surface-dark rounded-lg border border-line dark:border-line-dark border-t-2 border-t-accent dark:border-t-accent-dark p-3 col-span-2 sm:col-span-3">
|
||||
<p class="text-xs text-muted dark:text-muted-dark uppercase tracking-wide">Peak day</p>
|
||||
<p class="font-data text-xl font-medium mt-0.5">${kpi.peak_day.date} <span class="text-muted dark:text-muted-dark text-sm">— ${kpi.peak_day.count.toLocaleString()} requests</span></p>
|
||||
</div>`;
|
||||
}
|
||||
}
|
||||
|
||||
function initUploadedFilesGrid() {
|
||||
const container = document.getElementById('uploaded-files-grid');
|
||||
const deleteBtn = document.getElementById('delete-selected-btn');
|
||||
const selectedIds = new Set();
|
||||
|
||||
function updateDeleteButton() {
|
||||
deleteBtn.disabled = selectedIds.size === 0;
|
||||
deleteBtn.textContent = selectedIds.size > 0 ? `Delete selected (${selectedIds.size})` : 'Delete selected';
|
||||
}
|
||||
|
||||
// Selection is intentionally NOT restored across a Grid.js
|
||||
// re-render (page/sort/search) — simpler and always visually
|
||||
// honest, at the cost of losing selection if you page away
|
||||
// mid-selection. See the comment above the HTML for this panel.
|
||||
container.addEventListener('change', (e) => {
|
||||
if (!e.target.matches('.file-select-checkbox')) return;
|
||||
const id = parseInt(e.target.value, 10);
|
||||
if (e.target.checked) selectedIds.add(id); else selectedIds.delete(id);
|
||||
updateDeleteButton();
|
||||
});
|
||||
|
||||
const columns = [
|
||||
{
|
||||
name: '',
|
||||
formatter: (cell, row) => {
|
||||
const rawStatus = row.cells[6].data;
|
||||
const disabled = rawStatus === 'processing';
|
||||
return window.gridHtml(
|
||||
`<input type="checkbox" class="file-select-checkbox w-4 h-4 rounded border-line dark:border-line-dark text-accent focus:ring-accent cursor-pointer disabled:cursor-not-allowed disabled:opacity-40"
|
||||
value="${cell}" ${disabled ? 'disabled title="Still analyzing — wait for it to finish"' : ''}>`
|
||||
);
|
||||
},
|
||||
},
|
||||
{ name: 'Filename' },
|
||||
{ name: 'Server' },
|
||||
{ name: 'Status' },
|
||||
{ name: 'Uploaded' },
|
||||
{ name: 'Size' },
|
||||
{ name: 'Raw Status', hidden: true },
|
||||
];
|
||||
|
||||
const filesGrid = window.initGrid(
|
||||
'uploaded-files-grid',
|
||||
container.dataset.endpoint,
|
||||
columns,
|
||||
{},
|
||||
);
|
||||
|
||||
deleteBtn.addEventListener('click', () => {
|
||||
if (selectedIds.size === 0) return;
|
||||
if (!confirm(`Delete ${selectedIds.size} file(s) and all data derived from them? This can't be undone.`)) return;
|
||||
|
||||
const token = document.querySelector('meta[name="csrf-token"]')?.content;
|
||||
fetch('{{ url_for("uploads.bulk_delete_uploads") }}', {
|
||||
method: 'DELETE',
|
||||
headers: { 'Content-Type': 'application/json', 'X-CSRFToken': token || '' },
|
||||
body: JSON.stringify({ ids: Array.from(selectedIds) }),
|
||||
})
|
||||
.then((r) => r.json())
|
||||
.then(({ data }) => {
|
||||
selectedIds.clear();
|
||||
updateDeleteButton();
|
||||
filesGrid.forceRender();
|
||||
if (data.skipped && data.skipped.length) {
|
||||
alert('Some files were skipped:\n' + data.skipped.map((s) => `#${s.id} — ${s.reason}`).join('\n'));
|
||||
}
|
||||
// Deleting a file can change rollups for any date it
|
||||
// touched — refresh the whole page's KPIs/charts via the
|
||||
// same mechanism the date-range picker itself uses.
|
||||
window.htmx.trigger(document.getElementById('date-range-form'), 'change');
|
||||
});
|
||||
});
|
||||
}
|
||||
})();
|
||||
</script>
|
||||
</div>
|
||||
@@ -0,0 +1,5 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Overview — Kavosh{% endblock %}
|
||||
{% block content %}
|
||||
{% include "overview/_content.html" %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,13 @@
|
||||
from flask import Blueprint
|
||||
from flask_login import login_required
|
||||
|
||||
bp = Blueprint("security", __name__, template_folder="templates")
|
||||
|
||||
|
||||
@bp.before_request
|
||||
@login_required
|
||||
def require_login():
|
||||
pass
|
||||
|
||||
|
||||
from app.blueprints.security import routes # noqa: E402,F401 registers routes
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Context-builder query functions for the Security tab (Chapter 10),
|
||||
with the IP investigation panel's traffic breakdown upgraded to full-
|
||||
traffic data (bounded per-IP rollup, added as an explicit follow-up to
|
||||
Chapter 10's scope gap).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from datetime import date
|
||||
|
||||
from sqlalchemy import func
|
||||
|
||||
from app.extensions import db
|
||||
from app.models.blocklist_suggestion import BlocklistSuggestion
|
||||
from app.models.bot_hit import BotHit
|
||||
from app.models.ip_registry import IPRegistry
|
||||
from app.models.ip_traffic_stats import IpPathStatsDaily, IpStatusStatsDaily
|
||||
from app.models.suspicious_event import SuspiciousEvent
|
||||
from app.services.severity_scoring import SeverityInputs, compute_effective_severity
|
||||
from app.utils.dates import day_bounds
|
||||
|
||||
IP_HISTORY_EVENT_LIMIT = 50
|
||||
IP_HISTORY_PATH_LIMIT = 20
|
||||
|
||||
|
||||
def get_suspicious_events(
|
||||
from_date: date, to_date: date, severity: str | None, rule_type: str | None, page: int, per_page: int,
|
||||
) -> tuple[list[list], int]:
|
||||
"""suspicious_events is small/indexed/indefinitely-retained (Ch06) —
|
||||
same precedent as Ch09's bot_hits queries, so loading + escalating in
|
||||
Python doesn't violate Ch03 rule 6 (that targets raw per-request rows).
|
||||
"""
|
||||
start, end = day_bounds(from_date, to_date)
|
||||
query = db.session.query(SuspiciousEvent).filter(
|
||||
SuspiciousEvent.timestamp >= start, SuspiciousEvent.timestamp < end
|
||||
)
|
||||
if rule_type:
|
||||
query = query.filter(SuspiciousEvent.rule_matched.like(f"{rule_type}:%"))
|
||||
events = query.order_by(SuspiciousEvent.timestamp.desc()).all()
|
||||
|
||||
by_ip: dict[str, list[SuspiciousEvent]] = defaultdict(list)
|
||||
for e in events:
|
||||
by_ip[e.ip].append(e)
|
||||
|
||||
enriched = []
|
||||
for e in events:
|
||||
ip_events = by_ip[e.ip]
|
||||
timestamps = sorted(ev.timestamp for ev in ip_events)
|
||||
avg_interval = (
|
||||
(timestamps[-1] - timestamps[0]).total_seconds() / (len(timestamps) - 1)
|
||||
if len(timestamps) > 1 else None
|
||||
)
|
||||
effective = compute_effective_severity(
|
||||
SeverityInputs(base_severity=e.severity, ip_event_count=len(ip_events), avg_interval_seconds=avg_interval)
|
||||
)
|
||||
if severity and effective != severity:
|
||||
continue
|
||||
enriched.append([e.timestamp.isoformat(), e.ip, e.path, e.rule_matched, effective])
|
||||
|
||||
total = len(enriched)
|
||||
offset = (page - 1) * per_page
|
||||
return enriched[offset : offset + per_page], total
|
||||
|
||||
|
||||
def get_sensitive_path_summary(from_date: date, to_date: date) -> list[dict]:
|
||||
"""Grouped by request path; filtered to Ch07's sensitive_path rule
|
||||
category. One ranked list, not sub-grouped into config/admin/VCS —
|
||||
Ch07's dictionary has no such taxonomy to reuse.
|
||||
"""
|
||||
start, end = day_bounds(from_date, to_date)
|
||||
rows = (
|
||||
db.session.query(
|
||||
SuspiciousEvent.path,
|
||||
func.count().label("hit_count"),
|
||||
func.count(func.distinct(SuspiciousEvent.ip)).label("distinct_ip_count"),
|
||||
)
|
||||
.filter(
|
||||
SuspiciousEvent.timestamp >= start, SuspiciousEvent.timestamp < end,
|
||||
SuspiciousEvent.rule_matched.like("sensitive_path:%"),
|
||||
)
|
||||
.group_by(SuspiciousEvent.path)
|
||||
.order_by(func.count().desc())
|
||||
.all()
|
||||
)
|
||||
return [{"path": r.path, "hit_count": r.hit_count, "distinct_ip_count": r.distinct_ip_count} for r in rows]
|
||||
|
||||
|
||||
def get_ip_history(ip: str) -> dict | None:
|
||||
"""Pulled from ip_registry (identity + true total_requests), plus
|
||||
bot_hits/suspicious_events (flagged activity), plus the bounded
|
||||
per-IP traffic rollup (top_paths / status_code_distribution — true
|
||||
full-traffic breakdown, added as a follow-up to Ch10's original scope
|
||||
gap). Retention caveat: the per-IP rollup covers roughly the last 30
|
||||
days (see aggregator.py / flask cleanup).
|
||||
"""
|
||||
registry = db.session.get(IPRegistry, ip)
|
||||
if registry is None:
|
||||
return None
|
||||
|
||||
path_rows = (
|
||||
db.session.query(IpPathStatsDaily.path, func.sum(IpPathStatsDaily.count).label("count"))
|
||||
.filter(IpPathStatsDaily.ip == ip)
|
||||
.group_by(IpPathStatsDaily.path)
|
||||
.order_by(func.sum(IpPathStatsDaily.count).desc())
|
||||
.limit(IP_HISTORY_PATH_LIMIT)
|
||||
.all()
|
||||
)
|
||||
status_rows = (
|
||||
db.session.query(IpStatusStatsDaily.status_bucket, func.sum(IpStatusStatsDaily.count).label("count"))
|
||||
.filter(IpStatusStatsDaily.ip == ip)
|
||||
.group_by(IpStatusStatsDaily.status_bucket)
|
||||
.all()
|
||||
)
|
||||
|
||||
bot_rows = (
|
||||
db.session.query(BotHit).filter(BotHit.ip == ip)
|
||||
.order_by(BotHit.timestamp.desc()).limit(IP_HISTORY_EVENT_LIMIT).all()
|
||||
)
|
||||
suspicious_rows = (
|
||||
db.session.query(SuspiciousEvent).filter(SuspiciousEvent.ip == ip)
|
||||
.order_by(SuspiciousEvent.timestamp.desc()).limit(IP_HISTORY_EVENT_LIMIT).all()
|
||||
)
|
||||
spoofed_bot_names = sorted({b.bot_name for b in bot_rows if not b.verified})
|
||||
|
||||
return {
|
||||
"ip": ip,
|
||||
"first_seen": registry.first_seen.isoformat(),
|
||||
"last_seen": registry.last_seen.isoformat(),
|
||||
"total_requests": registry.total_requests,
|
||||
"reputation_score": registry.reputation_score,
|
||||
"is_flagged": registry.is_flagged,
|
||||
"spoofed_bot_names": spoofed_bot_names,
|
||||
"top_paths": [[r.path, r.count] for r in path_rows],
|
||||
"status_code_distribution": {r.status_bucket: r.count for r in status_rows},
|
||||
"traffic_window_note": "Path/status breakdown reflects roughly the last 30 days (bounded retention).",
|
||||
"recent_suspicious_events": [
|
||||
{"timestamp": s.timestamp.isoformat(), "path": s.path, "rule_matched": s.rule_matched, "severity": s.severity}
|
||||
for s in suspicious_rows
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def format_blocklist(suggestions: list[BlocklistSuggestion], fmt: str) -> str:
|
||||
"""Ch10: '.htaccess Deny/iptables/fail2ban-style'. 'plain' (a bare IP
|
||||
list) is the most portable interpretation of "fail2ban-style input"
|
||||
without assuming a specific fail2ban jail configuration Ch10 doesn't
|
||||
specify.
|
||||
"""
|
||||
ips = [s.ip for s in suggestions]
|
||||
if fmt == "htaccess":
|
||||
return "".join(f"Deny from {ip}\n" for ip in ips)
|
||||
if fmt == "iptables":
|
||||
return "".join(f"iptables -A INPUT -s {ip} -j DROP\n" for ip in ips)
|
||||
return "".join(f"{ip}\n" for ip in ips)
|
||||
@@ -0,0 +1,71 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from flask import Response, jsonify, render_template, request
|
||||
|
||||
from app.blueprints.security import bp
|
||||
from app.blueprints.security.queries import (
|
||||
format_blocklist, get_ip_history, get_sensitive_path_summary, get_suspicious_events,
|
||||
)
|
||||
from app.extensions import db
|
||||
from app.models.blocklist_suggestion import BlocklistSuggestion
|
||||
from app.utils.dates import parse_date_range
|
||||
from app.utils.htmx import render_htmx_aware
|
||||
from app.utils.pagination import parse_pagination
|
||||
|
||||
|
||||
@bp.route("/security")
|
||||
def security():
|
||||
from_date, to_date = parse_date_range(request)
|
||||
severity = request.args.get("severity") or ""
|
||||
rule_type = request.args.get("rule_type") or ""
|
||||
return render_htmx_aware(
|
||||
request, full_template="security/index.html", partial_template="security/_content.html",
|
||||
from_date=from_date, to_date=to_date, severity=severity, rule_type=rule_type,
|
||||
)
|
||||
|
||||
|
||||
@bp.get("/api/security/events")
|
||||
def api_security_events():
|
||||
from_date, to_date = parse_date_range(request)
|
||||
page, per_page = parse_pagination(request)
|
||||
severity = request.args.get("severity") or None
|
||||
rule_type = request.args.get("rule_type") or None
|
||||
rows, total = get_suspicious_events(from_date, to_date, severity, rule_type, page, per_page)
|
||||
return jsonify(
|
||||
data={"rows": rows, "total": total},
|
||||
meta={"from": from_date.isoformat(), "to": to_date.isoformat(), "page": page, "per_page": per_page},
|
||||
)
|
||||
|
||||
|
||||
@bp.get("/api/security/sensitive-paths")
|
||||
def api_sensitive_paths():
|
||||
from_date, to_date = parse_date_range(request)
|
||||
return jsonify(data=get_sensitive_path_summary(from_date, to_date), meta={"from": from_date.isoformat(), "to": to_date.isoformat()})
|
||||
|
||||
|
||||
@bp.get("/api/security/ip/<ip>")
|
||||
def api_ip_history(ip: str):
|
||||
history = get_ip_history(ip)
|
||||
if history is None:
|
||||
return render_template("security/_ip_not_found.html", ip=ip), 404
|
||||
return render_template("security/_ip_history.html", ip_data=history)
|
||||
|
||||
|
||||
@bp.get("/api/security/export-blocklist")
|
||||
def api_export_blocklist():
|
||||
fmt = request.args.get("format", "plain")
|
||||
include_all = request.args.get("all", "false").lower() == "true"
|
||||
query = BlocklistSuggestion.query
|
||||
if not include_all:
|
||||
query = query.filter_by(exported=False)
|
||||
suggestions = query.order_by(BlocklistSuggestion.created_at).all()
|
||||
|
||||
body = format_blocklist(suggestions, fmt)
|
||||
for s in suggestions:
|
||||
s.exported = True
|
||||
db.session.commit()
|
||||
|
||||
return Response(
|
||||
body, mimetype="text/plain",
|
||||
headers={"Content-Disposition": "attachment; filename=kavosh-blocklist.txt"},
|
||||
)
|
||||
@@ -0,0 +1,125 @@
|
||||
<div id="security-content"
|
||||
hx-get="{{ url_for('security.security') }}"
|
||||
hx-trigger="change from:#security-filter-form"
|
||||
hx-include="#security-filter-form"
|
||||
hx-target="#security-content"
|
||||
hx-swap="outerHTML"
|
||||
data-from="{{ from_date.isoformat() }}"
|
||||
data-to="{{ to_date.isoformat() }}"
|
||||
data-severity="{{ severity }}"
|
||||
data-rule-type="{{ rule_type }}">
|
||||
|
||||
<div class="flex flex-wrap items-end justify-between gap-4 mb-6">
|
||||
<div>
|
||||
<h1 class="font-display font-bold text-xl">Suspicious Requests & IP History</h1>
|
||||
<p class="text-sm text-muted dark:text-muted-dark">Who's poking at your site, and how hard</p>
|
||||
</div>
|
||||
<form id="security-filter-form" class="flex flex-wrap gap-3 items-end">
|
||||
<label class="text-sm text-muted dark:text-muted-dark">From
|
||||
<input type="date" name="from" value="{{ from_date.isoformat() }}" class="block border border-line dark:border-line-dark rounded-md px-2 py-1 mt-1 bg-surface dark:bg-surface-dark text-ink dark:text-ink-dark font-data text-sm">
|
||||
</label>
|
||||
<label class="text-sm text-muted dark:text-muted-dark">To
|
||||
<input type="date" name="to" value="{{ to_date.isoformat() }}" class="block border border-line dark:border-line-dark rounded-md px-2 py-1 mt-1 bg-surface dark:bg-surface-dark text-ink dark:text-ink-dark font-data text-sm">
|
||||
</label>
|
||||
<label class="text-sm text-muted dark:text-muted-dark">Severity
|
||||
<select name="severity" class="block border border-line dark:border-line-dark rounded-md px-2 py-1 mt-1 bg-surface dark:bg-surface-dark text-ink dark:text-ink-dark text-sm">
|
||||
<option value="" {{ 'selected' if not severity }}>All</option>
|
||||
<option value="low" {{ 'selected' if severity == 'low' }}>Low</option>
|
||||
<option value="medium" {{ 'selected' if severity == 'medium' }}>Medium</option>
|
||||
<option value="high" {{ 'selected' if severity == 'high' }}>High</option>
|
||||
</select>
|
||||
</label>
|
||||
<label class="text-sm text-muted dark:text-muted-dark">Rule Type
|
||||
<select name="rule_type" class="block border border-line dark:border-line-dark rounded-md px-2 py-1 mt-1 bg-surface dark:bg-surface-dark text-ink dark:text-ink-dark text-sm">
|
||||
<option value="" {{ 'selected' if not rule_type }}>All</option>
|
||||
<option value="sensitive_path" {{ 'selected' if rule_type == 'sensitive_path' }}>Sensitive Path</option>
|
||||
<option value="injection" {{ 'selected' if rule_type == 'injection' }}>Injection</option>
|
||||
<option value="scanner_ua" {{ 'selected' if rule_type == 'scanner_ua' }}>Scanner UA</option>
|
||||
<option value="spoofed_bot" {{ 'selected' if rule_type == 'spoofed_bot' }}>Spoofed Bot</option>
|
||||
</select>
|
||||
</label>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div class="mb-6">
|
||||
<h3 class="text-xs font-medium uppercase tracking-wide text-muted dark:text-muted-dark mb-2">Suspicious events</h3>
|
||||
<div id="suspicious-events-grid" data-endpoint="{{ url_for('security.api_security_events') }}"></div>
|
||||
</div>
|
||||
|
||||
<div class="mb-6">
|
||||
<h3 class="text-xs font-medium uppercase tracking-wide text-muted dark:text-muted-dark mb-2">Sensitive-path probes</h3>
|
||||
<div id="sensitive-paths-panel" class="border border-line dark:border-line-dark rounded-xl bg-surface dark:bg-surface-dark overflow-hidden"
|
||||
data-endpoint="{{ url_for('security.api_sensitive_paths') }}"></div>
|
||||
</div>
|
||||
|
||||
<div class="mb-6 border border-line dark:border-line-dark rounded-xl p-4 bg-surface dark:bg-surface-dark">
|
||||
<h3 class="font-display font-semibold text-sm mb-3">Export blocklist</h3>
|
||||
<div class="flex flex-wrap gap-2 items-center">
|
||||
<a href="{{ url_for('security.api_export_blocklist') }}"
|
||||
class="bg-accent dark:bg-accent-dark text-white dark:text-paper-dark rounded-md px-3 py-1.5 text-sm font-medium hover:opacity-90 transition-opacity">Download new (.txt)</a>
|
||||
<a href="{{ url_for('security.api_export_blocklist', all='true') }}"
|
||||
class="border border-line dark:border-line-dark rounded-md px-3 py-1.5 text-sm hover:bg-paper dark:hover:bg-paper-dark transition-colors">Re-export all</a>
|
||||
<select id="blocklist-format" class="border border-line dark:border-line-dark rounded-md px-2 py-1.5 text-sm bg-surface dark:bg-surface-dark text-ink dark:text-ink-dark" onchange="updateBlocklistLinks(this.value)">
|
||||
<option value="plain">Plain IP list</option>
|
||||
<option value="htaccess">.htaccess Deny</option>
|
||||
<option value="iptables">iptables</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="ip-history-modal" class="fixed inset-0 bg-ink/40 dark:bg-ink-dark/60 items-center justify-center empty:hidden flex z-50"></div>
|
||||
|
||||
<script>
|
||||
(function initSecurityWidgets() {
|
||||
const root = document.getElementById('security-content');
|
||||
const from = root.dataset.from, to = root.dataset.to;
|
||||
const severity = root.dataset.severity, ruleType = root.dataset.ruleType;
|
||||
|
||||
window.initGrid(
|
||||
'suspicious-events-grid',
|
||||
document.getElementById('suspicious-events-grid').dataset.endpoint,
|
||||
[
|
||||
{ name: 'Timestamp' },
|
||||
{
|
||||
name: 'IP',
|
||||
formatter: (cell) => window.gridHtml(
|
||||
`<button class="text-accent dark:text-accent-dark underline" hx-get="/api/security/ip/${cell}" hx-target="#ip-history-modal" hx-swap="innerHTML">${cell}</button>`
|
||||
),
|
||||
},
|
||||
{ name: 'Path' }, { name: 'Rule Matched' },
|
||||
{
|
||||
name: 'Severity',
|
||||
formatter: (cell) => {
|
||||
const tone = { low: 'text-muted dark:text-muted-dark', medium: 'text-warn dark:text-warn-dark', high: 'text-danger dark:text-danger-dark' }[cell] || '';
|
||||
return window.gridHtml(`<span class="font-medium ${tone}">${cell}</span>`);
|
||||
},
|
||||
},
|
||||
],
|
||||
{ from, to, severity, rule_type: ruleType },
|
||||
);
|
||||
|
||||
const pathsEl = document.getElementById('sensitive-paths-panel');
|
||||
fetch(`${pathsEl.dataset.endpoint}?from=${from}&to=${to}`)
|
||||
.then((r) => r.json())
|
||||
.then(({ data }) => {
|
||||
pathsEl.innerHTML = data.length
|
||||
? `<table class="w-full text-sm font-data">
|
||||
<thead><tr class="text-left text-muted dark:text-muted-dark text-xs uppercase tracking-wide bg-surface-raised dark:bg-surface-raised-dark font-sans">
|
||||
<th class="px-3 py-2">Path</th><th class="px-3 py-2">Hits</th><th class="px-3 py-2">Distinct IPs</th>
|
||||
</tr></thead>
|
||||
<tbody class="divide-y divide-line dark:divide-line-dark">${
|
||||
data.map((r) => `<tr><td class="px-3 py-2">${r.path}</td><td class="px-3 py-2">${r.hit_count}</td><td class="px-3 py-2">${r.distinct_ip_count}</td></tr>`).join('')
|
||||
}</tbody></table>`
|
||||
: `<p class="text-sm text-muted dark:text-muted-dark p-4">No sensitive-path probes in range.</p>`;
|
||||
});
|
||||
|
||||
window.updateBlocklistLinks = (fmt) => {
|
||||
document.querySelectorAll('a[href*="export-blocklist"]').forEach((a) => {
|
||||
const url = new URL(a.href, window.location.origin);
|
||||
url.searchParams.set('format', fmt);
|
||||
a.href = url.toString();
|
||||
});
|
||||
};
|
||||
})();
|
||||
</script>
|
||||
</div>
|
||||
@@ -0,0 +1,26 @@
|
||||
<div class="bg-surface dark:bg-surface-dark rounded-xl p-6 max-w-lg w-full relative border border-line dark:border-line-dark">
|
||||
<button class="absolute top-3 right-3 text-muted dark:text-muted-dark hover:text-ink dark:hover:text-ink-dark" onclick="document.getElementById('ip-history-modal').innerHTML=''">
|
||||
<svg class="w-4 h-4"><use href="/static/dist/icons.svg#x"/></svg>
|
||||
</button>
|
||||
<h3 class="font-display font-semibold text-lg mb-3 font-data">{{ ip_data.ip }}</h3>
|
||||
<dl class="text-sm grid grid-cols-2 gap-y-1.5 mb-4 font-data">
|
||||
<dt class="text-muted dark:text-muted-dark font-sans">First seen</dt><dd>{{ ip_data.first_seen }}</dd>
|
||||
<dt class="text-muted dark:text-muted-dark font-sans">Last seen</dt><dd>{{ ip_data.last_seen }}</dd>
|
||||
<dt class="text-muted dark:text-muted-dark font-sans">Total requests</dt><dd>{{ ip_data.total_requests }}</dd>
|
||||
<dt class="text-muted dark:text-muted-dark font-sans">Reputation score</dt><dd>{{ ip_data.reputation_score }}</dd>
|
||||
<dt class="text-muted dark:text-muted-dark font-sans">Flagged</dt>
|
||||
<dd class="{{ 'text-danger dark:text-danger-dark font-medium' if ip_data.is_flagged else '' }}">{{ 'Yes' if ip_data.is_flagged else 'No' }}</dd>
|
||||
{% if ip_data.spoofed_bot_names %}
|
||||
<dt class="text-muted dark:text-muted-dark font-sans">Spoofed bot claims</dt><dd class="text-danger dark:text-danger-dark">{{ ip_data.spoofed_bot_names | join(', ') }}</dd>
|
||||
{% endif %}
|
||||
</dl>
|
||||
<p class="text-xs text-muted dark:text-muted-dark mb-3">{{ ip_data.traffic_window_note }}</p>
|
||||
<h4 class="font-medium text-sm mb-1">Top paths</h4>
|
||||
<ul class="text-sm mb-3 font-data text-ink dark:text-ink-dark space-y-0.5">{% for path, count in ip_data.top_paths %}<li>{{ path }} <span class="text-muted dark:text-muted-dark">— {{ count }}</span></li>{% endfor %}</ul>
|
||||
<h4 class="font-medium text-sm mb-1">Status codes</h4>
|
||||
<ul class="text-sm mb-3 font-data text-ink dark:text-ink-dark space-y-0.5">{% for code, count in ip_data.status_code_distribution.items() %}<li>{{ code }} <span class="text-muted dark:text-muted-dark">— {{ count }}</span></li>{% endfor %}</ul>
|
||||
{% if ip_data.recent_suspicious_events %}
|
||||
<h4 class="font-medium text-sm mb-1">Recent flagged events</h4>
|
||||
<ul class="text-sm font-data text-ink dark:text-ink-dark space-y-0.5">{% for e in ip_data.recent_suspicious_events %}<li>{{ e.timestamp }} — {{ e.path }} <span class="text-muted dark:text-muted-dark">({{ e.rule_matched }}, {{ e.severity }})</span></li>{% endfor %}</ul>
|
||||
{% endif %}
|
||||
</div>
|
||||
@@ -0,0 +1,4 @@
|
||||
<div class="bg-surface dark:bg-surface-dark rounded-xl p-6 max-w-sm w-full border border-line dark:border-line-dark">
|
||||
<p class="text-sm text-ink dark:text-ink-dark">No history found for <span class="font-data">{{ ip }}</span> — it hasn't been seen yet.</p>
|
||||
<button onclick="document.getElementById('ip-history-modal').innerHTML=''" class="mt-3 text-sm text-accent dark:text-accent-dark hover:underline">Close</button>
|
||||
</div>
|
||||
@@ -0,0 +1,5 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}Security — Kavosh{% endblock %}
|
||||
{% block content %}
|
||||
{% include "security/_content.html" %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,13 @@
|
||||
from flask import Blueprint
|
||||
from flask_login import login_required
|
||||
|
||||
bp = Blueprint("seo", __name__, template_folder="templates")
|
||||
|
||||
|
||||
@bp.before_request
|
||||
@login_required
|
||||
def require_login():
|
||||
pass
|
||||
|
||||
|
||||
from app.blueprints.seo import routes # noqa: E402,F401 registers routes
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Context-builder query functions for the SEO tab (Chapter 09).
|
||||
|
||||
Per Ch09's own Output instruction, bot-related widgets query bot_hits
|
||||
directly (small, indefinitely-retained, indexed — Ch06) rather than a new
|
||||
rollup; only the human-side of the crawled-vs-visited comparison needs
|
||||
the new human_path_stats_daily table.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from datetime import date
|
||||
|
||||
from sqlalchemy import case, func
|
||||
|
||||
from app.extensions import db
|
||||
from app.models.bot_hit import BotHit
|
||||
from app.models.human_path_stats import HumanPathStatsDaily
|
||||
from app.utils.dates import day_bounds
|
||||
|
||||
# ASSUMPTION (flagged): Ch09 doesn't define "major crawler" for the
|
||||
# crawl-frequency chart's series cap.
|
||||
TOP_N_BOTS_FOR_CHART = 6
|
||||
|
||||
|
||||
def get_bot_summary(from_date: date, to_date: date) -> list[dict]:
|
||||
start, end = day_bounds(from_date, to_date)
|
||||
rows = (
|
||||
db.session.query(
|
||||
BotHit.bot_name,
|
||||
func.count().label("hits"),
|
||||
func.sum(case((BotHit.verified.is_(True), 1), else_=0)).label("verified_hits"),
|
||||
func.max(BotHit.timestamp).label("last_seen"),
|
||||
)
|
||||
.filter(BotHit.timestamp >= start, BotHit.timestamp < end)
|
||||
.group_by(BotHit.bot_name)
|
||||
.order_by(func.count().desc())
|
||||
.all()
|
||||
)
|
||||
return [
|
||||
{
|
||||
"bot_name": r.bot_name,
|
||||
"hits": r.hits,
|
||||
"verified_pct": round(r.verified_hits / r.hits * 100, 1) if r.hits else 0.0,
|
||||
"last_seen": r.last_seen.isoformat() if r.last_seen else None,
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
def get_crawl_chart_data(from_date: date, to_date: date, bot_name: str | None) -> dict:
|
||||
start, end = day_bounds(from_date, to_date)
|
||||
base_filters = [BotHit.timestamp >= start, BotHit.timestamp < end]
|
||||
|
||||
if bot_name:
|
||||
allowed_bots = [bot_name]
|
||||
else:
|
||||
top_rows = (
|
||||
db.session.query(BotHit.bot_name, func.count().label("hits"))
|
||||
.filter(*base_filters)
|
||||
.group_by(BotHit.bot_name)
|
||||
.order_by(func.count().desc())
|
||||
.limit(TOP_N_BOTS_FOR_CHART)
|
||||
.all()
|
||||
)
|
||||
allowed_bots = [r.bot_name for r in top_rows]
|
||||
|
||||
if not allowed_bots:
|
||||
return {"series": []}
|
||||
|
||||
rows = (
|
||||
db.session.query(func.date(BotHit.timestamp).label("day"), BotHit.bot_name, func.count().label("count"))
|
||||
.filter(*base_filters, BotHit.bot_name.in_(allowed_bots))
|
||||
.group_by("day", BotHit.bot_name)
|
||||
.order_by("day")
|
||||
.all()
|
||||
)
|
||||
points_by_bot: dict[str, list[dict]] = defaultdict(list)
|
||||
for r in rows:
|
||||
points_by_bot[r.bot_name].append({"t": r.day, "count": r.count})
|
||||
|
||||
return {"series": [{"bot_name": b, "points": points_by_bot.get(b, [])} for b in allowed_bots]}
|
||||
|
||||
|
||||
def get_bot_status_codes(from_date: date, to_date: date) -> dict:
|
||||
start, end = day_bounds(from_date, to_date)
|
||||
bucket = case(
|
||||
(BotHit.status_code < 300, "2xx"),
|
||||
(BotHit.status_code < 400, "3xx"),
|
||||
(BotHit.status_code < 500, "4xx"),
|
||||
else_="5xx",
|
||||
)
|
||||
rows = (
|
||||
db.session.query(bucket.label("bucket"), func.count().label("count"))
|
||||
.filter(BotHit.timestamp >= start, BotHit.timestamp < end)
|
||||
.group_by("bucket")
|
||||
.all()
|
||||
)
|
||||
breakdown = {"2xx": 0, "3xx": 0, "4xx": 0, "5xx": 0}
|
||||
for r in rows:
|
||||
breakdown[r.bucket] = r.count
|
||||
|
||||
# Ch09 calls out 404 by name specifically, not just the 4xx bucket.
|
||||
not_found_404 = (
|
||||
db.session.query(func.count())
|
||||
.filter(BotHit.timestamp >= start, BotHit.timestamp < end, BotHit.status_code == 404)
|
||||
.scalar()
|
||||
)
|
||||
return {"breakdown": breakdown, "not_found_404": not_found_404 or 0}
|
||||
|
||||
|
||||
def get_crawled_vs_visited(from_date: date, to_date: date, page: int, per_page: int) -> tuple[list[list], int]:
|
||||
"""Diffed table: one row per path, bot hits vs. human hits."""
|
||||
start, end = day_bounds(from_date, to_date)
|
||||
bot_rows = (
|
||||
db.session.query(BotHit.path, func.count().label("hits"))
|
||||
.filter(BotHit.timestamp >= start, BotHit.timestamp < end)
|
||||
.group_by(BotHit.path)
|
||||
.all()
|
||||
)
|
||||
human_rows = (
|
||||
db.session.query(HumanPathStatsDaily.path, func.sum(HumanPathStatsDaily.count).label("hits"))
|
||||
.filter(HumanPathStatsDaily.date >= from_date, HumanPathStatsDaily.date <= to_date)
|
||||
.group_by(HumanPathStatsDaily.path)
|
||||
.all()
|
||||
)
|
||||
bot_counts = {r.path: r.hits for r in bot_rows}
|
||||
human_counts = {r.path: r.hits for r in human_rows}
|
||||
|
||||
combined = []
|
||||
for path in set(bot_counts) | set(human_counts):
|
||||
b, h = bot_counts.get(path, 0), human_counts.get(path, 0)
|
||||
total = b + h
|
||||
combined.append([path, b, h, round(b / total * 100, 1) if total else 0.0])
|
||||
|
||||
# ASSUMPTION (flagged): sorted by bot hits desc — Ch09 doesn't specify.
|
||||
combined.sort(key=lambda row: row[1], reverse=True)
|
||||
|
||||
total_count = len(combined)
|
||||
offset = (page - 1) * per_page
|
||||
return combined[offset : offset + per_page], total_count
|
||||
@@ -0,0 +1,56 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from flask import jsonify, request
|
||||
|
||||
from app.blueprints.seo import bp
|
||||
from app.blueprints.seo.queries import (
|
||||
get_bot_status_codes,
|
||||
get_bot_summary,
|
||||
get_crawl_chart_data,
|
||||
get_crawled_vs_visited,
|
||||
)
|
||||
from app.utils.dates import parse_date_range
|
||||
from app.utils.htmx import render_htmx_aware
|
||||
from app.utils.pagination import parse_pagination
|
||||
|
||||
|
||||
@bp.route("/seo")
|
||||
def seo():
|
||||
from_date, to_date = parse_date_range(request)
|
||||
return render_htmx_aware(
|
||||
request, full_template="seo/index.html", partial_template="seo/_content.html",
|
||||
from_date=from_date, to_date=to_date,
|
||||
)
|
||||
|
||||
|
||||
@bp.get("/api/seo/bot-summary")
|
||||
def api_bot_summary():
|
||||
from_date, to_date = parse_date_range(request)
|
||||
return jsonify(data=get_bot_summary(from_date, to_date), meta={"from": from_date.isoformat(), "to": to_date.isoformat()})
|
||||
|
||||
|
||||
@bp.get("/api/seo/crawl-chart-data")
|
||||
def api_crawl_chart_data():
|
||||
from_date, to_date = parse_date_range(request)
|
||||
bot_name = request.args.get("bot")
|
||||
return jsonify(
|
||||
data=get_crawl_chart_data(from_date, to_date, bot_name),
|
||||
meta={"from": from_date.isoformat(), "to": to_date.isoformat(), "bot": bot_name},
|
||||
)
|
||||
|
||||
|
||||
@bp.get("/api/seo/bot-status-codes")
|
||||
def api_bot_status_codes():
|
||||
from_date, to_date = parse_date_range(request)
|
||||
return jsonify(data=get_bot_status_codes(from_date, to_date), meta={"from": from_date.isoformat(), "to": to_date.isoformat()})
|
||||
|
||||
|
||||
@bp.get("/api/seo/crawled-vs-visited")
|
||||
def api_crawled_vs_visited():
|
||||
from_date, to_date = parse_date_range(request)
|
||||
page, per_page = parse_pagination(request)
|
||||
rows, total = get_crawled_vs_visited(from_date, to_date, page, per_page)
|
||||
return jsonify(
|
||||
data={"rows": rows, "total": total},
|
||||
meta={"from": from_date.isoformat(), "to": to_date.isoformat(), "page": page, "per_page": per_page},
|
||||
)
|
||||
@@ -0,0 +1,102 @@
|
||||
<div id="seo-content"
|
||||
hx-get="{{ url_for('seo.seo') }}"
|
||||
hx-trigger="change from:#seo-date-range-form"
|
||||
hx-include="#seo-date-range-form"
|
||||
hx-target="#seo-content"
|
||||
hx-swap="outerHTML"
|
||||
data-from="{{ from_date.isoformat() }}"
|
||||
data-to="{{ to_date.isoformat() }}">
|
||||
|
||||
<div class="flex flex-wrap items-end justify-between gap-4 mb-6">
|
||||
<div>
|
||||
<h1 class="font-display font-bold text-xl">SEO & Bot Behavior</h1>
|
||||
<p class="text-sm text-muted dark:text-muted-dark">How search engines are crawling your site</p>
|
||||
</div>
|
||||
<form id="seo-date-range-form" class="flex gap-3 items-end">
|
||||
<label class="text-sm text-muted dark:text-muted-dark">From
|
||||
<input type="date" name="from" value="{{ from_date.isoformat() }}"
|
||||
class="block border border-line dark:border-line-dark rounded-md px-2 py-1 mt-1 bg-surface dark:bg-surface-dark text-ink dark:text-ink-dark font-data text-sm">
|
||||
</label>
|
||||
<label class="text-sm text-muted dark:text-muted-dark">To
|
||||
<input type="date" name="to" value="{{ to_date.isoformat() }}"
|
||||
class="block border border-line dark:border-line-dark rounded-md px-2 py-1 mt-1 bg-surface dark:bg-surface-dark text-ink dark:text-ink-dark font-data text-sm">
|
||||
</label>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div id="bot-summary-cards" class="grid grid-cols-2 sm:grid-cols-3 gap-3 mb-6"
|
||||
data-endpoint="{{ url_for('seo.api_bot_summary') }}"></div>
|
||||
|
||||
<div class="grid grid-cols-1 lg:grid-cols-2 gap-4 mb-6">
|
||||
<div class="border border-line dark:border-line-dark rounded-xl p-4 bg-surface dark:bg-surface-dark h-72">
|
||||
<h3 class="text-xs font-medium uppercase tracking-wide text-muted dark:text-muted-dark mb-2">Crawl frequency</h3>
|
||||
<div class="h-56"><canvas id="crawl-frequency-chart" data-endpoint="{{ url_for('seo.api_crawl_chart_data') }}"></canvas></div>
|
||||
</div>
|
||||
<div class="border border-line dark:border-line-dark rounded-xl p-4 bg-surface dark:bg-surface-dark h-72">
|
||||
<h3 class="text-xs font-medium uppercase tracking-wide text-muted dark:text-muted-dark mb-2">Status codes served to bots</h3>
|
||||
<div class="h-44"><canvas id="bot-status-codes-chart" data-endpoint="{{ url_for('seo.api_bot_status_codes') }}"></canvas></div>
|
||||
<p id="bot-404-note" class="text-xs text-muted dark:text-muted-dark mt-2"></p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<h3 class="text-xs font-medium uppercase tracking-wide text-muted dark:text-muted-dark mb-2">Most-crawled vs. most-visited URLs</h3>
|
||||
<div id="crawled-vs-visited-grid" data-endpoint="{{ url_for('seo.api_crawled_vs_visited') }}"></div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
(function initSeoWidgets() {
|
||||
const root = document.getElementById('seo-content');
|
||||
const from = root.dataset.from, to = root.dataset.to;
|
||||
const withRange = (url) => `${url}?from=${from}&to=${to}`;
|
||||
|
||||
fetch(withRange(document.getElementById('bot-summary-cards').dataset.endpoint))
|
||||
.then((r) => r.json())
|
||||
.then(({ data }) => {
|
||||
document.getElementById('bot-summary-cards').innerHTML = data.length ? data.map((bot) => `
|
||||
<div class="bg-surface dark:bg-surface-dark rounded-lg border border-line dark:border-line-dark border-t-2 ${bot.verified_pct < 100 ? 'border-t-warn dark:border-t-warn-dark' : 'border-t-ok dark:border-t-ok-dark'} p-3">
|
||||
<p class="text-xs text-muted dark:text-muted-dark uppercase tracking-wide">${bot.bot_name}</p>
|
||||
<p class="font-data text-xl font-medium mt-0.5">${bot.hits.toLocaleString()} <span class="text-sm text-muted dark:text-muted-dark font-sans">hits</span></p>
|
||||
<p class="text-xs mt-1 ${bot.verified_pct < 100 ? 'text-warn dark:text-warn-dark' : 'text-muted dark:text-muted-dark'}">
|
||||
${bot.verified_pct}% verified${bot.verified_pct < 100 ? ' — some spoofed' : ''}
|
||||
</p>
|
||||
<p class="text-xs text-muted dark:text-muted-dark font-data mt-0.5">Last seen: ${bot.last_seen ?? '—'}</p>
|
||||
</div>`).join('') : `<p class="text-sm text-muted dark:text-muted-dark col-span-full">No bot activity in this range.</p>`;
|
||||
});
|
||||
|
||||
const crawlEl = document.getElementById('crawl-frequency-chart');
|
||||
fetch(withRange(crawlEl.dataset.endpoint))
|
||||
.then((r) => r.json())
|
||||
.then(({ data }) => window.initChart('crawl-frequency-chart', {
|
||||
type: 'line',
|
||||
data: {
|
||||
labels: [...new Set(data.series.flatMap((s) => s.points.map((p) => p.t)))].sort(),
|
||||
datasets: data.series.map((s, i) => ({
|
||||
label: s.bot_name,
|
||||
data: s.points.map((p) => ({ x: p.t, y: p.count })),
|
||||
tension: 0.3,
|
||||
borderColor: window.KAVOSH_CHART_PALETTE[i % window.KAVOSH_CHART_PALETTE.length],
|
||||
})),
|
||||
},
|
||||
}));
|
||||
|
||||
const statusEl = document.getElementById('bot-status-codes-chart');
|
||||
fetch(withRange(statusEl.dataset.endpoint))
|
||||
.then((r) => r.json())
|
||||
.then(({ data }) => {
|
||||
window.initChart('bot-status-codes-chart', {
|
||||
type: 'bar',
|
||||
data: { labels: Object.keys(data.breakdown), datasets: [{ label: 'Bot Requests', data: Object.values(data.breakdown), backgroundColor: '#0E7C86' }] },
|
||||
});
|
||||
document.getElementById('bot-404-note').textContent = `${data.not_found_404} 404s served to bots in range — wasted crawl budget.`;
|
||||
});
|
||||
|
||||
window.initGrid(
|
||||
'crawled-vs-visited-grid',
|
||||
document.getElementById('crawled-vs-visited-grid').dataset.endpoint,
|
||||
[{ name: 'Path' }, { name: 'Bot Hits' }, { name: 'Human Hits' }, { name: 'Bot Share %' }],
|
||||
{ from, to },
|
||||
);
|
||||
})();
|
||||
</script>
|
||||
</div>
|
||||
@@ -0,0 +1,5 @@
|
||||
{% extends "base.html" %}
|
||||
{% block title %}SEO & Bots — Kavosh{% endblock %}
|
||||
{% block content %}
|
||||
{% include "seo/_content.html" %}
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,15 @@
|
||||
from flask import Blueprint
|
||||
from flask_login import login_required
|
||||
|
||||
bp = Blueprint("uploads", __name__, template_folder="templates")
|
||||
|
||||
|
||||
@bp.before_request
|
||||
@login_required
|
||||
def require_login():
|
||||
"""Ch01: dashboard reachable from one AUTHENTICATED shell; Ch12:
|
||||
single-admin login. All routes on this blueprint require a session."""
|
||||
pass
|
||||
|
||||
|
||||
from app.blueprints.uploads import routes # noqa: E402,F401 registers routes
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Query/formatting helpers for the "Uploaded files" list (project-owner
|
||||
follow-up request). Kept separate from routes.py to match this project's
|
||||
established per-blueprint queries.py convention (Ch08/09/10).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from app.models.log_file import LogFile
|
||||
|
||||
|
||||
def get_uploaded_files(page: int, per_page: int) -> tuple[list[list], int]:
|
||||
"""Newest first, independent of the dashboard date-range picker —
|
||||
this lists uploads by when they arrived, not by which log dates they
|
||||
contain (a single file can span many dates).
|
||||
"""
|
||||
query = LogFile.query.order_by(LogFile.uploaded_at.desc())
|
||||
total = query.count()
|
||||
rows = query.offset((page - 1) * per_page).limit(per_page).all()
|
||||
|
||||
result = []
|
||||
for lf in rows:
|
||||
result.append([
|
||||
lf.id,
|
||||
lf.filename,
|
||||
lf.server_type,
|
||||
_status_display(lf),
|
||||
lf.uploaded_at.strftime("%Y-%m-%d %H:%M"),
|
||||
_human_size(lf.size_bytes),
|
||||
lf.status, # raw status (hidden column) — lets the client disable
|
||||
# the select checkbox for files still "processing"
|
||||
])
|
||||
return result, total
|
||||
|
||||
|
||||
def _status_display(lf: LogFile) -> str:
|
||||
if lf.status == "processing":
|
||||
if lf.total_lines:
|
||||
pct = round(lf.processed_lines / lf.total_lines * 100)
|
||||
return f"processing ({pct}%)"
|
||||
return "processing"
|
||||
if lf.status == "error":
|
||||
return f"error: {lf.error_message}" if lf.error_message else "error"
|
||||
return lf.status
|
||||
|
||||
|
||||
def _human_size(num_bytes: int) -> str:
|
||||
size = float(num_bytes)
|
||||
for unit in ("B", "KB", "MB", "GB"):
|
||||
if size < 1024 or unit == "GB":
|
||||
return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}"
|
||||
size /= 1024
|
||||
return f"{size:.1f} GB"
|
||||
@@ -0,0 +1,188 @@
|
||||
"""Upload endpoint (Chapter 04): validated, streamed-to-disk save.
|
||||
|
||||
Parsing happens in a background thread triggered right after this request
|
||||
completes (app/services/background.py) — never synchronously inside this
|
||||
request (Chapter 03, rule 5 still holds: the response returns immediately
|
||||
regardless of file size).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from flask import current_app, jsonify, render_template, request
|
||||
from werkzeug.utils import secure_filename
|
||||
|
||||
from app.blueprints.uploads import bp
|
||||
from app.blueprints.uploads.queries import get_uploaded_files
|
||||
from app.extensions import db, limiter
|
||||
from app.models.log_file import LogFile
|
||||
from app.services.background import trigger_processing
|
||||
from app.services.file_deletion import delete_log_files
|
||||
from app.utils.pagination import parse_pagination
|
||||
from app.utils.upload_paths import upload_path_for
|
||||
|
||||
_ALLOWED_EXTENSIONS = {".log", ".txt", ".gz"}
|
||||
_CHUNK_SIZE = 64 * 1024 # 64 KB per read — never buffer the whole upload
|
||||
_GZIP_MAGIC = b"\x1f\x8b"
|
||||
|
||||
|
||||
class UploadRejected(Exception):
|
||||
"""Raised when an upload fails extension/content validation."""
|
||||
|
||||
|
||||
def _validate_extension(filename: str) -> str:
|
||||
ext = Path(filename).suffix.lower()
|
||||
if ext not in _ALLOWED_EXTENSIONS:
|
||||
raise UploadRejected(f"Unsupported extension {ext!r}; allowed: {_ALLOWED_EXTENSIONS}")
|
||||
return ext
|
||||
|
||||
|
||||
def _sniff_content(first_chunk: bytes, ext: str) -> None:
|
||||
"""Light content sniff — don't just trust the client-supplied MIME type."""
|
||||
if ext == ".gz":
|
||||
if not first_chunk.startswith(_GZIP_MAGIC):
|
||||
raise UploadRejected("File has a .gz extension but isn't gzip-magic-prefixed.")
|
||||
return
|
||||
if b"\x00" in first_chunk:
|
||||
raise UploadRejected("File extension claims text but content looks binary.")
|
||||
|
||||
|
||||
def _stream_to_temp(file_storage, tmp_path: Path) -> tuple[int, str, bytes]:
|
||||
"""Stream the upload to disk in bounded chunks; return (size, sha256_hex, first_chunk).
|
||||
|
||||
Never calls file.read() on the whole stream (Chapter 03, rule 1).
|
||||
"""
|
||||
sha256 = hashlib.sha256()
|
||||
size = 0
|
||||
first_chunk: bytes | None = None
|
||||
with tmp_path.open("wb") as out:
|
||||
while True:
|
||||
chunk = file_storage.stream.read(_CHUNK_SIZE)
|
||||
if not chunk:
|
||||
break
|
||||
if first_chunk is None:
|
||||
first_chunk = chunk
|
||||
sha256.update(chunk)
|
||||
size += len(chunk)
|
||||
out.write(chunk)
|
||||
if first_chunk is None:
|
||||
raise UploadRejected("Uploaded file is empty.")
|
||||
return size, sha256.hexdigest(), first_chunk
|
||||
|
||||
|
||||
@bp.post("/uploads")
|
||||
@limiter.limit("20 per minute") # Chapter 12: rate limiting on /uploads at minimum
|
||||
def upload_log_file():
|
||||
"""Validate, stream, and register an uploaded access log (Ch04/Ch11)."""
|
||||
file_storage = request.files.get("logfile")
|
||||
if file_storage is None or not file_storage.filename:
|
||||
return render_template("uploads/_error.html", message="No file provided."), 400
|
||||
|
||||
upload_dir = Path(current_app.config["UPLOAD_DIR"])
|
||||
(upload_dir / "tmp").mkdir(parents=True, exist_ok=True)
|
||||
tmp_path = upload_dir / "tmp" / f"{uuid.uuid4().hex}.part"
|
||||
|
||||
try:
|
||||
ext = _validate_extension(file_storage.filename)
|
||||
size_bytes, checksum, first_chunk = _stream_to_temp(file_storage, tmp_path)
|
||||
_sniff_content(first_chunk, ext)
|
||||
|
||||
max_bytes = current_app.config["UPLOAD_MAX_SIZE_MB"] * 1024 * 1024
|
||||
if size_bytes > max_bytes:
|
||||
raise UploadRejected(f"File exceeds {current_app.config['UPLOAD_MAX_SIZE_MB']}MB limit.")
|
||||
except UploadRejected as exc:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
return render_template("uploads/_error.html", message=str(exc)), 400
|
||||
|
||||
existing = LogFile.query.filter_by(checksum=checksum).first()
|
||||
if existing is not None:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
return render_template("uploads/_duplicate.html", log_file=existing)
|
||||
|
||||
log_file = LogFile(
|
||||
filename=secure_filename(file_storage.filename),
|
||||
server_type=request.form.get("server_type", "apache"),
|
||||
format_string=request.form.get("format_string", ""),
|
||||
status="queued",
|
||||
size_bytes=size_bytes,
|
||||
checksum=checksum,
|
||||
)
|
||||
db.session.add(log_file)
|
||||
db.session.commit() # need the assigned id before the final rename
|
||||
|
||||
tmp_path.rename(upload_path_for(log_file))
|
||||
|
||||
# Chapter 12 simplification: no cron required — kick off processing
|
||||
# immediately in a background thread. The response below returns as
|
||||
# soon as the file is queued (Ch03 rule 5 still holds: this request
|
||||
# never blocks on parsing), while the thread runs independently.
|
||||
trigger_processing(current_app._get_current_object(), log_file.id)
|
||||
|
||||
return render_template("uploads/_queued.html", log_file=log_file)
|
||||
|
||||
|
||||
@bp.get("/api/uploads/<int:log_file_id>/status")
|
||||
def upload_status(log_file_id: int):
|
||||
"""Polled every 3s by the browser (hx-trigger) until done/error (Ch04)."""
|
||||
log_file = db.get_or_404(LogFile, log_file_id)
|
||||
|
||||
if request.headers.get("Accept") == "application/json":
|
||||
return jsonify(
|
||||
data={
|
||||
"id": log_file.id,
|
||||
"status": log_file.status,
|
||||
"processed_lines": log_file.processed_lines,
|
||||
"total_lines": log_file.total_lines,
|
||||
},
|
||||
meta={},
|
||||
)
|
||||
|
||||
template = {
|
||||
"done": "uploads/_status_done.html",
|
||||
"error": "uploads/_status_error.html",
|
||||
# BUG FIX: this key was missing, so "queued" fell through to the
|
||||
# "processing" fallback below — a file that hadn't been picked up
|
||||
# by `flask process-logs` yet displayed as "Processing X: 0 lines"
|
||||
# instead of "Queued — waiting for the next parse cycle", making a
|
||||
# cron job that simply hasn't run yet indistinguishable from one
|
||||
# that's actually hung mid-parse.
|
||||
"queued": "uploads/_queued.html",
|
||||
}.get(log_file.status, "uploads/_status_processing.html")
|
||||
return render_template(template, log_file=log_file)
|
||||
|
||||
|
||||
@bp.get("/api/uploads")
|
||||
def list_uploads():
|
||||
"""Uploaded-files list (project-owner follow-up request) — Grid.js-
|
||||
backed, same page/per_page convention as every other table (Ch11).
|
||||
Sorted by upload recency, independent of the dashboard date-range
|
||||
picker (a single file can span many log dates).
|
||||
"""
|
||||
page, per_page = parse_pagination(request)
|
||||
rows, total = get_uploaded_files(page, per_page)
|
||||
return jsonify(
|
||||
data={"rows": rows, "total": total},
|
||||
meta={"page": page, "per_page": per_page},
|
||||
)
|
||||
|
||||
|
||||
@bp.delete("/api/uploads")
|
||||
def bulk_delete_uploads():
|
||||
"""Delete one or more uploaded files and everything derived from them
|
||||
(log_entries/bot_hits/suspicious_events, the raw file on disk, and a
|
||||
correct rollup recompute for the affected dates — see
|
||||
app/services/file_deletion.py for why a rollup recompute is needed
|
||||
rather than a simple per-file delete).
|
||||
|
||||
Body: {"ids": [1, 2, 3]}. Files currently "processing" are skipped,
|
||||
not force-deleted, to avoid racing the background parse thread.
|
||||
"""
|
||||
body = request.get_json(silent=True) or {}
|
||||
ids = body.get("ids")
|
||||
if not isinstance(ids, list) or not ids or not all(isinstance(i, int) for i in ids):
|
||||
return jsonify(error={"code": "invalid_request", "message": "Expected {\"ids\": [int, ...]}."}), 400
|
||||
|
||||
result = delete_log_files(ids)
|
||||
return jsonify(data={"deleted": result.deleted, "skipped": result.skipped}, meta={})
|
||||
@@ -0,0 +1,4 @@
|
||||
<div class="border border-warn/30 dark:border-warn-dark/30 bg-warn/5 dark:bg-warn-dark/10 rounded-lg p-3 text-sm">
|
||||
<span class="font-data">{{ log_file.filename }}</span>
|
||||
<span class="text-muted dark:text-muted-dark">matches an already-uploaded file (status: {{ log_file.status }}); skipped re-upload.</span>
|
||||
</div>
|
||||
@@ -0,0 +1,3 @@
|
||||
<div class="border border-danger/30 dark:border-danger-dark/30 bg-danger/5 dark:bg-danger-dark/10 rounded-lg p-3 text-sm text-danger dark:text-danger-dark">
|
||||
{{ message }}
|
||||
</div>
|
||||
@@ -0,0 +1,10 @@
|
||||
<div id="upload-status-{{ log_file.id }}"
|
||||
hx-get="{{ url_for('uploads.upload_status', log_file_id=log_file.id) }}"
|
||||
hx-trigger="every 1s" hx-swap="outerHTML"
|
||||
class="border border-line dark:border-line-dark rounded-lg p-3 bg-paper dark:bg-paper-dark">
|
||||
<div class="flex items-center gap-2 text-sm">
|
||||
<svg class="w-4 h-4 text-muted dark:text-muted-dark animate-spin"><use href="/static/dist/icons.svg#loader-circle"/></svg>
|
||||
<span class="font-data">{{ log_file.filename }}</span>
|
||||
<span class="text-muted dark:text-muted-dark">— queued, starting shortly…</span>
|
||||
</div>
|
||||
</div>
|
||||
@@ -0,0 +1,7 @@
|
||||
<div id="upload-status-{{ log_file.id }}" class="border border-ok/30 dark:border-ok-dark/30 bg-ok/5 dark:bg-ok-dark/10 rounded-lg p-3">
|
||||
<div class="flex items-center gap-2 text-sm">
|
||||
<span class="w-2 h-2 rounded-full bg-ok dark:bg-ok-dark shrink-0"></span>
|
||||
<span class="font-data">{{ log_file.filename }}</span>
|
||||
<span class="text-ok dark:text-ok-dark">— done, {{ "{:,}".format(log_file.processed_lines) }} lines analyzed</span>
|
||||
</div>
|
||||
</div>
|
||||
@@ -0,0 +1,7 @@
|
||||
<div id="upload-status-{{ log_file.id }}" class="border border-danger/30 dark:border-danger-dark/30 bg-danger/5 dark:bg-danger-dark/10 rounded-lg p-3">
|
||||
<div class="flex items-center gap-2 text-sm">
|
||||
<span class="w-2 h-2 rounded-full bg-danger dark:bg-danger-dark shrink-0"></span>
|
||||
<span class="font-data">{{ log_file.filename }}</span>
|
||||
</div>
|
||||
<p class="text-danger dark:text-danger-dark text-xs mt-1">{{ log_file.error_message }}</p>
|
||||
</div>
|
||||
@@ -0,0 +1,19 @@
|
||||
{% set pct = ((log_file.processed_lines / log_file.total_lines) * 100) if log_file.total_lines else None %}
|
||||
<div id="upload-status-{{ log_file.id }}"
|
||||
hx-get="{{ url_for('uploads.upload_status', log_file_id=log_file.id) }}"
|
||||
hx-trigger="every 1s" hx-swap="outerHTML"
|
||||
class="border border-line dark:border-line-dark rounded-lg p-3 bg-paper dark:bg-paper-dark">
|
||||
<div class="flex items-center justify-between text-sm mb-2">
|
||||
<span class="font-data">{{ log_file.filename }}</span>
|
||||
<span class="font-data text-muted dark:text-muted-dark">
|
||||
{% if pct is not none %}{{ pct | round(0) | int }}%{% else %}analyzing…{% endif %}
|
||||
</span>
|
||||
</div>
|
||||
<div class="w-full h-2 rounded-full bg-line dark:bg-line-dark overflow-hidden">
|
||||
<div class="h-full rounded-full bg-accent dark:bg-accent-dark transition-all duration-500 ease-out"
|
||||
style="width: {{ pct | round(1) if pct is not none else 8 }}%"></div>
|
||||
</div>
|
||||
<p class="text-xs text-muted dark:text-muted-dark mt-1.5 font-data">
|
||||
{{ "{:,}".format(log_file.processed_lines) }}{% if log_file.total_lines %} / {{ "{:,}".format(log_file.total_lines) }}{% endif %} lines
|
||||
</p>
|
||||
</div>
|
||||
Reference in New Issue
Block a user