start project

2026-08-07 21:17:17 +03:30
commit ea1e1eead6
121 changed files with 8108 additions and 0 deletions
@@ -0,0 +1,19 @@
+from app.models.blocklist_suggestion import BlocklistSuggestion
+from app.models.bot_hit import BotHit
+from app.models.browser_stats import BrowserStatsDaily
+from app.models.human_path_stats import HumanPathStatsDaily
+from app.models.ip_registry import IPRegistry
+from app.models.ip_traffic_stats import IpPathStatsDaily, IpStatusStatsDaily
+from app.models.log_entry import LogEntry
+from app.models.log_file import LogFile
+from app.models.referrer_stats import ReferrerStatsDaily
+from app.models.request_stats import RequestStatsDaily, RequestStatsHourly
+from app.models.suspicious_event import SuspiciousEvent
+from app.models.user import User
+
+__all__ = [
+    "LogFile", "LogEntry", "RequestStatsHourly", "RequestStatsDaily",
+    "BotHit", "IPRegistry", "SuspiciousEvent", "BlocklistSuggestion",
+    "ReferrerStatsDaily", "BrowserStatsDaily", "HumanPathStatsDaily",
+    "IpPathStatsDaily", "IpStatusStatsDaily", "User",
+]
@@ -0,0 +1,18 @@
+"""blocklist_suggestions table (Chapter 06). `id` isn't in Ch06's column
+list — added because `ip` alone can't be the key (the same IP may be
+re-flagged with a different reason later)."""
+from __future__ import annotations
+
+from datetime import datetime
+
+from app.extensions import db
+
+
+class BlocklistSuggestion(db.Model):
+    __tablename__ = "blocklist_suggestions"
+
+    id: int = db.Column(db.Integer, primary_key=True)
+    ip: str = db.Column(db.String(45), nullable=False, index=True)
+    reason: str = db.Column(db.String(255), nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)
+    exported: bool = db.Column(db.Boolean, nullable=False, default=False)
@@ -0,0 +1,22 @@
+"""bot_hits table (Chapter 06) — SEO section (Ch09)."""
+from __future__ import annotations
+
+from app.extensions import db
+
+
+class BotHit(db.Model):
+    __tablename__ = "bot_hits"
+
+    id: int = db.Column(db.Integer, primary_key=True)
+    log_file_id: int = db.Column(db.Integer, db.ForeignKey("log_files.id", ondelete="CASCADE"), nullable=False)
+    timestamp = db.Column(db.DateTime, nullable=False)
+    ip: str = db.Column(db.String(45), nullable=False)
+    bot_name: str = db.Column(db.String(64), nullable=False)
+    verified: bool = db.Column(db.Boolean, nullable=False, default=False)
+    path: str = db.Column(db.Text, nullable=False)
+    status_code: int = db.Column(db.SmallInteger, nullable=False)
+
+    __table_args__ = (
+        db.Index("ix_bot_hits_timestamp", "timestamp"),
+        db.Index("ix_bot_hits_bot_name", "bot_name"),
+    )
@@ -0,0 +1,14 @@
+"""NEW table (Method A, Ch08 follow-up) — not in Chapter 06's original
+list. Human-only browser/OS breakdown widget (Ch08)."""
+from __future__ import annotations
+
+from app.extensions import db
+
+
+class BrowserStatsDaily(db.Model):
+    __tablename__ = "browser_stats_daily"
+
+    date = db.Column(db.Date, primary_key=True)
+    browser: str = db.Column(db.String(64), primary_key=True)
+    os: str = db.Column(db.String(64), primary_key=True)
+    count: int = db.Column(db.Integer, nullable=False, default=0)
@@ -0,0 +1,15 @@
+"""NEW table (Method A, Ch09 follow-up) — not in Chapter 06's original
+list. request_stats_hourly/_daily aggregate ALL traffic with no is_bot
+split, so Ch09's "most-visited-by-humans" comparison had no rollup to
+read. Bot hits excluded at rollup-write time (Ch07's is_bot flag)."""
+from __future__ import annotations
+
+from app.extensions import db
+
+
+class HumanPathStatsDaily(db.Model):
+    __tablename__ = "human_path_stats_daily"
+
+    date = db.Column(db.Date, primary_key=True)
+    path: str = db.Column(db.String(2048), primary_key=True)
+    count: int = db.Column(db.Integer, nullable=False, default=0)
@@ -0,0 +1,22 @@
+"""ip_registry table (Chapter 06 + Chapter 07 extension).
+
+last_verified_bot_result: NOT in Ch06's literal column list — added in
+Chapter 07. last_verified_at alone can't tell a cache hit *what* was
+verified, only *when*; this stores the outcome.
+"""
+from __future__ import annotations
+
+from app.extensions import db
+
+
+class IPRegistry(db.Model):
+    __tablename__ = "ip_registry"
+
+    ip: str = db.Column(db.String(45), primary_key=True)
+    first_seen = db.Column(db.DateTime, nullable=False)
+    last_seen = db.Column(db.DateTime, nullable=False)
+    total_requests: int = db.Column(db.Integer, nullable=False, default=0)
+    reputation_score: int = db.Column(db.Integer, nullable=False, default=0)
+    is_flagged: bool = db.Column(db.Boolean, nullable=False, default=False)
+    last_verified_at = db.Column(db.DateTime, nullable=True)
+    last_verified_bot_result: bool | None = db.Column(db.Boolean, nullable=True)
@@ -0,0 +1,35 @@
+"""NEW tables (explicit follow-up to Ch10's flagged scope gap): bounded
+per-IP traffic breakdown so the IP investigation panel reflects TRUE
+full traffic, not just bot_hits/suspicious_events activity.
+
+CARDINALITY NOTE: unlike the other Method-A tables, this one's row count
+scales with distinct IPs per day, which is unbounded for a probed site.
+Two mitigations: ip_path_stats_daily keeps only the top N paths per IP
+per day (not every ip x path pair); both tables use log_entries' ~30-day
+retention (Ch06), enforced by `flask cleanup` (Chapter 12).
+"""
+from __future__ import annotations
+
+from app.extensions import db
+
+
+class IpPathStatsDaily(db.Model):
+    __tablename__ = "ip_path_stats_daily"
+
+    date = db.Column(db.Date, primary_key=True)
+    ip: str = db.Column(db.String(45), primary_key=True)
+    path: str = db.Column(db.String(2048), primary_key=True)
+    count: int = db.Column(db.Integer, nullable=False, default=0)
+
+    __table_args__ = (db.Index("ix_ip_path_stats_ip", "ip"),)
+
+
+class IpStatusStatsDaily(db.Model):
+    __tablename__ = "ip_status_stats_daily"
+
+    date = db.Column(db.Date, primary_key=True)
+    ip: str = db.Column(db.String(45), primary_key=True)
+    status_bucket: str = db.Column(db.String(8), primary_key=True)  # 2xx|3xx|4xx|5xx
+    count: int = db.Column(db.Integer, nullable=False, default=0)
+
+    __table_args__ = (db.Index("ix_ip_status_stats_ip", "ip"),)
@@ -0,0 +1,29 @@
+"""Optional, time-boxed raw storage (Chapter 06) — retention (~30 days)
+enforced by `flask cleanup` (Chapter 12)."""
+from __future__ import annotations
+
+from app.extensions import db
+
+
+class LogEntry(db.Model):
+    __tablename__ = "log_entries"
+
+    id: int = db.Column(db.Integer, primary_key=True)
+    log_file_id: int = db.Column(db.Integer, db.ForeignKey("log_files.id", ondelete="CASCADE"), nullable=False)
+    timestamp = db.Column(db.DateTime, nullable=False)  # normalized to UTC (Ch07)
+    ip: str = db.Column(db.String(45), nullable=False)  # IPv4 or IPv6
+    method: str = db.Column(db.String(10), nullable=False)
+    path: str = db.Column(db.Text, nullable=False)
+    status_code: int = db.Column(db.SmallInteger, nullable=False)
+    bytes_sent: int = db.Column(db.Integer, nullable=False, default=0)
+    referrer: str | None = db.Column(db.Text, nullable=True)
+    user_agent: str | None = db.Column(db.Text, nullable=True)
+    is_bot: bool = db.Column(db.Boolean, nullable=False, default=False)
+    flagged: bool = db.Column(db.Boolean, nullable=False, default=False)
+
+    __table_args__ = (
+        db.Index("ix_log_entries_file_ts", "log_file_id", "timestamp"),
+        db.Index("ix_log_entries_ip", "ip"),
+        db.Index("ix_log_entries_path", "path"),
+        db.Index("ix_log_entries_timestamp", "timestamp"),
+    )
@@ -0,0 +1,22 @@
+"""LogFile model (Chapter 06)."""
+from __future__ import annotations
+
+from datetime import datetime
+
+from app.extensions import db
+
+
+class LogFile(db.Model):
+    __tablename__ = "log_files"
+
+    id: int = db.Column(db.Integer, primary_key=True)
+    filename: str = db.Column(db.String(255), nullable=False)
+    server_type: str = db.Column(db.String(16), nullable=False)  # apache|litespeed
+    format_string: str = db.Column(db.Text, nullable=False)
+    uploaded_at: datetime = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)
+    status: str = db.Column(db.String(16), nullable=False, default="queued")
+    total_lines: int | None = db.Column(db.Integer, nullable=True)
+    processed_lines: int = db.Column(db.Integer, nullable=False, default=0)
+    size_bytes: int = db.Column(db.BigInteger, nullable=False)
+    checksum: str = db.Column(db.String(64), nullable=False, index=True)  # sha256 hex
+    error_message: str | None = db.Column(db.Text, nullable=True)
@@ -0,0 +1,14 @@
+"""NEW table (Method A, Ch08 follow-up) — not in Chapter 06's original
+list. Gives Top Referrers a rollup data source instead of scanning
+log_entries live. Domain-bucketed to keep cardinality bounded."""
+from __future__ import annotations
+
+from app.extensions import db
+
+
+class ReferrerStatsDaily(db.Model):
+    __tablename__ = "referrer_stats_daily"
+
+    date = db.Column(db.Date, primary_key=True)
+    referrer_domain: str = db.Column(db.String(255), primary_key=True)
+    count: int = db.Column(db.Integer, nullable=False, default=0)
@@ -0,0 +1,26 @@
+"""Rollup tables (Chapter 06) — primary source for Overview widgets (Ch08).
+Both are site-wide (no log_file_id), consistent with Ch01's single-site scope.
+"""
+from __future__ import annotations
+
+from app.extensions import db
+
+
+class RequestStatsHourly(db.Model):
+    __tablename__ = "request_stats_hourly"
+
+    date_hour = db.Column(db.DateTime, primary_key=True)
+    path: str = db.Column(db.String(2048), primary_key=True)
+    status_code: int = db.Column(db.SmallInteger, primary_key=True)
+    count: int = db.Column(db.Integer, nullable=False, default=0)
+    bytes_sent_sum: int = db.Column(db.BigInteger, nullable=False, default=0)
+
+
+class RequestStatsDaily(db.Model):
+    __tablename__ = "request_stats_daily"
+
+    date = db.Column(db.Date, primary_key=True)
+    count: int = db.Column(db.Integer, nullable=False, default=0)
+    unique_ips: int = db.Column(db.Integer, nullable=False, default=0)
+    bytes_sum: int = db.Column(db.BigInteger, nullable=False, default=0)
+    error_count: int = db.Column(db.Integer, nullable=False, default=0)
@@ -0,0 +1,22 @@
+"""suspicious_events table (Chapter 06) — Security section (Ch10)."""
+from __future__ import annotations
+
+from app.extensions import db
+
+
+class SuspiciousEvent(db.Model):
+    __tablename__ = "suspicious_events"
+
+    id: int = db.Column(db.Integer, primary_key=True)
+    log_file_id: int = db.Column(db.Integer, db.ForeignKey("log_files.id", ondelete="CASCADE"), nullable=False)
+    ip: str = db.Column(db.String(45), nullable=False)
+    timestamp = db.Column(db.DateTime, nullable=False)
+    path: str = db.Column(db.Text, nullable=False)
+    rule_matched: str = db.Column(db.String(128), nullable=False)
+    severity: str = db.Column(db.String(8), nullable=False)  # low|medium|high (provisional; Ch10 escalates)
+
+    __table_args__ = (
+        db.Index("ix_suspicious_events_timestamp", "timestamp"),
+        db.Index("ix_suspicious_events_ip", "ip"),
+        db.Index("ix_suspicious_events_severity", "severity"),
+    )
@@ -0,0 +1,27 @@
+"""Single-admin user model (Chapter 12). Not in Chapter 06's table list —
+that chapter scopes log-analytics tables; auth is a separate concern this
+chapter owns. Chapter 01 confirms single-admin, no self-registration.
+"""
+from __future__ import annotations
+
+from datetime import datetime
+
+import bcrypt
+from flask_login import UserMixin
+
+from app.extensions import db
+
+
+class User(db.Model, UserMixin):
+    __tablename__ = "users"
+
+    id: int = db.Column(db.Integer, primary_key=True)
+    email: str = db.Column(db.String(255), unique=True, nullable=False, index=True)
+    password_hash: str = db.Column(db.String(255), nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow)
+
+    def set_password(self, raw_password: str) -> None:
+        self.password_hash = bcrypt.hashpw(raw_password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8")
+
+    def check_password(self, raw_password: str) -> bool:
+        return bcrypt.checkpw(raw_password.encode("utf-8"), self.password_hash.encode("utf-8"))