start project

2026-08-07 21:17:17 +03:30
commit ea1e1eead6
121 changed files with 8108 additions and 0 deletions
@@ -0,0 +1,126 @@
+"""Initial schema (Chapter 06): log_files, log_entries, request_stats_hourly/
+daily, bot_hits, ip_registry, suspicious_events, blocklist_suggestions.
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0001_initial_schema"
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "log_files",
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column("filename", sa.String(255), nullable=False),
+        sa.Column("server_type", sa.String(16), nullable=False),
+        sa.Column("format_string", sa.Text, nullable=False),
+        sa.Column("uploaded_at", sa.DateTime, nullable=False),
+        sa.Column("status", sa.String(16), nullable=False, server_default="queued"),
+        sa.Column("total_lines", sa.Integer, nullable=True),
+        sa.Column("processed_lines", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("size_bytes", sa.BigInteger, nullable=False),
+        sa.Column("checksum", sa.String(64), nullable=False),
+        sa.Column("error_message", sa.Text, nullable=True),
+    )
+    op.create_index("ix_log_files_checksum", "log_files", ["checksum"])
+
+    op.create_table(
+        "log_entries",
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column("log_file_id", sa.Integer, sa.ForeignKey("log_files.id", ondelete="CASCADE"), nullable=False),
+        sa.Column("timestamp", sa.DateTime, nullable=False),
+        sa.Column("ip", sa.String(45), nullable=False),
+        sa.Column("method", sa.String(10), nullable=False),
+        sa.Column("path", sa.Text, nullable=False),
+        sa.Column("status_code", sa.SmallInteger, nullable=False),
+        sa.Column("bytes_sent", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("referrer", sa.Text, nullable=True),
+        sa.Column("user_agent", sa.Text, nullable=True),
+        sa.Column("is_bot", sa.Boolean, nullable=False, server_default=sa.false()),
+        sa.Column("flagged", sa.Boolean, nullable=False, server_default=sa.false()),
+    )
+    op.create_index("ix_log_entries_file_ts", "log_entries", ["log_file_id", "timestamp"])
+    op.create_index("ix_log_entries_ip", "log_entries", ["ip"])
+    op.create_index("ix_log_entries_path", "log_entries", ["path"])
+    op.create_index("ix_log_entries_timestamp", "log_entries", ["timestamp"])
+
+    op.create_table(
+        "request_stats_hourly",
+        sa.Column("date_hour", sa.DateTime, primary_key=True),
+        sa.Column("path", sa.String(2048), primary_key=True),
+        sa.Column("status_code", sa.SmallInteger, primary_key=True),
+        sa.Column("count", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("bytes_sent_sum", sa.BigInteger, nullable=False, server_default="0"),
+    )
+
+    op.create_table(
+        "request_stats_daily",
+        sa.Column("date", sa.Date, primary_key=True),
+        sa.Column("count", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("unique_ips", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("bytes_sum", sa.BigInteger, nullable=False, server_default="0"),
+        sa.Column("error_count", sa.Integer, nullable=False, server_default="0"),
+    )
+
+    op.create_table(
+        "bot_hits",
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column("log_file_id", sa.Integer, sa.ForeignKey("log_files.id", ondelete="CASCADE"), nullable=False),
+        sa.Column("timestamp", sa.DateTime, nullable=False),
+        sa.Column("ip", sa.String(45), nullable=False),
+        sa.Column("bot_name", sa.String(64), nullable=False),
+        sa.Column("verified", sa.Boolean, nullable=False, server_default=sa.false()),
+        sa.Column("path", sa.Text, nullable=False),
+        sa.Column("status_code", sa.SmallInteger, nullable=False),
+    )
+    op.create_index("ix_bot_hits_timestamp", "bot_hits", ["timestamp"])
+    op.create_index("ix_bot_hits_bot_name", "bot_hits", ["bot_name"])
+
+    op.create_table(
+        "ip_registry",
+        sa.Column("ip", sa.String(45), primary_key=True),
+        sa.Column("first_seen", sa.DateTime, nullable=False),
+        sa.Column("last_seen", sa.DateTime, nullable=False),
+        sa.Column("total_requests", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("reputation_score", sa.Integer, nullable=False, server_default="0"),
+        sa.Column("is_flagged", sa.Boolean, nullable=False, server_default=sa.false()),
+        sa.Column("last_verified_at", sa.DateTime, nullable=True),
+    )
+
+    op.create_table(
+        "suspicious_events",
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column("log_file_id", sa.Integer, sa.ForeignKey("log_files.id", ondelete="CASCADE"), nullable=False),
+        sa.Column("ip", sa.String(45), nullable=False),
+        sa.Column("timestamp", sa.DateTime, nullable=False),
+        sa.Column("path", sa.Text, nullable=False),
+        sa.Column("rule_matched", sa.String(128), nullable=False),
+        sa.Column("severity", sa.String(8), nullable=False),
+    )
+    op.create_index("ix_suspicious_events_timestamp", "suspicious_events", ["timestamp"])
+    op.create_index("ix_suspicious_events_ip", "suspicious_events", ["ip"])
+    op.create_index("ix_suspicious_events_severity", "suspicious_events", ["severity"])
+
+    op.create_table(
+        "blocklist_suggestions",
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column("ip", sa.String(45), nullable=False),
+        sa.Column("reason", sa.String(255), nullable=False),
+        sa.Column("created_at", sa.DateTime, nullable=False),
+        sa.Column("exported", sa.Boolean, nullable=False, server_default=sa.false()),
+    )
+    op.create_index("ix_blocklist_suggestions_ip", "blocklist_suggestions", ["ip"])
+
+
+def downgrade():
+    op.drop_table("blocklist_suggestions")
+    op.drop_table("suspicious_events")
+    op.drop_table("ip_registry")
+    op.drop_table("bot_hits")
+    op.drop_table("request_stats_daily")
+    op.drop_table("request_stats_hourly")
+    op.drop_table("log_entries")
+    op.drop_table("log_files")
@@ -0,0 +1,16 @@
+"""Add ip_registry.last_verified_bot_result — flagged Ch07 schema deviation."""
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0002_ip_registry_verification_result"
+down_revision = "0001_initial_schema"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("ip_registry", sa.Column("last_verified_bot_result", sa.Boolean, nullable=True))
+
+
+def downgrade():
+    op.drop_column("ip_registry", "last_verified_bot_result")
@@ -0,0 +1,29 @@
+"""Add referrer_stats_daily and browser_stats_daily (Method A, Ch08)."""
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0003_referrer_and_browser_stats"
+down_revision = "0002_ip_registry_verification_result"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "referrer_stats_daily",
+        sa.Column("date", sa.Date, primary_key=True),
+        sa.Column("referrer_domain", sa.String(255), primary_key=True),
+        sa.Column("count", sa.Integer, nullable=False, server_default="0"),
+    )
+    op.create_table(
+        "browser_stats_daily",
+        sa.Column("date", sa.Date, primary_key=True),
+        sa.Column("browser", sa.String(64), primary_key=True),
+        sa.Column("os", sa.String(64), primary_key=True),
+        sa.Column("count", sa.Integer, nullable=False, server_default="0"),
+    )
+
+
+def downgrade():
+    op.drop_table("browser_stats_daily")
+    op.drop_table("referrer_stats_daily")
@@ -0,0 +1,21 @@
+"""Add human_path_stats_daily (Method A, Ch09's most-crawled-vs-visited)."""
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0004_human_path_stats"
+down_revision = "0003_referrer_and_browser_stats"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "human_path_stats_daily",
+        sa.Column("date", sa.Date, primary_key=True),
+        sa.Column("path", sa.String(2048), primary_key=True),
+        sa.Column("count", sa.Integer, nullable=False, server_default="0"),
+    )
+
+
+def downgrade():
+    op.drop_table("human_path_stats_daily")
@@ -0,0 +1,34 @@
+"""Add ip_path_stats_daily and ip_status_stats_daily (bounded per-IP
+traffic rollup, explicit follow-up to Chapter 10)."""
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0005_ip_traffic_stats"
+down_revision = "0004_human_path_stats"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "ip_path_stats_daily",
+        sa.Column("date", sa.Date, primary_key=True),
+        sa.Column("ip", sa.String(45), primary_key=True),
+        sa.Column("path", sa.String(2048), primary_key=True),
+        sa.Column("count", sa.Integer, nullable=False, server_default="0"),
+    )
+    op.create_index("ix_ip_path_stats_ip", "ip_path_stats_daily", ["ip"])
+
+    op.create_table(
+        "ip_status_stats_daily",
+        sa.Column("date", sa.Date, primary_key=True),
+        sa.Column("ip", sa.String(45), primary_key=True),
+        sa.Column("status_bucket", sa.String(8), primary_key=True),
+        sa.Column("count", sa.Integer, nullable=False, server_default="0"),
+    )
+    op.create_index("ix_ip_status_stats_ip", "ip_status_stats_daily", ["ip"])
+
+
+def downgrade():
+    op.drop_table("ip_status_stats_daily")
+    op.drop_table("ip_path_stats_daily")
@@ -0,0 +1,23 @@
+"""Add users table (Chapter 12 single-admin auth)."""
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0006_users"
+down_revision = "0005_ip_traffic_stats"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "users",
+        sa.Column("id", sa.Integer, primary_key=True),
+        sa.Column("email", sa.String(255), nullable=False),
+        sa.Column("password_hash", sa.String(255), nullable=False),
+        sa.Column("created_at", sa.DateTime, nullable=False),
+    )
+    op.create_index("ix_users_email", "users", ["email"], unique=True)
+
+
+def downgrade():
+    op.drop_table("users")