127 lines
5.9 KiB
Python
127 lines
5.9 KiB
Python
"""Initial schema (Chapter 06): log_files, log_entries, request_stats_hourly/
|
|
daily, bot_hits, ip_registry, suspicious_events, blocklist_suggestions.
|
|
"""
|
|
from alembic import op
|
|
import sqlalchemy as sa
|
|
|
|
revision = "0001_initial_schema"
|
|
down_revision = None
|
|
branch_labels = None
|
|
depends_on = None
|
|
|
|
|
|
def upgrade():
|
|
op.create_table(
|
|
"log_files",
|
|
sa.Column("id", sa.Integer, primary_key=True),
|
|
sa.Column("filename", sa.String(255), nullable=False),
|
|
sa.Column("server_type", sa.String(16), nullable=False),
|
|
sa.Column("format_string", sa.Text, nullable=False),
|
|
sa.Column("uploaded_at", sa.DateTime, nullable=False),
|
|
sa.Column("status", sa.String(16), nullable=False, server_default="queued"),
|
|
sa.Column("total_lines", sa.Integer, nullable=True),
|
|
sa.Column("processed_lines", sa.Integer, nullable=False, server_default="0"),
|
|
sa.Column("size_bytes", sa.BigInteger, nullable=False),
|
|
sa.Column("checksum", sa.String(64), nullable=False),
|
|
sa.Column("error_message", sa.Text, nullable=True),
|
|
)
|
|
op.create_index("ix_log_files_checksum", "log_files", ["checksum"])
|
|
|
|
op.create_table(
|
|
"log_entries",
|
|
sa.Column("id", sa.Integer, primary_key=True),
|
|
sa.Column("log_file_id", sa.Integer, sa.ForeignKey("log_files.id", ondelete="CASCADE"), nullable=False),
|
|
sa.Column("timestamp", sa.DateTime, nullable=False),
|
|
sa.Column("ip", sa.String(45), nullable=False),
|
|
sa.Column("method", sa.String(10), nullable=False),
|
|
sa.Column("path", sa.Text, nullable=False),
|
|
sa.Column("status_code", sa.SmallInteger, nullable=False),
|
|
sa.Column("bytes_sent", sa.Integer, nullable=False, server_default="0"),
|
|
sa.Column("referrer", sa.Text, nullable=True),
|
|
sa.Column("user_agent", sa.Text, nullable=True),
|
|
sa.Column("is_bot", sa.Boolean, nullable=False, server_default=sa.false()),
|
|
sa.Column("flagged", sa.Boolean, nullable=False, server_default=sa.false()),
|
|
)
|
|
op.create_index("ix_log_entries_file_ts", "log_entries", ["log_file_id", "timestamp"])
|
|
op.create_index("ix_log_entries_ip", "log_entries", ["ip"])
|
|
op.create_index("ix_log_entries_path", "log_entries", ["path"])
|
|
op.create_index("ix_log_entries_timestamp", "log_entries", ["timestamp"])
|
|
|
|
op.create_table(
|
|
"request_stats_hourly",
|
|
sa.Column("date_hour", sa.DateTime, primary_key=True),
|
|
sa.Column("path", sa.String(2048), primary_key=True),
|
|
sa.Column("status_code", sa.SmallInteger, primary_key=True),
|
|
sa.Column("count", sa.Integer, nullable=False, server_default="0"),
|
|
sa.Column("bytes_sent_sum", sa.BigInteger, nullable=False, server_default="0"),
|
|
)
|
|
|
|
op.create_table(
|
|
"request_stats_daily",
|
|
sa.Column("date", sa.Date, primary_key=True),
|
|
sa.Column("count", sa.Integer, nullable=False, server_default="0"),
|
|
sa.Column("unique_ips", sa.Integer, nullable=False, server_default="0"),
|
|
sa.Column("bytes_sum", sa.BigInteger, nullable=False, server_default="0"),
|
|
sa.Column("error_count", sa.Integer, nullable=False, server_default="0"),
|
|
)
|
|
|
|
op.create_table(
|
|
"bot_hits",
|
|
sa.Column("id", sa.Integer, primary_key=True),
|
|
sa.Column("log_file_id", sa.Integer, sa.ForeignKey("log_files.id", ondelete="CASCADE"), nullable=False),
|
|
sa.Column("timestamp", sa.DateTime, nullable=False),
|
|
sa.Column("ip", sa.String(45), nullable=False),
|
|
sa.Column("bot_name", sa.String(64), nullable=False),
|
|
sa.Column("verified", sa.Boolean, nullable=False, server_default=sa.false()),
|
|
sa.Column("path", sa.Text, nullable=False),
|
|
sa.Column("status_code", sa.SmallInteger, nullable=False),
|
|
)
|
|
op.create_index("ix_bot_hits_timestamp", "bot_hits", ["timestamp"])
|
|
op.create_index("ix_bot_hits_bot_name", "bot_hits", ["bot_name"])
|
|
|
|
op.create_table(
|
|
"ip_registry",
|
|
sa.Column("ip", sa.String(45), primary_key=True),
|
|
sa.Column("first_seen", sa.DateTime, nullable=False),
|
|
sa.Column("last_seen", sa.DateTime, nullable=False),
|
|
sa.Column("total_requests", sa.Integer, nullable=False, server_default="0"),
|
|
sa.Column("reputation_score", sa.Integer, nullable=False, server_default="0"),
|
|
sa.Column("is_flagged", sa.Boolean, nullable=False, server_default=sa.false()),
|
|
sa.Column("last_verified_at", sa.DateTime, nullable=True),
|
|
)
|
|
|
|
op.create_table(
|
|
"suspicious_events",
|
|
sa.Column("id", sa.Integer, primary_key=True),
|
|
sa.Column("log_file_id", sa.Integer, sa.ForeignKey("log_files.id", ondelete="CASCADE"), nullable=False),
|
|
sa.Column("ip", sa.String(45), nullable=False),
|
|
sa.Column("timestamp", sa.DateTime, nullable=False),
|
|
sa.Column("path", sa.Text, nullable=False),
|
|
sa.Column("rule_matched", sa.String(128), nullable=False),
|
|
sa.Column("severity", sa.String(8), nullable=False),
|
|
)
|
|
op.create_index("ix_suspicious_events_timestamp", "suspicious_events", ["timestamp"])
|
|
op.create_index("ix_suspicious_events_ip", "suspicious_events", ["ip"])
|
|
op.create_index("ix_suspicious_events_severity", "suspicious_events", ["severity"])
|
|
|
|
op.create_table(
|
|
"blocklist_suggestions",
|
|
sa.Column("id", sa.Integer, primary_key=True),
|
|
sa.Column("ip", sa.String(45), nullable=False),
|
|
sa.Column("reason", sa.String(255), nullable=False),
|
|
sa.Column("created_at", sa.DateTime, nullable=False),
|
|
sa.Column("exported", sa.Boolean, nullable=False, server_default=sa.false()),
|
|
)
|
|
op.create_index("ix_blocklist_suggestions_ip", "blocklist_suggestions", ["ip"])
|
|
|
|
|
|
def downgrade():
|
|
op.drop_table("blocklist_suggestions")
|
|
op.drop_table("suspicious_events")
|
|
op.drop_table("ip_registry")
|
|
op.drop_table("bot_hits")
|
|
op.drop_table("request_stats_daily")
|
|
op.drop_table("request_stats_hourly")
|
|
op.drop_table("log_entries")
|
|
op.drop_table("log_files")
|