start project

app/services/log_parser/__init__.py

@@ -0,0 +1,47 @@
+"""Log parser abstraction (Chapter 07)."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Protocol, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from app.models.log_file import LogFile
+
+
+@dataclass
+class ParsedEntry:
+    timestamp: datetime
+    ip: str
+    method: str
+    path: str
+    status_code: int
+    bytes_sent: int
+    referrer: str | None
+    user_agent: str | None
+
+
+class LogParser(Protocol):
+    def parse_line(self, line: str) -> ParsedEntry | None: ...
+
+
+def get_parser_for(log_file: "LogFile") -> LogParser:
+    """Build the right LogParser for a given upload's format_string.
+
+    ASSUMPTION (flagged): a blank format_string — which the upload form
+    allows — falls back to Combined, since Chapter 07 states no default
+    for that case. Anything else is compiled as-is; a real custom
+    LiteSpeed format is never silently replaced by a preset.
+    """
+    # Imported lazily to avoid a circular import (apache.py imports
+    # ParsedEntry from this module).
+    from app.services.log_parser.apache import FormatCompiledParser, combined_parser, common_parser
+    from app.services.log_parser.format_compiler import compile_format
+
+    fmt = (log_file.format_string or "").strip()
+    key = fmt.lower()
+    if key in ("", "combined"):
+        return combined_parser()
+    if key == "common":
+        return common_parser()
+    return FormatCompiledParser(compile_format(fmt))

app/services/log_parser/apache.py

@@ -0,0 +1,57 @@
+"""Apache Common/Combined presets + the generic parser built from
+format_compiler (Chapter 07)."""
+from __future__ import annotations
+
+from app.services.log_parser import ParsedEntry
+from app.services.log_parser.format_compiler import (
+    CompiledFormat, compile_format, parse_apache_timestamp, parse_request_line, validate_ip,
+)
+
+COMMON_LOG_FORMAT = '%h %l %u %t "%r" %>s %b'
+COMBINED_LOG_FORMAT = '%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-agent}i"'
+
+
+class FormatCompiledParser:
+    """LogParser implementation driven by any CompiledFormat (Ch04/07 protocol)."""
+
+    def __init__(self, compiled: CompiledFormat):
+        self._compiled = compiled
+
+    def parse_line(self, line: str) -> ParsedEntry | None:
+        match = self._compiled.pattern.match(line.rstrip("\n"))
+        if match is None:
+            return None  # malformed line — skip, never raise (Ch07)
+        fields = match.groupdict()
+
+        ip = validate_ip(fields["ip"])
+        if ip is None:
+            return None
+
+        req = parse_request_line(fields["request"])
+        if req is None:
+            return None
+        method, path = req
+
+        try:
+            timestamp = parse_apache_timestamp(fields["timestamp"])
+            status_code = int(fields["status"])
+            bytes_sent = 0 if fields["bytes"] == "-" else int(fields["bytes"])
+        except (ValueError, KeyError):
+            return None
+
+        referrer = fields.get("referrer")
+        user_agent = fields.get("user_agent")
+        return ParsedEntry(
+            timestamp=timestamp, ip=ip, method=method, path=path,
+            status_code=status_code, bytes_sent=bytes_sent,
+            referrer=None if referrer in (None, "-") else referrer,
+            user_agent=None if user_agent in (None, "-") else user_agent,
+        )
+
+
+def common_parser() -> FormatCompiledParser:
+    return FormatCompiledParser(compile_format(COMMON_LOG_FORMAT))
+
+
+def combined_parser() -> FormatCompiledParser:
+    return FormatCompiledParser(compile_format(COMBINED_LOG_FORMAT))

app/services/log_parser/format_compiler.py

@@ -0,0 +1,95 @@
+"""LogFormat directive -> compiled regex + named groups (Chapter 07).
+
+One code path for Common, Combined, and arbitrary custom formats — presets
+below are just specific format strings run through this same compiler.
+"""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from ipaddress import ip_address
+
+# Directive -> (group name, regex pattern). NOTE: %r and %{...}i patterns
+# deliberately do NOT include quote characters — the quotes around them
+# are literal text already present in the format string (e.g. `"%r"`),
+# and get regex-escaped by the literal-text path below. %t is different:
+# its brackets are part of what %t itself produces, not literal format-
+# string text, so they belong in this directive's own pattern.
+_SIMPLE_DIRECTIVES: dict[str, tuple[str, str]] = {
+    "%h": ("ip", r"(?P<ip>\S+)"),
+    "%l": ("ident", r"(?P<ident>\S+)"),
+    "%u": ("user", r"(?P<user>\S+)"),
+    "%t": ("timestamp", r"\[(?P<timestamp>[^\]]+)\]"),
+    "%r": ("request", r'(?P<request>[^"]*)'),
+    "%>s": ("status", r"(?P<status>\d{3})"),
+    "%s": ("status", r"(?P<status>\d{3})"),
+    "%b": ("bytes", r"(?P<bytes>\d+|-)"),
+}
+
+_HEADER_DIRECTIVE_RE = re.compile(r'%\{([^}]+)\}i')
+_KNOWN_HEADER_GROUPS = {"referer": "referrer", "user-agent": "user_agent"}
+_DIRECTIVE_TOKEN_RE = re.compile(r"%>?\{[^}]+\}i|%>?[a-zA-Z]")
+
+
+@dataclass(frozen=True)
+class CompiledFormat:
+    pattern: re.Pattern
+    group_names: frozenset[str]
+
+
+def compile_format(format_string: str) -> CompiledFormat:
+    """Compile a LogFormat directive (Common/Combined preset or a pasted
+    custom vhost format) into one regex."""
+    regex_parts: list[str] = []
+    group_names: set[str] = set()
+    pos = 0
+    for match in _DIRECTIVE_TOKEN_RE.finditer(format_string):
+        literal = format_string[pos:match.start()]
+        if literal:
+            regex_parts.append(re.escape(literal))
+        regex_parts.append(_directive_to_pattern(match.group(0), group_names))
+        pos = match.end()
+    regex_parts.append(re.escape(format_string[pos:]))
+
+    pattern = re.compile("^" + "".join(regex_parts) + r"\s*$")
+    return CompiledFormat(pattern=pattern, group_names=frozenset(group_names))
+
+
+def _directive_to_pattern(token: str, group_names: set[str]) -> str:
+    header_match = _HEADER_DIRECTIVE_RE.fullmatch(token)
+    if header_match:
+        header_name = header_match.group(1).lower()
+        group = _KNOWN_HEADER_GROUPS.get(header_name, re.sub(r"[^a-z0-9]+", "_", header_name))
+        group_names.add(group)
+        return f'(?P<{group}>[^"]*)'
+
+    if token not in _SIMPLE_DIRECTIVES:
+        raise ValueError(f"Unsupported LogFormat directive: {token!r}")
+    group, pattern = _SIMPLE_DIRECTIVES[token]
+    group_names.add(group)
+    return pattern
+
+
+def parse_apache_timestamp(raw: str) -> datetime:
+    """'10/Oct/2026:13:55:36 -0700' -> naive UTC datetime (Ch07: store normalized to UTC)."""
+    dt = datetime.strptime(raw, "%d/%b/%Y:%H:%M:%S %z")
+    return dt.astimezone(timezone.utc).replace(tzinfo=None)
+
+
+def parse_request_line(raw: str) -> tuple[str, str] | None:
+    """Split '%r' ("GET /path HTTP/1.1") into (method, path); None if malformed."""
+    parts = raw.split()
+    if len(parts) != 3:
+        return None
+    method, path, _protocol = parts
+    return method, path
+
+
+def validate_ip(raw: str) -> str | None:
+    """Return `raw` if a valid IPv4/IPv6 address, else None (Ch07)."""
+    try:
+        ip_address(raw)
+    except ValueError:
+        return None
+    return raw

app/services/log_parser/litespeed.py

@@ -0,0 +1,10 @@
+"""LiteSpeed uses the same LogFormat directives as Apache (Chapter 07):
+Combined generally works unmodified. get_parser_for() still always
+compiles the vhost's real format_string though — this module is just the
+named default, never a silent override of a custom format.
+"""
+from __future__ import annotations
+
+from app.services.log_parser.apache import combined_parser as default_litespeed_parser
+
+__all__ = ["default_litespeed_parser"]