// Command trafficpilot is a local HTTP forward proxy that meters traffic per
// host. Point a program's HTTP proxy setting at it and every request that
// program makes is measured: which host, how many bytes out, how many bytes
// back, how long it took. Plain HTTP is proxied in absolute-form; HTTPS is
// TUNNELLED with CONNECT and never decrypted, so the host and the byte totals
// are visible and the payload is not.
package main

import (
	"bufio"
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"os/signal"
	"path"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"syscall"
	"time"
)

const version = "1.0.0"

// ---------------------------------------------------------------------------
// Shared Techlosoft UX helpers (identical across the tool family).
// ---------------------------------------------------------------------------

func reorderFlags(args []string, valueFlags map[string]bool) []string {
	var flags, positional []string
	for i := 0; i < len(args); i++ {
		a := args[i]
		name := strings.TrimLeft(a, "-")
		if strings.HasPrefix(a, "-") && valueFlags[name] {
			flags = append(flags, a)
			if i+1 < len(args) {
				i++
				flags = append(flags, args[i])
			}
			continue
		}
		if strings.HasPrefix(a, "-") {
			flags = append(flags, a)
			continue
		}
		positional = append(positional, a)
	}
	return append(flags, positional...)
}

func humanBytes(n int64) string {
	const unit = 1024
	if n < unit {
		return fmt.Sprintf("%d B", n)
	}
	div, exp := int64(unit), 0
	for x := n / unit; x >= unit; x /= unit {
		div *= unit
		exp++
	}
	return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
}

// ---------------------------------------------------------------------------
// Traffic records
// ---------------------------------------------------------------------------

// Result values for a metered request.
const (
	resultOK      = "ok"
	resultRefused = "refused"
	resultError   = "error"
)

// Protocol values. "http" means the proxy relayed a plain HTTP request and
// therefore saw the path and the payload. "connect" means the proxy opened an
// opaque TCP tunnel: only the host and the byte counts are knowable.
const (
	protoHTTP    = "http"
	protoConnect = "connect"
)

// Event is one completed request, appended to the traffic log as a JSON line.
//
// Byte accounting, precisely:
//
//	req_bytes       every byte written to the upstream host (for plain HTTP:
//	                request line + headers + body; for CONNECT: every byte the
//	                client pushed through the tunnel, TLS handshake included)
//	resp_bytes      every byte read back from the upstream host (status line +
//	                headers + body, on the wire, undecoded)
//	req_body_bytes  request payload only (plain HTTP only)
//	resp_body_bytes response payload only, exactly as handed to the client
//	                (plain HTTP only; inside a tunnel there is no payload the
//	                proxy can identify)
type Event struct {
	TS            string  `json:"ts"`
	Method        string  `json:"method"`
	Host          string  `json:"host"`
	Path          string  `json:"path,omitempty"`
	Proto         string  `json:"proto"`
	Status        int     `json:"status"`
	ReqBytes      int64   `json:"req_bytes"`
	RespBytes     int64   `json:"resp_bytes"`
	ReqBodyBytes  int64   `json:"req_body_bytes"`
	RespBodyBytes int64   `json:"resp_body_bytes"`
	DurationMS    float64 `json:"duration_ms"`
	Result        string  `json:"result"`
	Reason        string  `json:"reason,omitempty"`
}

// logWriter appends events to the traffic log. Every line is fsynced so a
// signal or a crash cannot lose a record, and a mutex keeps concurrent
// requests from interleaving inside a line.
type logWriter struct {
	mu   sync.Mutex
	path string
	f    *os.File
}

func openLog(p string) (*logWriter, error) {
	if p == "" {
		return &logWriter{}, nil
	}
	f, err := os.OpenFile(p, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
	if err != nil {
		return nil, fmt.Errorf("cannot open traffic log %s: %w", p, err)
	}
	return &logWriter{path: p, f: f}, nil
}

func (w *logWriter) append(e Event) error {
	if w == nil || w.f == nil {
		return nil
	}
	b, err := json.Marshal(e)
	if err != nil {
		return err
	}
	w.mu.Lock()
	defer w.mu.Unlock()
	if _, err := w.f.Write(append(b, '\n')); err != nil {
		return fmt.Errorf("cannot write traffic log %s: %w", w.path, err)
	}
	return w.f.Sync()
}

func (w *logWriter) Close() error {
	if w == nil || w.f == nil {
		return nil
	}
	w.mu.Lock()
	defer w.mu.Unlock()
	return w.f.Close()
}

// ---------------------------------------------------------------------------
// Host policy
// ---------------------------------------------------------------------------

type stringList []string

func (s *stringList) String() string { return strings.Join(*s, ",") }

func (s *stringList) Set(v string) error {
	v = strings.TrimSpace(v)
	if v == "" {
		return errors.New("empty host pattern")
	}
	*s = append(*s, v)
	return nil
}

// validateGlobs rejects a pattern the matcher could never use.
func validateGlobs(kind string, pats []string) error {
	for _, p := range pats {
		if strings.ContainsAny(p, "/ ") {
			return fmt.Errorf("malformed --%s pattern %q: a host pattern cannot contain %q", kind, p, "/ or space")
		}
		if _, err := path.Match(p, "example.com"); err != nil {
			return fmt.Errorf("malformed --%s pattern %q: %v", kind, p, err)
		}
	}
	return nil
}

// matchHost reports whether hostname matches any pattern. Matching is
// case-insensitive, is done against the hostname with the port removed, and
// uses shell-style globs ("*.example.com", "127.0.0.*", "*").
func matchHost(pats []string, hostname string) bool {
	h := strings.ToLower(hostname)
	for _, p := range pats {
		lp := strings.ToLower(p)
		if lp == h {
			return true
		}
		if ok, err := path.Match(lp, h); err == nil && ok {
			return true
		}
	}
	return false
}

// hostname strips the port from a host:port authority.
func hostname(hostport string) string {
	if h, _, err := net.SplitHostPort(hostport); err == nil {
		return strings.ToLower(h)
	}
	return strings.ToLower(hostport)
}

// ---------------------------------------------------------------------------
// Byte counting plumbing
// ---------------------------------------------------------------------------

type counters struct {
	read    atomic.Int64
	written atomic.Int64
}

// countedConn counts every byte that crosses an upstream connection, which is
// the only honest place to measure: it sees exactly what went on the wire,
// headers and framing included.
type countedConn struct {
	net.Conn
	c *counters
}

func (c *countedConn) Read(b []byte) (int, error) {
	n, err := c.Conn.Read(b)
	c.c.read.Add(int64(n))
	return n, err
}

func (c *countedConn) Write(b []byte) (int, error) {
	n, err := c.Conn.Write(b)
	c.c.written.Add(int64(n))
	return n, err
}

type counterKey struct{}

// countingReader counts the request payload as it is streamed upstream.
type countingReader struct {
	r io.Reader
	n atomic.Int64
}

func (c *countingReader) Read(b []byte) (int, error) {
	n, err := c.r.Read(b)
	c.n.Add(int64(n))
	return n, err
}

// ---------------------------------------------------------------------------
// The proxy
// ---------------------------------------------------------------------------

type proxy struct {
	logw     *logWriter
	allow    []string
	block    []string
	maxBody  int64
	jsonOut  bool
	tr       *http.Transport
	dialer   *net.Dialer
	requests atomic.Int64
	refused  atomic.Int64
	failed   atomic.Int64
	bytesIn  atomic.Int64
	bytesOut atomic.Int64
	quiet    bool
}

const (
	dialTimeout   = 10 * time.Second
	headerTimeout = 30 * time.Second
)

func newProxy(lw *logWriter, allow, block []string, maxBody int64, jsonOut bool) *proxy {
	p := &proxy{
		logw:    lw,
		allow:   allow,
		block:   block,
		maxBody: maxBody,
		jsonOut: jsonOut,
		dialer:  &net.Dialer{Timeout: dialTimeout},
	}
	p.tr = &http.Transport{
		Proxy: nil,
		// One connection per request keeps the byte counters attributable to
		// exactly one request; a pooled connection could not be.
		DisableKeepAlives: true,
		// The proxy must not transparently decompress: the client asked for
		// what it asked for, and the metered bytes must be wire bytes.
		DisableCompression:    true,
		ResponseHeaderTimeout: headerTimeout,
		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
			conn, err := p.dialer.DialContext(ctx, network, addr)
			if err != nil {
				return nil, err
			}
			if c, ok := ctx.Value(counterKey{}).(*counters); ok {
				return &countedConn{Conn: conn, c: c}, nil
			}
			return conn, nil
		},
	}
	return p
}

// hop-by-hop headers are consumed by the proxy and never forwarded.
var hopHeaders = []string{
	"Connection",
	"Proxy-Connection",
	"Keep-Alive",
	"Proxy-Authenticate",
	"Proxy-Authorization",
	"Te",
	"Trailer",
	"Transfer-Encoding",
	"Upgrade",
}

func stripHopHeaders(h http.Header) {
	for _, name := range strings.Split(h.Get("Connection"), ",") {
		if n := strings.TrimSpace(name); n != "" {
			h.Del(n)
		}
	}
	for _, name := range hopHeaders {
		h.Del(name)
	}
}

func (p *proxy) record(e Event) {
	p.requests.Add(1)
	p.bytesOut.Add(e.ReqBytes)
	p.bytesIn.Add(e.RespBytes)
	switch e.Result {
	case resultRefused:
		p.refused.Add(1)
	case resultError:
		p.failed.Add(1)
	}
	if err := p.logw.append(e); err != nil {
		fmt.Fprintf(os.Stderr, "trafficpilot: %v\n", err)
	}
	if p.quiet {
		return
	}
	if p.jsonOut {
		b, err := json.Marshal(e)
		if err == nil {
			fmt.Println(string(b))
		}
		return
	}
	mark := " "
	switch e.Result {
	case resultRefused:
		mark = "!"
	case resultError:
		mark = "x"
	}
	target := e.Host
	if e.Path != "" {
		target += e.Path
	}
	fmt.Printf("%s %s %-7s %-3d %-46s out %10s  in %10s  %8.1fms%s\n",
		mark, time.Now().UTC().Format("15:04:05"), e.Method, e.Status,
		truncate(target, 46), humanBytes(e.ReqBytes), humanBytes(e.RespBytes),
		e.DurationMS, reasonSuffix(e.Reason))
}

func reasonSuffix(reason string) string {
	if reason == "" {
		return ""
	}
	return "  " + reason
}

func truncate(s string, n int) string {
	if len(s) <= n {
		return s
	}
	if n <= 3 {
		return s[:n]
	}
	return s[:n-3] + "..."
}

// allowed applies --block then --allow. Block always wins; if any --allow
// pattern was given the host must match one of them.
func (p *proxy) allowed(h string) (bool, string) {
	if matchHost(p.block, h) {
		return false, "blocked by --block"
	}
	if len(p.allow) > 0 && !matchHost(p.allow, h) {
		return false, "not in --allow list"
	}
	return true, ""
}

func (p *proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	start := time.Now()
	if r.Method == http.MethodConnect {
		p.handleConnect(w, r, start)
		return
	}
	if !r.URL.IsAbs() || r.URL.Host == "" {
		// A direct (origin-form) request: the client is talking to the proxy
		// as if it were a web server. Say so instead of guessing.
		msg := "trafficpilot is a forward proxy: point HTTP_PROXY/https_proxy at it, or use curl -x. " +
			"It does not serve origin-form requests.\n"
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusBadRequest)
		_, _ = io.WriteString(w, msg)
		p.record(Event{
			TS: start.UTC().Format(time.RFC3339Nano), Method: r.Method,
			Host: hostname(r.Host), Path: r.URL.Path, Proto: protoHTTP,
			Status: http.StatusBadRequest, DurationMS: msSince(start),
			Result: resultError, Reason: "not a proxy-form request",
		})
		return
	}
	p.handleHTTP(w, r, start)
}

func msSince(t time.Time) float64 {
	return float64(time.Since(t).Microseconds()) / 1000.0
}

func (p *proxy) handleHTTP(w http.ResponseWriter, r *http.Request, start time.Time) {
	host := hostname(r.URL.Host)
	ev := Event{
		TS: start.UTC().Format(time.RFC3339Nano), Method: r.Method,
		Host: host, Path: r.URL.RequestURI(), Proto: protoHTTP,
	}
	if ok, why := p.allowed(host); !ok {
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusForbidden)
		_, _ = fmt.Fprintf(w, "trafficpilot refused %s: %s\n", host, why)
		ev.Status = http.StatusForbidden
		ev.Result = resultRefused
		ev.Reason = why
		ev.DurationMS = msSince(start)
		p.record(ev)
		return
	}

	ctrs := &counters{}
	ctx := context.WithValue(r.Context(), counterKey{}, ctrs)
	out := r.Clone(ctx)
	out.RequestURI = ""
	out.Close = false
	stripHopHeaders(out.Header)
	body := &countingReader{r: r.Body}
	if r.Body != nil {
		out.Body = io.NopCloser(body)
	}

	resp, err := p.tr.RoundTrip(out)
	if err != nil {
		ev.Status = http.StatusBadGateway
		ev.Result = resultError
		ev.Reason = cleanErr(err)
		ev.ReqBytes = ctrs.written.Load()
		ev.RespBytes = ctrs.read.Load()
		ev.ReqBodyBytes = body.n.Load()
		ev.DurationMS = msSince(start)
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusBadGateway)
		_, _ = fmt.Fprintf(w, "trafficpilot could not reach %s: %s\n", r.URL.Host, ev.Reason)
		p.record(ev)
		return
	}
	defer resp.Body.Close()

	dst := w.Header()
	for k, vs := range resp.Header {
		for _, v := range vs {
			dst.Add(k, v)
		}
	}
	stripHopHeaders(dst)
	w.WriteHeader(resp.StatusCode)

	var src io.Reader = resp.Body
	truncated := false
	if p.maxBody > 0 {
		src = io.LimitReader(resp.Body, p.maxBody)
	}
	n, copyErr := io.Copy(w, src)
	if p.maxBody > 0 && n == p.maxBody {
		// Did the body actually have more? Peek one byte.
		var probe [1]byte
		if m, _ := resp.Body.Read(probe[:]); m > 0 {
			truncated = true
		}
	}
	if f, ok := w.(http.Flusher); ok {
		f.Flush()
	}
	// Drain whatever is left so the upstream byte counter sees the whole
	// response, unless we deliberately cut it short.
	if !truncated && copyErr == nil {
		_, _ = io.Copy(io.Discard, resp.Body)
	}

	ev.Status = resp.StatusCode
	ev.ReqBytes = ctrs.written.Load()
	ev.RespBytes = ctrs.read.Load()
	ev.ReqBodyBytes = body.n.Load()
	ev.RespBodyBytes = n
	ev.DurationMS = msSince(start)
	ev.Result = resultOK
	switch {
	case truncated:
		ev.Result = resultError
		ev.Reason = fmt.Sprintf("response body cut at --max-body %d", p.maxBody)
	case copyErr != nil:
		ev.Result = resultError
		ev.Reason = "client read aborted: " + cleanErr(copyErr)
	}
	p.record(ev)
}

func (p *proxy) handleConnect(w http.ResponseWriter, r *http.Request, start time.Time) {
	target := r.Host
	if target == "" {
		target = r.URL.Host
	}
	host := hostname(target)
	ev := Event{
		TS: start.UTC().Format(time.RFC3339Nano), Method: http.MethodConnect,
		Host: host, Proto: protoConnect,
	}
	if _, _, err := net.SplitHostPort(target); err != nil {
		target = net.JoinHostPort(target, "443")
	}
	if ok, why := p.allowed(host); !ok {
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusForbidden)
		_, _ = fmt.Fprintf(w, "trafficpilot refused %s: %s\n", host, why)
		ev.Status = http.StatusForbidden
		ev.Result = resultRefused
		ev.Reason = why
		ev.DurationMS = msSince(start)
		p.record(ev)
		return
	}

	hj, ok := w.(http.Hijacker)
	if !ok {
		http.Error(w, "trafficpilot: CONNECT unsupported on this connection", http.StatusInternalServerError)
		return
	}

	upstream, err := p.dialer.DialContext(r.Context(), "tcp", target)
	if err != nil {
		ev.Status = http.StatusBadGateway
		ev.Result = resultError
		ev.Reason = cleanErr(err)
		ev.DurationMS = msSince(start)
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusBadGateway)
		_, _ = fmt.Fprintf(w, "trafficpilot could not reach %s: %s\n", target, ev.Reason)
		p.record(ev)
		return
	}

	client, brw, err := hj.Hijack()
	if err != nil {
		_ = upstream.Close()
		ev.Status = http.StatusInternalServerError
		ev.Result = resultError
		ev.Reason = "cannot hijack client connection: " + cleanErr(err)
		ev.DurationMS = msSince(start)
		p.record(ev)
		return
	}
	if _, err := brw.WriteString("HTTP/1.1 200 Connection established\r\n\r\n"); err != nil {
		_ = upstream.Close()
		_ = client.Close()
		return
	}
	if err := brw.Flush(); err != nil {
		_ = upstream.Close()
		_ = client.Close()
		return
	}

	// From here the proxy is a pipe. Everything inside is TLS: the byte
	// counts are real, the contents are not knowable and are never inspected.
	var up, down int64
	var wg sync.WaitGroup
	wg.Add(2)
	go func() {
		defer wg.Done()
		n, _ := io.Copy(upstream, brw.Reader)
		atomic.StoreInt64(&up, n)
		halfClose(upstream)
	}()
	go func() {
		defer wg.Done()
		n, _ := io.Copy(client, upstream)
		atomic.StoreInt64(&down, n)
		halfClose(client)
	}()
	wg.Wait()
	_ = upstream.Close()
	_ = client.Close()

	ev.Status = http.StatusOK
	ev.ReqBytes = atomic.LoadInt64(&up)
	ev.RespBytes = atomic.LoadInt64(&down)
	ev.DurationMS = msSince(start)
	ev.Result = resultOK
	ev.Reason = "tunnelled, not decrypted"
	p.record(ev)
}

func halfClose(c net.Conn) {
	if t, ok := c.(*net.TCPConn); ok {
		_ = t.CloseWrite()
		return
	}
	_ = c.Close()
}

func cleanErr(err error) string {
	msg := strings.ReplaceAll(err.Error(), "\n", " ")
	if len(msg) > 200 {
		msg = msg[:197] + "..."
	}
	return msg
}

// ---------------------------------------------------------------------------
// proxy command
// ---------------------------------------------------------------------------

func cmdProxy(args []string) error {
	if wantsHelp(args) {
		usageProxy(os.Stdout)
		return nil
	}
	fs := flag.NewFlagSet("proxy", flag.ContinueOnError)
	fs.SetOutput(io.Discard)
	listen := fs.String("listen", "127.0.0.1:8080", "address to listen on (host:port)")
	logPath := fs.String("log", "", "append a JSON line per request to this file")
	maxBody := fs.Int64("max-body", 0, "cut a relayed response body after N bytes (0 = unlimited)")
	asJSON := fs.Bool("json", false, "print each metered request as JSON")
	quiet := fs.Bool("quiet", false, "do not print per-request lines")
	var allow, block stringList
	fs.Var(&allow, "allow", "only these host globs may be proxied (repeatable)")
	fs.Var(&block, "block", "refuse these host globs (repeatable)")
	args = reorderFlags(args, map[string]bool{
		"listen": true, "log": true, "max-body": true, "allow": true, "block": true,
	})
	if err := fs.Parse(args); err != nil {
		failUsage(usageProxy, "%v", err)
	}
	if fs.NArg() > 0 {
		failUsage(usageProxy, "unexpected argument %q", fs.Arg(0))
	}
	if strings.TrimSpace(*listen) == "" {
		failUsage(usageProxy, "--listen is required")
	}
	if _, _, err := net.SplitHostPort(*listen); err != nil {
		failUsage(usageProxy, "--listen must be host:port, got %q", *listen)
	}
	if *maxBody < 0 {
		failUsage(usageProxy, "--max-body cannot be negative")
	}
	if err := validateGlobs("allow", allow); err != nil {
		failUsage(usageProxy, "%v", err)
	}
	if err := validateGlobs("block", block); err != nil {
		failUsage(usageProxy, "%v", err)
	}

	lw, err := openLog(*logPath)
	if err != nil {
		return err
	}
	defer lw.Close()

	ln, err := net.Listen("tcp", *listen)
	if err != nil {
		return fmt.Errorf("cannot listen on %s: %v", *listen, cleanErr(err))
	}

	p := newProxy(lw, allow, block, *maxBody, *asJSON)
	p.quiet = *quiet
	srv := &http.Server{
		Handler:           p,
		ReadHeaderTimeout: headerTimeout,
	}

	started := time.Now().UTC()
	fmt.Printf("trafficpilot %s proxy listening on %s\n", version, ln.Addr().String())
	if *logPath != "" {
		fmt.Printf("  traffic log : %s (one JSON line per completed request)\n", *logPath)
	} else {
		fmt.Printf("  traffic log : (none; pass --log traffic.jsonl to keep a record)\n")
	}
	if len(allow) > 0 {
		fmt.Printf("  allow       : %s\n", strings.Join(allow, " "))
	}
	if len(block) > 0 {
		fmt.Printf("  block       : %s\n", strings.Join(block, " "))
	}
	if *maxBody > 0 {
		fmt.Printf("  max-body    : %d bytes per response\n", *maxBody)
	}
	fmt.Printf("  configure a client with:  http_proxy=http://%s https_proxy=http://%s\n", ln.Addr(), ln.Addr())
	fmt.Printf("  HTTPS is tunnelled with CONNECT and NOT decrypted: host and byte counts only.\n")
	fmt.Println(strings.Repeat("-", 108))

	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()

	serveErr := make(chan error, 1)
	go func() {
		err := srv.Serve(ln)
		if errors.Is(err, http.ErrServerClosed) {
			err = nil
		}
		serveErr <- err
	}()

	select {
	case err := <-serveErr:
		if err != nil {
			return fmt.Errorf("proxy stopped: %v", cleanErr(err))
		}
	case <-ctx.Done():
		fmt.Printf("\nsignal received, closing listener and flushing traffic log\n")
		shutCtx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
		defer cancel()
		if err := srv.Shutdown(shutCtx); err != nil {
			// Hijacked CONNECT tunnels are not tracked by Shutdown; cut them.
			_ = srv.Close()
		}
	}
	elapsed := time.Since(started).Round(time.Millisecond)
	fmt.Printf("served %d request(s) in %s: %d refused, %d failed, %s out, %s in\n",
		p.requests.Load(), elapsed, p.refused.Load(), p.failed.Load(),
		humanBytes(p.bytesOut.Load()), humanBytes(p.bytesIn.Load()))
	return nil
}

// ---------------------------------------------------------------------------
// Reading the traffic log
// ---------------------------------------------------------------------------

// SkipNote explains one log line that could not be used.
type SkipNote struct {
	Line   int    `json:"line"`
	Reason string `json:"reason"`
}

func parseLine(n int, line string) (Event, *SkipNote) {
	var e Event
	if err := json.Unmarshal([]byte(line), &e); err != nil {
		return e, &SkipNote{Line: n, Reason: "not valid JSON: " + cleanErr(err)}
	}
	if strings.TrimSpace(e.Host) == "" {
		return e, &SkipNote{Line: n, Reason: "record has no host"}
	}
	if e.TS == "" {
		return e, &SkipNote{Line: n, Reason: "record has no timestamp"}
	}
	if _, err := time.Parse(time.RFC3339Nano, e.TS); err != nil {
		return e, &SkipNote{Line: n, Reason: fmt.Sprintf("bad timestamp %q", e.TS)}
	}
	return e, nil
}

func readLog(p string) ([]Event, []SkipNote, error) {
	f, err := os.Open(p)
	if err != nil {
		return nil, nil, err
	}
	defer f.Close()
	var events []Event
	var skipped []SkipNote
	sc := bufio.NewScanner(f)
	sc.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
	n := 0
	for sc.Scan() {
		n++
		line := strings.TrimSpace(sc.Text())
		if line == "" {
			continue
		}
		e, note := parseLine(n, line)
		if note != nil {
			skipped = append(skipped, *note)
			continue
		}
		events = append(events, e)
	}
	if err := sc.Err(); err != nil {
		return nil, nil, fmt.Errorf("cannot read %s: %v", p, cleanErr(err))
	}
	return events, skipped, nil
}

// ---------------------------------------------------------------------------
// report
// ---------------------------------------------------------------------------

// HostStat is the per-host arithmetic. BytesIn is what the host sent back
// (download); BytesOut is what was sent to it (upload). SharePercent is that
// host's slice of all metered bytes in the log.
type HostStat struct {
	Host          string  `json:"host"`
	Requests      int     `json:"requests"`
	BytesIn       int64   `json:"bytes_in"`
	BytesOut      int64   `json:"bytes_out"`
	BytesTotal    int64   `json:"bytes_total"`
	AvgDurationMS float64 `json:"avg_duration_ms"`
	SharePercent  float64 `json:"share_percent"`
	Refused       int     `json:"refused"`
	Errors        int     `json:"errors"`
	Tunnelled     int     `json:"tunnelled"`
	FirstSeen     string  `json:"first_seen"`
	LastSeen      string  `json:"last_seen"`
}

// Totals is the whole-log summary.
type Totals struct {
	Hosts      int     `json:"hosts"`
	Requests   int     `json:"requests"`
	BytesIn    int64   `json:"bytes_in"`
	BytesOut   int64   `json:"bytes_out"`
	BytesTotal int64   `json:"bytes_total"`
	AvgMS      float64 `json:"avg_duration_ms"`
	Refused    int     `json:"refused"`
	Errors     int     `json:"errors"`
	Tunnelled  int     `json:"tunnelled"`
}

// ReportPayload is the `report --json` document.
type ReportPayload struct {
	Log         string     `json:"log"`
	LogSize     string     `json:"log_size"`
	Records     int        `json:"records"`
	Skipped     []SkipNote `json:"skipped"`
	WindowStart string     `json:"window_start,omitempty"`
	WindowEnd   string     `json:"window_end,omitempty"`
	Hosts       []HostStat `json:"hosts"`
	Totals      Totals     `json:"totals"`
	BusiestHost string     `json:"busiest_host"`
	Shown       int        `json:"shown"`
}

func buildReport(events []Event) ([]HostStat, Totals, string) {
	byHost := map[string]*HostStat{}
	durs := map[string]float64{}
	var tot Totals
	var totalDur float64
	for _, e := range events {
		h, ok := byHost[e.Host]
		if !ok {
			h = &HostStat{Host: e.Host, FirstSeen: e.TS, LastSeen: e.TS}
			byHost[e.Host] = h
		}
		h.Requests++
		h.BytesIn += e.RespBytes
		h.BytesOut += e.ReqBytes
		h.BytesTotal += e.RespBytes + e.ReqBytes
		durs[e.Host] += e.DurationMS
		if e.TS < h.FirstSeen {
			h.FirstSeen = e.TS
		}
		if e.TS > h.LastSeen {
			h.LastSeen = e.TS
		}
		switch e.Result {
		case resultRefused:
			h.Refused++
			tot.Refused++
		case resultError:
			h.Errors++
			tot.Errors++
		}
		if e.Proto == protoConnect {
			h.Tunnelled++
			tot.Tunnelled++
		}
		tot.Requests++
		tot.BytesIn += e.RespBytes
		tot.BytesOut += e.ReqBytes
		totalDur += e.DurationMS
	}
	tot.BytesTotal = tot.BytesIn + tot.BytesOut
	tot.Hosts = len(byHost)
	if tot.Requests > 0 {
		tot.AvgMS = totalDur / float64(tot.Requests)
	}
	stats := make([]HostStat, 0, len(byHost))
	for _, h := range byHost {
		if h.Requests > 0 {
			h.AvgDurationMS = durs[h.Host] / float64(h.Requests)
		}
		if tot.BytesTotal > 0 {
			h.SharePercent = float64(h.BytesTotal) / float64(tot.BytesTotal) * 100
		}
		stats = append(stats, *h)
	}
	sort.SliceStable(stats, func(i, j int) bool {
		if stats[i].BytesTotal != stats[j].BytesTotal {
			return stats[i].BytesTotal > stats[j].BytesTotal
		}
		if stats[i].Requests != stats[j].Requests {
			return stats[i].Requests > stats[j].Requests
		}
		return stats[i].Host < stats[j].Host
	})
	busiest := ""
	if len(stats) > 0 {
		busiest = stats[0].Host
	}
	return stats, tot, busiest
}

func cmdReport(args []string) error {
	if wantsHelp(args) {
		usageReport(os.Stdout)
		return nil
	}
	fs := flag.NewFlagSet("report", flag.ContinueOnError)
	fs.SetOutput(io.Discard)
	logPath := fs.String("log", "", "traffic log JSONL written by `trafficpilot proxy`")
	top := fs.Int("top", 0, "show only the N busiest hosts (0 = all)")
	asJSON := fs.Bool("json", false, "emit JSON instead of text")
	args = reorderFlags(args, map[string]bool{"log": true, "top": true})
	if err := fs.Parse(args); err != nil {
		failUsage(usageReport, "%v", err)
	}
	if fs.NArg() > 0 {
		failUsage(usageReport, "unexpected argument %q", fs.Arg(0))
	}
	if *logPath == "" {
		failUsage(usageReport, "--log is required")
	}
	if *top < 0 {
		failUsage(usageReport, "--top cannot be negative")
	}
	events, skipped, err := readLog(*logPath)
	if err != nil {
		if errors.Is(err, os.ErrNotExist) {
			return fmt.Errorf("traffic log not found: %s (run `trafficpilot proxy --log %s` first)", *logPath, *logPath)
		}
		return err
	}
	size := int64(0)
	if fi, err := os.Stat(*logPath); err == nil {
		size = fi.Size()
	}
	stats, tot, busiest := buildReport(events)
	shown := stats
	if *top > 0 && *top < len(shown) {
		shown = shown[:*top]
	}
	rep := ReportPayload{
		Log: *logPath, LogSize: humanBytes(size), Records: len(events),
		Skipped: skipped, Hosts: shown, Totals: tot, BusiestHost: busiest,
		Shown: len(shown),
	}
	if rep.Skipped == nil {
		rep.Skipped = []SkipNote{}
	}
	if rep.Hosts == nil {
		rep.Hosts = []HostStat{}
	}
	if len(events) > 0 {
		start, end := events[0].TS, events[0].TS
		for _, e := range events {
			if e.TS < start {
				start = e.TS
			}
			if e.TS > end {
				end = e.TS
			}
		}
		rep.WindowStart, rep.WindowEnd = start, end
	}
	if *asJSON {
		enc := json.NewEncoder(os.Stdout)
		enc.SetIndent("", "  ")
		return enc.Encode(rep)
	}

	fmt.Printf("trafficpilot report  %s  (%s, %d record(s))\n", rep.Log, rep.LogSize, rep.Records)
	for _, s := range skipped {
		fmt.Printf("  skipped line %d: %s\n", s.Line, s.Reason)
	}
	if len(events) == 0 {
		fmt.Println("no usable records: nothing has been metered yet")
		return nil
	}
	fmt.Printf("window: %s -> %s\n", rep.WindowStart, rep.WindowEnd)
	fmt.Println(strings.Repeat("=", 108))
	fmt.Printf("%-34s %6s %12s %12s %12s %9s %8s %6s %6s\n",
		"HOST", "REQS", "IN", "OUT", "TOTAL", "SHARE", "AVG", "REFUS", "ERR")
	fmt.Println(strings.Repeat("-", 108))
	for _, s := range shown {
		fmt.Printf("%-34s %6d %12s %12s %12s %8.2f%% %7.1fms %6d %6d\n",
			truncate(s.Host, 34), s.Requests, humanBytes(s.BytesIn), humanBytes(s.BytesOut),
			humanBytes(s.BytesTotal), s.SharePercent, s.AvgDurationMS, s.Refused, s.Errors)
	}
	fmt.Println(strings.Repeat("-", 108))
	if len(shown) < len(stats) {
		fmt.Printf("(showing top %d of %d hosts)\n", len(shown), len(stats))
	}
	fmt.Printf("TOTAL: %d host(s), %d request(s) | in %s | out %s | both %s | avg %.1fms | %d refused | %d error(s) | %d tunnelled\n",
		tot.Hosts, tot.Requests, humanBytes(tot.BytesIn), humanBytes(tot.BytesOut),
		humanBytes(tot.BytesTotal), tot.AvgMS, tot.Refused, tot.Errors, tot.Tunnelled)
	if busiest != "" {
		b := stats[0]
		fmt.Printf("BUSIEST: %s with %s (%.2f%% of all metered bytes over %d request(s))\n",
			b.Host, humanBytes(b.BytesTotal), b.SharePercent, b.Requests)
	}
	fmt.Println()
	fmt.Println("in  = bytes the host sent back (download)   out = bytes sent to the host (upload)")
	fmt.Println("CONNECT rows are tunnelled HTTPS: byte totals are real, contents were never seen.")
	return nil
}

// ---------------------------------------------------------------------------
// watch
// ---------------------------------------------------------------------------

func cmdWatch(args []string) error {
	if wantsHelp(args) {
		usageWatch(os.Stdout)
		return nil
	}
	fs := flag.NewFlagSet("watch", flag.ContinueOnError)
	fs.SetOutput(io.Discard)
	logPath := fs.String("log", "", "traffic log JSONL to follow")
	interval := fs.Duration("interval", 2*time.Second, "summary interval")
	fromStart := fs.Bool("from-start", false, "include records already in the log")
	args = reorderFlags(args, map[string]bool{"log": true, "interval": true})
	if err := fs.Parse(args); err != nil {
		failUsage(usageWatch, "%v", err)
	}
	if fs.NArg() > 0 {
		failUsage(usageWatch, "unexpected argument %q", fs.Arg(0))
	}
	if *logPath == "" {
		failUsage(usageWatch, "--log is required")
	}
	if *interval <= 0 {
		failUsage(usageWatch, "--interval must be positive")
	}

	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()

	fmt.Printf("trafficpilot watch  %s  (every %s; Ctrl-C to stop)\n", *logPath, *interval)
	fmt.Println(strings.Repeat("-", 108))

	var offset int64
	if !*fromStart {
		if fi, err := os.Stat(*logPath); err == nil {
			offset = fi.Size()
		}
	}
	var totalReq int
	var totalIn, totalOut int64
	seenHosts := map[string]bool{}
	tick := time.NewTicker(*interval)
	defer tick.Stop()
	for {
		select {
		case <-ctx.Done():
			fmt.Printf("\nstopped: %d request(s) seen, %d host(s), %s in, %s out\n",
				totalReq, len(seenHosts), humanBytes(totalIn), humanBytes(totalOut))
			return nil
		case <-tick.C:
		}
		events, newOffset, err := tailFrom(*logPath, offset)
		if err != nil {
			if errors.Is(err, os.ErrNotExist) {
				fmt.Printf("[%s] waiting for %s to appear\n", time.Now().UTC().Format("15:04:05"), *logPath)
				continue
			}
			return err
		}
		offset = newOffset
		now := time.Now().UTC().Format("15:04:05")
		if len(events) == 0 {
			fmt.Printf("[%s] idle | cumulative %d req, %d host(s), %s in, %s out\n",
				now, totalReq, len(seenHosts), humanBytes(totalIn), humanBytes(totalOut))
			continue
		}
		stats, tot, busiest := buildReport(events)
		for _, s := range stats {
			seenHosts[s.Host] = true
		}
		totalReq += tot.Requests
		totalIn += tot.BytesIn
		totalOut += tot.BytesOut
		fmt.Printf("[%s] +%d req over %d host(s) | +%s in | +%s out | busiest %s | cumulative %d req, %s in\n",
			now, tot.Requests, tot.Hosts, humanBytes(tot.BytesIn), humanBytes(tot.BytesOut),
			busiest, totalReq, humanBytes(totalIn))
		for i, s := range stats {
			if i >= 3 {
				fmt.Printf("      ... and %d more host(s)\n", len(stats)-3)
				break
			}
			fmt.Printf("      %-34s %3d req  in %10s  out %10s  %6.1fms\n",
				truncate(s.Host, 34), s.Requests, humanBytes(s.BytesIn),
				humanBytes(s.BytesOut), s.AvgDurationMS)
		}
	}
}

// tailFrom reads whole lines added after offset and returns the new offset,
// which never advances past a partially written final line.
func tailFrom(p string, offset int64) ([]Event, int64, error) {
	f, err := os.Open(p)
	if err != nil {
		return nil, offset, err
	}
	defer f.Close()
	fi, err := f.Stat()
	if err != nil {
		return nil, offset, err
	}
	if fi.Size() < offset {
		// Truncated or rotated underneath us: start again from the top.
		offset = 0
	}
	if fi.Size() == offset {
		return nil, offset, nil
	}
	if _, err := f.Seek(offset, io.SeekStart); err != nil {
		return nil, offset, err
	}
	buf := make([]byte, fi.Size()-offset)
	n, err := io.ReadFull(f, buf)
	if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) && !errors.Is(err, io.EOF) {
		return nil, offset, err
	}
	buf = buf[:n]
	last := strings.LastIndexByte(string(buf), '\n')
	if last < 0 {
		return nil, offset, nil
	}
	chunk := string(buf[:last+1])
	var events []Event
	for i, line := range strings.Split(chunk, "\n") {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}
		e, note := parseLine(i+1, line)
		if note != nil {
			fmt.Printf("      (skipped a record: %s)\n", note.Reason)
			continue
		}
		events = append(events, e)
	}
	return events, offset + int64(last+1), nil
}

// ---------------------------------------------------------------------------
// Usage
// ---------------------------------------------------------------------------

func wantsHelp(args []string) bool {
	for _, a := range args {
		if a == "-h" || a == "--help" || a == "help" {
			return true
		}
	}
	return false
}

func failUsage(u func(io.Writer), format string, a ...any) {
	fmt.Fprintf(os.Stderr, "trafficpilot: "+format+"\n\n", a...)
	u(os.Stderr)
	os.Exit(1)
}

func usage(w io.Writer) {
	fmt.Fprintf(w, `trafficpilot %s - a local HTTP proxy that meters traffic per host

USAGE
    trafficpilot <command> [flags]

COMMANDS
    proxy     Run the metering forward proxy
    report    Per-host bytes, share of total and averages from a traffic log
    watch     Live tail of a traffic log as requests arrive
    help      Show this help

WHAT IT ANSWERS
    "What is actually using my bandwidth?" - measured, not guessed. Point a
    program's proxy setting at trafficpilot and every host it talks to, and
    every byte that costs you, is written to a JSON-lines traffic log.

WHAT IT DOES NOT DO
    HTTPS is TUNNELLED with CONNECT and NEVER DECRYPTED. trafficpilot sees the
    host and the byte counts, and nothing else. It installs no root
    certificate and intercepts no TLS. It also only sees a program's traffic
    if that program is configured to use it.

EXAMPLES
    trafficpilot proxy  --listen 127.0.0.1:8080 --log traffic.jsonl
    http_proxy=http://127.0.0.1:8080 https_proxy=http://127.0.0.1:8080 curl http://example.com/
    curl -x http://127.0.0.1:8080 http://example.com/
    trafficpilot report --log traffic.jsonl --top 10
    trafficpilot watch  --log traffic.jsonl --interval 2s

Run "trafficpilot <command> --help" for per-command flags.
`, version)
}

func usageProxy(w io.Writer) {
	fmt.Fprint(w, `trafficpilot proxy - the metering forward proxy

USAGE
    trafficpilot proxy --listen <host:port> [flags]

FLAGS
    --listen <host:port>  Address to listen on (default 127.0.0.1:8080)
    --log <path>          Append one JSON line per completed request
    --allow <host-glob>   Only these hosts may be proxied (repeatable)
    --block <host-glob>   Refuse these hosts (repeatable; block beats allow)
    --max-body <N>        Cut a relayed response body after N bytes (0 = off)
    --json                Print each metered request as JSON
    --quiet               Do not print per-request lines
    -h, --help            Show this help

HOW CLIENTS REACH IT
    curl -x http://127.0.0.1:8080 http://host/path
    http_proxy=http://127.0.0.1:8080 https_proxy=http://127.0.0.1:8080 <program>

WHAT IS MEASURED
    plain HTTP   absolute-form request URIs are relayed; the log records the
                 method, host, path, status, exact upstream bytes in and out,
                 the payload sizes on their own, and the duration
    HTTPS        CONNECT opens an opaque TCP tunnel. The log records the host
                 and the byte totals. There is NO path and NO content: the
                 bytes are TLS and trafficpilot does not decrypt them.

HOST GLOBS
    Shell-style, case-insensitive, matched against the hostname with the port
    removed: "example.com", "*.example.com", "127.0.0.*", "*".
    A refusal is answered with 403 and is written to the log as "refused".

SIGNALS
    Ctrl-C or SIGTERM closes the listener, flushes the log and prints a
    session summary. Every log line is fsynced as it is written.
`)
}

func usageReport(w io.Writer) {
	fmt.Fprint(w, `trafficpilot report - per-host traffic accounting

USAGE
    trafficpilot report --log <traffic.jsonl> [flags]

FLAGS
    --log <path>    Traffic log written by "trafficpilot proxy" (required)
    --top <N>       Show only the N busiest hosts (0 = all)
    --json          Emit JSON instead of text
    -h, --help      Show this help

PER HOST
    requests, bytes in (download), bytes out (upload), their total, the
    average request duration, and that host's share of all metered bytes.
    Plus whole-log totals and the busiest host.

DEFINITIONS
    bytes in       resp_bytes summed: every byte the host sent back
    bytes out      req_bytes summed: every byte sent to the host
    share %        host total bytes / all metered bytes * 100
    A malformed log line is skipped with a printed reason; the rest still count.
`)
}

func usageWatch(w io.Writer) {
	fmt.Fprint(w, `trafficpilot watch - live tail of a traffic log

USAGE
    trafficpilot watch --log <traffic.jsonl> [flags]

FLAGS
    --log <path>       Traffic log to follow (required)
    --interval <dur>   Summary interval (default 2s)
    --from-start       Include records already in the log
    -h, --help         Show this help

Each interval prints what arrived since the last one - requests, hosts, bytes
in and out, the busiest host - and a running cumulative total. A half-written
final line is never parsed; it is picked up once its newline lands.
`)
}

func main() {
	args := os.Args[1:]
	if len(args) == 0 {
		// Double-clicked in Explorer rather than run from a prompt: ask the
		// one question the program needs and stay on screen. Printing usage
		// and exiting here is what made the window vanish instantly.
		if interactiveConsole() {
			runGuided()
			return
		}
		usage(os.Stderr)
		os.Exit(1)
	}
	switch args[0] {
	case "-h", "--help", "help":
		usage(os.Stdout)
		return
	case "-v", "--version", "version":
		fmt.Printf("trafficpilot %s\n", version)
		return
	}
	var err error
	switch args[0] {
	case "proxy":
		err = cmdProxy(args[1:])
	case "report":
		err = cmdReport(args[1:])
	case "watch":
		err = cmdWatch(args[1:])
	default:
		fmt.Fprintf(os.Stderr, "trafficpilot: unknown command %q\n\n", args[0])
		usage(os.Stderr)
		os.Exit(1)
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "trafficpilot: %v\n", err)
		os.Exit(1)
	}
}