// Command locallens performs federated full-text search across the index files
// of many machines, attributing every hit to the machine it came from and
// identifying identical documents (by SHA-256 content hash) that exist on more
// than one machine.
package main

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"
)

const (
	indexFormat    = "locallens-index-v1"
	defaultMaxSize = 5 * 1024 * 1024
	sniffBytes     = 8192
)

// ---------------------------------------------------------------------------
// Shared Techlosoft UX helpers (identical across the tool family).
// ---------------------------------------------------------------------------

func reorderFlags(args []string, valueFlags map[string]bool) []string {
	var flags, positional []string
	for i := 0; i < len(args); i++ {
		a := args[i]
		name := strings.TrimLeft(a, "-")
		if strings.HasPrefix(a, "-") && valueFlags[name] {
			flags = append(flags, a)
			if i+1 < len(args) {
				i++
				flags = append(flags, args[i])
			}
			continue
		}
		if strings.HasPrefix(a, "-") {
			flags = append(flags, a)
			continue
		}
		positional = append(positional, a)
	}
	return append(flags, positional...)
}

func humanBytes(n int64) string {
	const unit = 1024
	if n < unit {
		return fmt.Sprintf("%d B", n)
	}
	div, exp := int64(unit), 0
	for x := n / unit; x >= unit; x /= unit {
		div *= unit
		exp++
	}
	return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp])
}

// ---------------------------------------------------------------------------
// Index model
// ---------------------------------------------------------------------------

// document is one indexed file on one machine.
type document struct {
	Path  string `json:"path"`
	Size  int64  `json:"size"`
	MTime string `json:"mtime"`
	Hash  string `json:"hash"`
}

// machineIndex is the on-disk index for a single machine. Postings maps a term
// to a list of [docID, termCount] pairs.
type machineIndex struct {
	Format   string              `json:"format"`
	Machine  string              `json:"machine"`
	Root     string              `json:"root"`
	Built    string              `json:"built"`
	Docs     []document          `json:"docs"`
	Postings map[string][][2]int `json:"postings"`

	source string
}

// skipped records an index file that could not be used, and why.
type skipped struct {
	Index  string `json:"index"`
	Reason string `json:"reason"`
}

// ---------------------------------------------------------------------------
// Usage
// ---------------------------------------------------------------------------

const usageText = `locallens - federated search across many machines' indexes (Techlosoft Search Intelligence, Team)

USAGE
  locallens index <dir> --out <index.json> --machine <name> [--max-size 5MB]
  locallens search <terms...> [--index <a.json>]... [--dir <indexes-dir>] [--machine NAME] [--limit N] [--json]
  locallens duplicates --dir <indexes-dir> [--index <a.json>]... [--json]
  locallens machines --dir <indexes-dir> [--index <a.json>]... [--json]
  locallens help | -h | --help

COMMANDS
  index       Index one machine's directory into a single portable index file.
              Text-like files only; binaries and oversize files are skipped.
              Records path, size, mtime and SHA-256 content hash per document.

  search      AND-match every term against EVERY supplied index at once.
              Results are grouped by machine and attributed to their machine.
              A document whose content hash is present on more than one machine
              is marked SHARED and every holder is listed.

  duplicates  Content-hash groups that span 2+ machines, with wasted bytes.
              Wasted bytes for a group = (machines_with_copy - 1) * size.

  machines    Inventory of the loaded indexes: machine, documents, bytes, built.

FLAGS
  --out <file>       (index)  destination index file
  --machine <name>   (index)  machine name to stamp on the index
                     (search) restrict results to this machine
  --max-size <size>  (index)  skip files larger than this (default 5MB)
                              accepts 900, 64KB, 5MB, 1GB, 5MiB
  --index <file>     index file to search; repeatable
  --dir <dir>        load every *.json index in this directory
  --limit N          (search) maximum hits to show (default 20, 0 = all)
  --json             machine-readable output

NOTES
  A malformed or unreadable index is skipped with a reason; the remaining
  indexes are still searched. Index files are never modified by search,
  duplicates or machines.
`

func usage() {
	fmt.Fprint(os.Stderr, usageText)
	os.Exit(1)
}

func helpAndExit() {
	fmt.Fprint(os.Stdout, usageText)
	os.Exit(0)
}

func fail(format string, args ...any) {
	fmt.Fprintf(os.Stderr, "locallens: "+format+"\n", args...)
	os.Exit(1)
}

func isHelp(a string) bool {
	return a == "-h" || a == "--help" || a == "help" || a == "-help"
}

// ---------------------------------------------------------------------------
// main
// ---------------------------------------------------------------------------

func main() {
	args := os.Args[1:]
	if len(args) == 0 {
		// Double-clicked in Explorer rather than run from a prompt: ask the
		// question the program needs and stay on screen. Printing usage
		// and exiting here is what made the window vanish instantly.
		if interactiveConsole() {
			runGuided()
			return
		}
		usage()
	}
	if isHelp(args[0]) {
		helpAndExit()
	}
	for _, a := range args[1:] {
		if isHelp(a) {
			helpAndExit()
		}
	}
	switch args[0] {
	case "index":
		cmdIndex(args[1:])
	case "search":
		cmdSearch(args[1:])
	case "duplicates":
		cmdDuplicates(args[1:])
	case "machines":
		cmdMachines(args[1:])
	default:
		fmt.Fprintf(os.Stderr, "locallens: unknown command %q\n\n", args[0])
		usage()
	}
}

// repeatable is a flag.Value that collects repeated string flags.
type repeatable []string

func (r *repeatable) String() string { return strings.Join(*r, ",") }

func (r *repeatable) Set(v string) error {
	*r = append(*r, v)
	return nil
}

func newFlagSet(name string) *flag.FlagSet {
	fs := flag.NewFlagSet(name, flag.ContinueOnError)
	fs.SetOutput(io.Discard)
	fs.Usage = func() {}
	return fs
}

// ---------------------------------------------------------------------------
// index
// ---------------------------------------------------------------------------

func cmdIndex(argv []string) {
	valueFlags := map[string]bool{
		"out": true, "o": true,
		"machine": true, "m": true,
		"max-size": true,
	}
	argv = reorderFlags(argv, valueFlags)

	fs := newFlagSet("index")
	out := fs.String("out", "", "output index file")
	fs.StringVar(out, "o", "", "output index file")
	machine := fs.String("machine", "", "machine name")
	fs.StringVar(machine, "m", "", "machine name")
	maxSize := fs.String("max-size", "5MB", "maximum file size to index")
	if err := fs.Parse(argv); err != nil {
		fmt.Fprintf(os.Stderr, "locallens: %v\n\n", err)
		usage()
	}
	rest := fs.Args()
	if len(rest) != 1 {
		fmt.Fprintf(os.Stderr, "locallens: index needs exactly one directory\n\n")
		usage()
	}
	if *out == "" {
		fmt.Fprintf(os.Stderr, "locallens: index needs --out <index.json>\n\n")
		usage()
	}
	if strings.TrimSpace(*machine) == "" {
		fmt.Fprintf(os.Stderr, "locallens: index needs --machine <name>\n\n")
		usage()
	}
	limit, err := parseSize(*maxSize)
	if err != nil {
		fail("bad --max-size %q: %v", *maxSize, err)
	}

	root := rest[0]
	info, err := os.Stat(root)
	if err != nil {
		fail("cannot read directory %s: %v", root, err)
	}
	if !info.IsDir() {
		fail("%s is not a directory", root)
	}
	absRoot, err := filepath.Abs(root)
	if err != nil {
		absRoot = root
	}

	idx := &machineIndex{
		Format:   indexFormat,
		Machine:  strings.TrimSpace(*machine),
		Root:     absRoot,
		Built:    time.Now().UTC().Format(time.RFC3339),
		Postings: map[string][][2]int{},
	}

	var totalBytes int64
	var skipBinary, skipBig, skipErr int

	walkErr := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
		if err != nil {
			skipErr++
			return nil
		}
		if d.IsDir() {
			return nil
		}
		if !d.Type().IsRegular() {
			return nil
		}
		st, err := d.Info()
		if err != nil {
			skipErr++
			return nil
		}
		if st.Size() > limit {
			skipBig++
			return nil
		}
		data, err := os.ReadFile(path)
		if err != nil {
			skipErr++
			return nil
		}
		if !looksTextual(data) {
			skipBinary++
			return nil
		}
		rel, err := filepath.Rel(root, path)
		if err != nil {
			rel = path
		}
		rel = filepath.ToSlash(rel)
		sum := sha256.Sum256(data)
		docID := len(idx.Docs)
		idx.Docs = append(idx.Docs, document{
			Path:  rel,
			Size:  st.Size(),
			MTime: st.ModTime().UTC().Format(time.RFC3339),
			Hash:  hex.EncodeToString(sum[:]),
		})
		totalBytes += st.Size()
		for term, count := range tokenize(string(data)) {
			idx.Postings[term] = append(idx.Postings[term], [2]int{docID, count})
		}
		return nil
	})
	if walkErr != nil {
		fail("walking %s: %v", root, walkErr)
	}

	for term := range idx.Postings {
		p := idx.Postings[term]
		sort.Slice(p, func(i, j int) bool { return p[i][0] < p[j][0] })
	}

	if dir := filepath.Dir(*out); dir != "" && dir != "." {
		if err := os.MkdirAll(dir, 0o755); err != nil {
			fail("cannot create %s: %v", dir, err)
		}
	}
	blob, err := json.Marshal(idx)
	if err != nil {
		fail("encoding index: %v", err)
	}
	if err := os.WriteFile(*out, blob, 0o644); err != nil {
		fail("writing %s: %v", *out, err)
	}

	fmt.Printf("indexed machine %q\n", idx.Machine)
	fmt.Printf("  root       %s\n", absRoot)
	fmt.Printf("  documents  %d\n", len(idx.Docs))
	fmt.Printf("  content    %s (%d bytes)\n", humanBytes(totalBytes), totalBytes)
	fmt.Printf("  terms      %d\n", len(idx.Postings))
	fmt.Printf("  skipped    %d (binary %d, oversize %d, unreadable %d)\n",
		skipBinary+skipBig+skipErr, skipBinary, skipBig, skipErr)
	fmt.Printf("  written    %s (%s)\n", *out, humanBytes(int64(len(blob))))
}

// parseSize accepts plain byte counts and suffixed sizes (KB/K/KiB, MB, GB...).
// Suffixes are binary multiples: 1KB == 1024 bytes.
func parseSize(s string) (int64, error) {
	t := strings.TrimSpace(s)
	if t == "" {
		return 0, fmt.Errorf("empty size")
	}
	upper := strings.ToUpper(t)
	mult := int64(1)
	for _, suf := range []struct {
		name string
		mul  int64
	}{
		{"KIB", 1 << 10}, {"MIB", 1 << 20}, {"GIB", 1 << 30}, {"TIB", 1 << 40},
		{"KB", 1 << 10}, {"MB", 1 << 20}, {"GB", 1 << 30}, {"TB", 1 << 40},
		{"K", 1 << 10}, {"M", 1 << 20}, {"G", 1 << 30}, {"T", 1 << 40},
		{"B", 1},
	} {
		if strings.HasSuffix(upper, suf.name) {
			mult = suf.mul
			upper = strings.TrimSpace(strings.TrimSuffix(upper, suf.name))
			break
		}
	}
	n, err := strconv.ParseFloat(upper, 64)
	if err != nil {
		return 0, fmt.Errorf("not a size")
	}
	if n < 0 {
		return 0, fmt.Errorf("negative size")
	}
	return int64(n * float64(mult)), nil
}

// looksTextual reports whether data appears to be text rather than a binary
// blob: no NUL bytes and few control characters in the sniffed prefix.
func looksTextual(data []byte) bool {
	if len(data) == 0 {
		return true
	}
	sample := data
	if len(sample) > sniffBytes {
		sample = sample[:sniffBytes]
	}
	ctrl := 0
	for _, b := range sample {
		if b == 0 {
			return false
		}
		if b < 0x20 && b != '\t' && b != '\n' && b != '\r' && b != '\f' && b != '\v' {
			ctrl++
		}
	}
	return ctrl*100 <= len(sample)*5
}

// tokenize splits text on non-alphanumeric runes, lowercases, and returns
// term -> occurrence count.
func tokenize(text string) map[string]int {
	counts := map[string]int{}
	var b strings.Builder
	flush := func() {
		if b.Len() == 0 {
			return
		}
		tok := b.String()
		b.Reset()
		if len(tok) > 64 {
			tok = tok[:64]
		}
		counts[tok]++
	}
	for _, r := range text {
		switch {
		case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
			b.WriteRune(r)
		case r >= 'A' && r <= 'Z':
			b.WriteRune(r + 32)
		default:
			flush()
		}
	}
	flush()
	return counts
}

func normalizeTerm(s string) string {
	var b strings.Builder
	for _, r := range s {
		switch {
		case r >= 'a' && r <= 'z', r >= '0' && r <= '9':
			b.WriteRune(r)
		case r >= 'A' && r <= 'Z':
			b.WriteRune(r + 32)
		}
	}
	return b.String()
}

// ---------------------------------------------------------------------------
// Index loading (shared by search / duplicates / machines)
// ---------------------------------------------------------------------------

// collectIndexPaths merges --index values with every *.json found in --dir.
func collectIndexPaths(explicit []string, dir string) ([]string, error) {
	var paths []string
	seen := map[string]bool{}
	add := func(p string) {
		key := p
		if abs, err := filepath.Abs(p); err == nil {
			key = abs
		}
		if seen[key] {
			return
		}
		seen[key] = true
		paths = append(paths, p)
	}
	for _, p := range explicit {
		add(p)
	}
	if dir != "" {
		info, err := os.Stat(dir)
		if err != nil {
			return nil, fmt.Errorf("cannot read --dir %s: %v", dir, err)
		}
		if !info.IsDir() {
			return nil, fmt.Errorf("--dir %s is not a directory", dir)
		}
		matches, err := filepath.Glob(filepath.Join(dir, "*.json"))
		if err != nil {
			return nil, fmt.Errorf("scanning %s: %v", dir, err)
		}
		sort.Strings(matches)
		if len(matches) == 0 && len(explicit) == 0 {
			return nil, fmt.Errorf("no index files (*.json) found in %s", dir)
		}
		for _, m := range matches {
			add(m)
		}
	}
	return paths, nil
}

// loadIndexes reads every path, returning usable indexes and per-file skip
// reasons. A broken index never stops the others from loading.
func loadIndexes(paths []string) ([]*machineIndex, []skipped) {
	var loaded []*machineIndex
	var bad []skipped
	for _, p := range paths {
		data, err := os.ReadFile(p)
		if err != nil {
			bad = append(bad, skipped{Index: p, Reason: "unreadable: " + cleanErr(err)})
			continue
		}
		var idx machineIndex
		if err := json.Unmarshal(data, &idx); err != nil {
			bad = append(bad, skipped{Index: p, Reason: "malformed JSON: " + cleanErr(err)})
			continue
		}
		if idx.Format != indexFormat {
			reason := fmt.Sprintf("not a locallens index (format %q, want %q)", idx.Format, indexFormat)
			bad = append(bad, skipped{Index: p, Reason: reason})
			continue
		}
		if strings.TrimSpace(idx.Machine) == "" {
			bad = append(bad, skipped{Index: p, Reason: "index has no machine name"})
			continue
		}
		if idx.Postings == nil {
			idx.Postings = map[string][][2]int{}
		}
		valid := true
		for term, plist := range idx.Postings {
			for _, pr := range plist {
				if pr[0] < 0 || pr[0] >= len(idx.Docs) {
					bad = append(bad, skipped{Index: p, Reason: fmt.Sprintf(
						"corrupt postings: term %q references document %d of %d", term, pr[0], len(idx.Docs))})
					valid = false
					break
				}
			}
			if !valid {
				break
			}
		}
		if !valid {
			continue
		}
		idx.source = p
		loaded = append(loaded, &idx)
	}
	return loaded, bad
}

func cleanErr(err error) string {
	return strings.TrimSpace(strings.ReplaceAll(err.Error(), "\n", " "))
}

// hashOwners maps content hash -> sorted distinct machine names holding it.
// Two indexes claiming the same machine name collapse to one owner, so a file
// is never called SHARED because of a re-indexed duplicate of one machine.
func hashOwners(indexes []*machineIndex) map[string][]string {
	sets := map[string]map[string]bool{}
	for _, idx := range indexes {
		for _, d := range idx.Docs {
			if sets[d.Hash] == nil {
				sets[d.Hash] = map[string]bool{}
			}
			sets[d.Hash][idx.Machine] = true
		}
	}
	owners := make(map[string][]string, len(sets))
	for h, set := range sets {
		names := make([]string, 0, len(set))
		for n := range set {
			names = append(names, n)
		}
		sort.Strings(names)
		owners[h] = names
	}
	return owners
}

func duplicateMachineNames(indexes []*machineIndex) []string {
	count := map[string]int{}
	for _, idx := range indexes {
		count[idx.Machine]++
	}
	var dup []string
	for name, n := range count {
		if n > 1 {
			dup = append(dup, name)
		}
	}
	sort.Strings(dup)
	return dup
}

func reportSkipped(bad []skipped) {
	for _, s := range bad {
		fmt.Fprintf(os.Stderr, "locallens: skipped index %s (%s)\n", s.Index, s.Reason)
	}
}

// resolveIndexes performs the whole load pipeline and exits on fatal problems.
func resolveIndexes(explicit []string, dir string, cmd string) ([]*machineIndex, []skipped) {
	if len(explicit) == 0 && dir == "" {
		fmt.Fprintf(os.Stderr, "locallens: %s needs at least one --index <file> or --dir <indexes-dir>\n\n", cmd)
		usage()
	}
	paths, err := collectIndexPaths(explicit, dir)
	if err != nil {
		fail("%s", err)
	}
	if len(paths) == 0 {
		fail("no index files to load")
	}
	loaded, bad := loadIndexes(paths)
	if len(loaded) == 0 {
		reportSkipped(bad)
		fail("no usable indexes (%d of %d skipped)", len(bad), len(paths))
	}
	return loaded, bad
}

// ---------------------------------------------------------------------------
// search
// ---------------------------------------------------------------------------

type hit struct {
	Machine   string   `json:"machine"`
	Index     string   `json:"index"`
	Path      string   `json:"path"`
	Size      int64    `json:"size"`
	MTime     string   `json:"mtime"`
	Hash      string   `json:"hash"`
	Score     int      `json:"score"`
	Shared    bool     `json:"shared"`
	CopiesOn  []string `json:"copies_on"`
	SharedNum int      `json:"machines_with_copy"`
}

type machineGroup struct {
	Machine string `json:"machine"`
	Hits    []hit  `json:"hits"`
}

type searchReport struct {
	Query          []string       `json:"query"`
	IndexesLoaded  int            `json:"indexes_loaded"`
	IndexesSkipped []skipped      `json:"indexes_skipped"`
	Machines       []string       `json:"machines_searched"`
	MachineFilter  string         `json:"machine_filter,omitempty"`
	TotalHits      int            `json:"total_hits"`
	Shown          int            `json:"shown"`
	Results        []machineGroup `json:"results"`
}

func cmdSearch(argv []string) {
	valueFlags := map[string]bool{
		"index": true, "i": true,
		"dir": true, "d": true,
		"machine": true, "m": true,
		"limit": true, "n": true,
	}
	argv = reorderFlags(argv, valueFlags)

	fs := newFlagSet("search")
	var idxFiles repeatable
	fs.Var(&idxFiles, "index", "index file (repeatable)")
	fs.Var(&idxFiles, "i", "index file (repeatable)")
	dir := fs.String("dir", "", "directory of index files")
	fs.StringVar(dir, "d", "", "directory of index files")
	machine := fs.String("machine", "", "restrict to this machine")
	fs.StringVar(machine, "m", "", "restrict to this machine")
	limit := fs.Int("limit", 20, "maximum hits")
	fs.IntVar(limit, "n", 20, "maximum hits")
	asJSON := fs.Bool("json", false, "JSON output")
	if err := fs.Parse(argv); err != nil {
		fmt.Fprintf(os.Stderr, "locallens: %v\n\n", err)
		usage()
	}

	rawTerms := fs.Args()
	if len(rawTerms) == 0 {
		fmt.Fprintf(os.Stderr, "locallens: search needs at least one term\n\n")
		usage()
	}
	var terms []string
	for _, t := range rawTerms {
		n := normalizeTerm(t)
		if n != "" {
			terms = append(terms, n)
		}
	}
	if len(terms) == 0 {
		fail("no searchable terms after normalization (terms are alphanumeric)")
	}

	indexes, bad := resolveIndexes(idxFiles, *dir, "search")
	owners := hashOwners(indexes)

	var machineNames []string
	seenMachine := map[string]bool{}
	for _, idx := range indexes {
		if !seenMachine[idx.Machine] {
			seenMachine[idx.Machine] = true
			machineNames = append(machineNames, idx.Machine)
		}
	}
	sort.Strings(machineNames)

	if *machine != "" && !seenMachine[*machine] {
		if !*asJSON {
			reportSkipped(bad)
		}
		fail("no loaded index belongs to machine %q (have: %s)", *machine, strings.Join(machineNames, ", "))
	}

	var all []hit
	for _, idx := range indexes {
		if *machine != "" && idx.Machine != *machine {
			continue
		}
		scores := map[int]int{}
		for ti, term := range terms {
			postings := idx.Postings[term]
			if len(postings) == 0 {
				scores = nil
				break
			}
			next := map[int]int{}
			for _, p := range postings {
				if ti == 0 {
					next[p[0]] = p[1]
					continue
				}
				if prev, ok := scores[p[0]]; ok {
					next[p[0]] = prev + p[1]
				}
			}
			scores = next
			if len(scores) == 0 {
				break
			}
		}
		for docID, score := range scores {
			d := idx.Docs[docID]
			copies := owners[d.Hash]
			all = append(all, hit{
				Machine:   idx.Machine,
				Index:     idx.source,
				Path:      d.Path,
				Size:      d.Size,
				MTime:     d.MTime,
				Hash:      d.Hash,
				Score:     score,
				Shared:    len(copies) > 1,
				CopiesOn:  copies,
				SharedNum: len(copies),
			})
		}
	}

	sort.Slice(all, func(i, j int) bool {
		if all[i].Score != all[j].Score {
			return all[i].Score > all[j].Score
		}
		if all[i].Machine != all[j].Machine {
			return all[i].Machine < all[j].Machine
		}
		return all[i].Path < all[j].Path
	})

	total := len(all)
	shown := all
	if *limit > 0 && len(shown) > *limit {
		shown = shown[:*limit]
	}

	groups := map[string][]hit{}
	var groupOrder []string
	for _, h := range shown {
		if _, ok := groups[h.Machine]; !ok {
			groupOrder = append(groupOrder, h.Machine)
		}
		groups[h.Machine] = append(groups[h.Machine], h)
	}
	sort.Strings(groupOrder)

	rep := searchReport{
		Query:          terms,
		IndexesLoaded:  len(indexes),
		IndexesSkipped: bad,
		Machines:       machineNames,
		MachineFilter:  *machine,
		TotalHits:      total,
		Shown:          len(shown),
		Results:        []machineGroup{},
	}
	for _, m := range groupOrder {
		rep.Results = append(rep.Results, machineGroup{Machine: m, Hits: groups[m]})
	}
	if rep.IndexesSkipped == nil {
		rep.IndexesSkipped = []skipped{}
	}

	if *asJSON {
		emitJSON(rep)
		return
	}

	reportSkipped(bad)
	scope := fmt.Sprintf("%d machine(s)", len(machineNames))
	if *machine != "" {
		scope = fmt.Sprintf("machine %s only", *machine)
	}
	fmt.Printf("query: %s   (AND across %d index file(s), %s)\n",
		strings.Join(terms, " AND "), len(indexes), scope)
	if len(bad) > 0 {
		fmt.Printf("indexes skipped: %d\n", len(bad))
	}
	if total == 0 {
		fmt.Println()
		fmt.Println("no matches on any machine")
		return
	}
	for _, m := range groupOrder {
		hits := groups[m]
		fmt.Printf("\n== %s  (%d hit(s))\n", m, len(hits))
		for _, h := range hits {
			mark := "local-only"
			if h.Shared {
				mark = "SHARED on " + strings.Join(h.CopiesOn, ", ")
			}
			fmt.Printf("  %-34s %10s  score %-4d %s\n", h.Path, humanBytes(h.Size), h.Score, mark)
			fmt.Printf("  %-34s sha256 %s\n", "", h.Hash)
		}
	}
	fmt.Printf("\n%d hit(s)", total)
	if len(shown) < total {
		fmt.Printf(", showing %d (use --limit 0 for all)", len(shown))
	}
	fmt.Println()
}

// ---------------------------------------------------------------------------
// duplicates
// ---------------------------------------------------------------------------

type copyRef struct {
	Machine string `json:"machine"`
	Path    string `json:"path"`
	Index   string `json:"index"`
}

type dupGroup struct {
	Hash        string    `json:"hash"`
	Size        int64     `json:"size"`
	Machines    []string  `json:"machines"`
	MachineNum  int       `json:"machines_with_copy"`
	Redundant   int       `json:"redundant_copies"`
	WastedBytes int64     `json:"wasted_bytes"`
	Copies      []copyRef `json:"copies"`
}

type dupReport struct {
	IndexesLoaded  int        `json:"indexes_loaded"`
	IndexesSkipped []skipped  `json:"indexes_skipped"`
	Machines       []string   `json:"machines"`
	Groups         []dupGroup `json:"groups"`
	TotalGroups    int        `json:"total_groups"`
	TotalRedundant int        `json:"total_redundant_copies"`
	TotalWasted    int64      `json:"total_wasted_bytes"`
}

func cmdDuplicates(argv []string) {
	valueFlags := map[string]bool{
		"index": true, "i": true,
		"dir": true, "d": true,
	}
	argv = reorderFlags(argv, valueFlags)

	fs := newFlagSet("duplicates")
	var idxFiles repeatable
	fs.Var(&idxFiles, "index", "index file (repeatable)")
	fs.Var(&idxFiles, "i", "index file (repeatable)")
	dir := fs.String("dir", "", "directory of index files")
	fs.StringVar(dir, "d", "", "directory of index files")
	asJSON := fs.Bool("json", false, "JSON output")
	if err := fs.Parse(argv); err != nil {
		fmt.Fprintf(os.Stderr, "locallens: %v\n\n", err)
		usage()
	}
	if len(fs.Args()) > 0 {
		fmt.Fprintf(os.Stderr, "locallens: duplicates takes no positional arguments (got %q)\n\n", fs.Args()[0])
		usage()
	}

	indexes, bad := resolveIndexes(idxFiles, *dir, "duplicates")

	type acc struct {
		size     int64
		machines map[string]bool
		copies   []copyRef
	}
	byHash := map[string]*acc{}
	for _, idx := range indexes {
		for _, d := range idx.Docs {
			a := byHash[d.Hash]
			if a == nil {
				a = &acc{size: d.Size, machines: map[string]bool{}}
				byHash[d.Hash] = a
			}
			a.machines[idx.Machine] = true
			a.copies = append(a.copies, copyRef{Machine: idx.Machine, Path: d.Path, Index: idx.source})
		}
	}

	var groups []dupGroup
	var totalWasted int64
	totalRedundant := 0
	for h, a := range byHash {
		if len(a.machines) < 2 {
			continue
		}
		names := make([]string, 0, len(a.machines))
		for n := range a.machines {
			names = append(names, n)
		}
		sort.Strings(names)
		sort.Slice(a.copies, func(i, j int) bool {
			if a.copies[i].Machine != a.copies[j].Machine {
				return a.copies[i].Machine < a.copies[j].Machine
			}
			return a.copies[i].Path < a.copies[j].Path
		})
		redundant := len(names) - 1
		wasted := int64(redundant) * a.size
		totalWasted += wasted
		totalRedundant += redundant
		groups = append(groups, dupGroup{
			Hash:        h,
			Size:        a.size,
			Machines:    names,
			MachineNum:  len(names),
			Redundant:   redundant,
			WastedBytes: wasted,
			Copies:      a.copies,
		})
	}
	sort.Slice(groups, func(i, j int) bool {
		if groups[i].WastedBytes != groups[j].WastedBytes {
			return groups[i].WastedBytes > groups[j].WastedBytes
		}
		return groups[i].Hash < groups[j].Hash
	})

	var machineNames []string
	seen := map[string]bool{}
	for _, idx := range indexes {
		if !seen[idx.Machine] {
			seen[idx.Machine] = true
			machineNames = append(machineNames, idx.Machine)
		}
	}
	sort.Strings(machineNames)

	rep := dupReport{
		IndexesLoaded:  len(indexes),
		IndexesSkipped: bad,
		Machines:       machineNames,
		Groups:         groups,
		TotalGroups:    len(groups),
		TotalRedundant: totalRedundant,
		TotalWasted:    totalWasted,
	}
	if rep.IndexesSkipped == nil {
		rep.IndexesSkipped = []skipped{}
	}
	if rep.Groups == nil {
		rep.Groups = []dupGroup{}
	}

	if *asJSON {
		emitJSON(rep)
		return
	}

	reportSkipped(bad)
	fmt.Printf("cross-machine duplicates: %d index file(s), %d machine(s): %s\n",
		len(indexes), len(machineNames), strings.Join(machineNames, ", "))
	if len(groups) == 0 {
		fmt.Println()
		fmt.Println("no content hash appears on more than one machine")
		return
	}
	for i, g := range groups {
		fmt.Printf("\n[%d] sha256 %s\n", i+1, g.Hash)
		fmt.Printf("    size %s (%d bytes) on %d machines: %s\n",
			humanBytes(g.Size), g.Size, g.MachineNum, strings.Join(g.Machines, ", "))
		fmt.Printf("    wasted %d x %d = %d bytes (%s)\n",
			g.Redundant, g.Size, g.WastedBytes, humanBytes(g.WastedBytes))
		for _, c := range g.Copies {
			fmt.Printf("      %-10s %s\n", c.Machine, c.Path)
		}
	}
	fmt.Printf("\n%d duplicate group(s), %d redundant copy/copies, %d bytes wasted (%s)\n",
		len(groups), totalRedundant, totalWasted, humanBytes(totalWasted))
}

// ---------------------------------------------------------------------------
// machines
// ---------------------------------------------------------------------------

type machineRow struct {
	Machine   string `json:"machine"`
	Index     string `json:"index"`
	Root      string `json:"root"`
	Documents int    `json:"documents"`
	Bytes     int64  `json:"bytes"`
	Terms     int    `json:"terms"`
	Built     string `json:"built"`
}

type machinesReport struct {
	IndexesLoaded   int          `json:"indexes_loaded"`
	IndexesSkipped  []skipped    `json:"indexes_skipped"`
	Indexes         []machineRow `json:"indexes"`
	DistinctMachine int          `json:"distinct_machines"`
	DuplicateNames  []string     `json:"duplicate_machine_names"`
	TotalDocuments  int          `json:"total_documents"`
	TotalBytes      int64        `json:"total_bytes"`
}

func cmdMachines(argv []string) {
	valueFlags := map[string]bool{
		"index": true, "i": true,
		"dir": true, "d": true,
	}
	argv = reorderFlags(argv, valueFlags)

	fs := newFlagSet("machines")
	var idxFiles repeatable
	fs.Var(&idxFiles, "index", "index file (repeatable)")
	fs.Var(&idxFiles, "i", "index file (repeatable)")
	dir := fs.String("dir", "", "directory of index files")
	fs.StringVar(dir, "d", "", "directory of index files")
	asJSON := fs.Bool("json", false, "JSON output")
	if err := fs.Parse(argv); err != nil {
		fmt.Fprintf(os.Stderr, "locallens: %v\n\n", err)
		usage()
	}
	if len(fs.Args()) > 0 {
		fmt.Fprintf(os.Stderr, "locallens: machines takes no positional arguments (got %q)\n\n", fs.Args()[0])
		usage()
	}

	indexes, bad := resolveIndexes(idxFiles, *dir, "machines")

	rows := make([]machineRow, 0, len(indexes))
	var totalBytes int64
	totalDocs := 0
	distinct := map[string]bool{}
	for _, idx := range indexes {
		var bytes int64
		for _, d := range idx.Docs {
			bytes += d.Size
		}
		rows = append(rows, machineRow{
			Machine:   idx.Machine,
			Index:     idx.source,
			Root:      idx.Root,
			Documents: len(idx.Docs),
			Bytes:     bytes,
			Terms:     len(idx.Postings),
			Built:     idx.Built,
		})
		totalBytes += bytes
		totalDocs += len(idx.Docs)
		distinct[idx.Machine] = true
	}
	sort.Slice(rows, func(i, j int) bool {
		if rows[i].Machine != rows[j].Machine {
			return rows[i].Machine < rows[j].Machine
		}
		return rows[i].Index < rows[j].Index
	})

	dupNames := duplicateMachineNames(indexes)
	rep := machinesReport{
		IndexesLoaded:   len(indexes),
		IndexesSkipped:  bad,
		Indexes:         rows,
		DistinctMachine: len(distinct),
		DuplicateNames:  dupNames,
		TotalDocuments:  totalDocs,
		TotalBytes:      totalBytes,
	}
	if rep.IndexesSkipped == nil {
		rep.IndexesSkipped = []skipped{}
	}
	if rep.DuplicateNames == nil {
		rep.DuplicateNames = []string{}
	}

	if *asJSON {
		emitJSON(rep)
		return
	}

	reportSkipped(bad)
	fmt.Printf("%-12s %9s %12s %-22s %s\n", "MACHINE", "DOCUMENTS", "BYTES", "BUILT", "INDEX")
	for _, r := range rows {
		fmt.Printf("%-12s %9d %12d %-22s %s\n", r.Machine, r.Documents, r.Bytes, r.Built, r.Index)
	}
	fmt.Printf("\n%d index file(s), %d distinct machine(s), %d document(s), %d bytes (%s)\n",
		len(rows), len(distinct), totalDocs, totalBytes, humanBytes(totalBytes))
	if len(dupNames) > 0 {
		fmt.Printf("note: machine name(s) claimed by more than one index: %s\n", strings.Join(dupNames, ", "))
		fmt.Println("      each index is listed separately; duplicate-detection counts them as one machine")
	}
	if len(bad) > 0 {
		fmt.Printf("note: %d index file(s) skipped (see stderr)\n", len(bad))
	}
}

func emitJSON(v any) {
	enc := json.NewEncoder(os.Stdout)
	enc.SetIndent("", "  ")
	if err := enc.Encode(v); err != nil {
		fail("encoding JSON: %v", err)
	}
}