Refactor: split monolith into cmd/palette + internal/{store,api,web,lang} (#35)

2026-09-09 01:33:39 -05:00
parent a3349b4a98
commit 4f1e901f04
43 changed files with 1251 additions and 1203 deletions
@@ -0,0 +1,110 @@
+package lang
+
+import (
+	"encoding/json"
+	"regexp"
+	"strings"
+
+	"github.com/go-enry/go-enry/v2"
+)
+
+// hintRule is a lightweight regex hint that nudges detection. Hints don't
+// decide on their own: matching hints are passed to enry's classifier as
+// candidate languages, and enry (trained on Linguist samples) makes the final
+// call. Adding a language later is usually a one-line addition here plus the
+// dropdown in web/templates/new.html.
+type hintRule struct {
+	lang string // enry language name
+	re   *regexp.Regexp
+}
+
+// hintRules are evaluated in order; keep more specific languages earlier so
+// ties break in their favor.
+var hintRules = []hintRule{
+	{"Dockerfile", regexp.MustCompile(`(?mi)^(FROM\s+\S+:\S*|RUN\s+\S+|COPY\s+\S+\s+\S+|ENTRYPOINT\s+|WORKDIR\s+/)`)},
+	{"Diff", regexp.MustCompile(`(?m)^(diff --git|--- a/|\+\+\+ b/|@@ -\d+)`)},
+	{"PHP", regexp.MustCompile(`(?m)<\?php|\$\w+\s*=\s*[^=]|\becho\s+["'$]`)},
+	{"HTML", regexp.MustCompile(`(?i)<(!DOCTYPE|html|head|body|div|span|script|p|a)\b`)},
+	{"XML", regexp.MustCompile(`(?m)^<\?xml\b|<\/?[a-zA-Z][\w.-]*:[\w.-]*[>\s]`)},
+	{"CSS", regexp.MustCompile(`(?m)(^|\})\s*[^{}@]+\{[^}]*:[^}]*\}|@(media|import|font-face)\b`)},
+	{"TypeScript", regexp.MustCompile(`(?m)(:\s*(string|number|boolean|any)\b|\binterface \w+ \{|\btype \w+ =|\bimplements \w+)`)},
+	{"TOML", regexp.MustCompile(`(?m)^\[[\w."-]+\]\s*$|^\w[\w-]*\s*=\s*("[^"]*"|\d+|true|false|\[[^\]]*\])\s*$`)},
+	{"INI", regexp.MustCompile(`(?m)^\[[\w.-]+\]\s*$|^\w[\w.-]*\s*=\s*\S+\s*$`)},
+	{"Ruby", regexp.MustCompile(`(?m)(\bdef \w+($|\s)|\brequire ['"]|\bputs \w+|@\w+\s*=\s*[^=]|\bend\b\s*$)`)},
+	{"Perl", regexp.MustCompile(`(?m)(\buse strict\b|\bmy \$\w+|->\{|sub \w+ \{)`)},
+	{"Lua", regexp.MustCompile(`(?m)(\bfunction\s+\w+\s*\(|\blocal \w+\s*=|\bthen\b|\belseif\b|\.\.\.)`)},
+	{"Go", regexp.MustCompile(`(?m)^\s*(package \w+|import \(|func (\w+|\() )`)},
+	{"Python", regexp.MustCompile(`(?m)^\s*(def \w+|class \w+|import \w+|from \w+ import |@\w+)`)},
+	{"JavaScript", regexp.MustCompile(`(?m)(\bconst \w+ = |require\(|import \w+ from |=> \{|\bconsole\.log\()` )},
+	{"Rust", regexp.MustCompile(`(?m)(\bfn \w+|let mut \b|\bimpl \b|use std::)`)},
+	{"Java", regexp.MustCompile(`(?m)(\bpublic (static |final |class )|\bSystem\.out\.print|import java\.)`)},
+	{"C", regexp.MustCompile(`(?m)(#include\s*<\w+\.h>|printf\(|\bint main\()` )},
+	{"C++", regexp.MustCompile(`(?m)(#include\s*<(iostream|vector|string)>|std::|\bcout\s*<<)`)},
+	{"SQL", regexp.MustCompile(`(?i)\b(SELECT .+ FROM|INSERT INTO|CREATE TABLE|UPDATE \w+ SET)\b`)},
+	{"YAML", regexp.MustCompile(`(?m)^(\w[\w-]*:\s*(\||\S)|  \w[\w-]*: |---\s*$)`)},
+	{"Markdown", regexp.MustCompile("(?m)^(#{1,6} \\S|\\|.*\\||-\\s\\[\\s?\\]|```)")},
+	{"Shell", regexp.MustCompile(`(?m)^(#!.*bash|#!.*sh|\w+\(\)\s*\{)` )},
+}
+
+// canonical maps enry display names to the lowercase ids we store and render.
+var canonical = map[string]string{
+	"Dockerfile": "dockerfile",
+	"C#":         "csharp",
+	"Shell":      "bash",
+}
+
+// guessLang detects a language from pasted content. Order: fast decisive
+// paths (empty, JSON, unambiguous markers enry can't see without a filename),
+// then enry strategies (shebangs, XML decl, modelines, content heuristics),
+// then enry's classifier seeded by our regex hints.
+// GuessLang detects a language from pasted content.
+func GuessLang(s string) string {
+	src := strings.TrimSpace(s)
+	if src == "" {
+		return ""
+	}
+
+	// JSON: must start with { or [ and parse — cheaper and more decisive
+	// than the classifier for pasted JSON, and handles compact single-line
+	// JSON that content heuristics miss.
+	if src[0] == '{' || src[0] == '[' {
+		var v any
+		if json.Unmarshal([]byte(src), &v) == nil {
+			return "json"
+		}
+	}
+
+	// enry's built-in strategies: shebangs, XML declaration, modelines,
+	// content heuristics.
+	if lang := enry.GetLanguage("", []byte(src)); lang != "" && lang != enry.OtherLanguage {
+		return normalizeLang(lang)
+	}
+
+	// collect hint-matched languages as classifier candidates
+	cands := []string{}
+	for _, h := range hintRules {
+		if h.re.MatchString(src) {
+			cands = append(cands, h.lang)
+		}
+	}
+	if len(cands) > 0 {
+		// enry's Bayesian classifier (trained on Linguist samples) picks the
+		// best of the hint candidates; fall back to the first hint if it
+		// can't decide.
+		if lang, _ := enry.GetLanguageByClassifier([]byte(src), cands); lang != "" {
+			return normalizeLang(lang)
+		}
+		return normalizeLang(cands[0])
+	}
+
+	return "text"
+}
+
+// normalizeLang maps enry display names to our lowercase stored ids.
+func normalizeLang(lang string) string {
+	if c, ok := canonical[lang]; ok {
+		return c
+	}
+	return strings.ToLower(lang)
+}
+
@@ -0,0 +1,92 @@
+package lang
+
+import (
+	"strings"
+	"testing"
+)
+
+// TestGuessLangExisting covers languages detected before the enry switch and
+// still expected to work after it.
+func TestGuessLangExisting(t *testing.T) {
+	cases := map[string]string{
+		"package main\n\nfunc main() {}\n":                                    "go",
+		"def foo():\n    return 1\n":                                          "python",
+		"const x = 1;\nconsole.log(x);\n":                                     "javascript",
+		"{\"a\": 1, \"b\": [2, 3]}\n":                                           "json",
+		"hello world just some text":                                          "text",
+		"":                                                                    "",
+		"fn main() {\n    let x = 1;\n}\n":                                    "rust",
+		"#include <stdio.h>\nint main() { printf(\"hi\"); }\n":                "c",
+		"SELECT id, name FROM users WHERE active = 1;\n":                      "sql",
+		"title: demo\nitems:\n  - one\n  - two\n":                             "yaml",
+		"# Demo\n\nsome *markdown* text with a [link](http://x)\n":            "markdown",
+		"#!/bin/bash\nset -euo pipefail\necho hi\n":                           "bash",
+	}
+	for src, want := range cases {
+		if got := GuessLang(src); got != want {
+			t.Errorf("GuessLang(%q) = %q, want %q", src, got, want)
+		}
+	}
+}
+
+// TestGuessLangNewLanguages covers the languages added to the dropdown as part
+// of the enry integration (#41).
+func TestGuessLangNewLanguages(t *testing.T) {
+	cases := map[string]string{
+		"interface User {\n  name: string;\n  age: number;\n}\n":                    "typescript",
+		"<!DOCTYPE html>\n<html>\n<head><title>hi</title></head>\n</html>\n":        "html",
+		".container {\n  display: flex;\n  padding: 4px;\n}\n":                      "css",
+		"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<root><item>x</item></root>\n": "xml",
+		"<?php\nfunction hi() { echo 'x'; }\n":                                      "php",
+		"def greet(name)\n  puts \"hi #{name}\"\nend\n":                             "ruby",
+		"use strict;\nmy $x = 5;\nprint \"x is $x\\n\";\n":                          "perl",
+		"local x = 10\nfunction add(a, b)\n  return a + b\nend\n":                   "lua",
+		"FROM golang:1.22\nRUN go build -o app .\nCMD [\"./app\"]\n":                "dockerfile",
+		"[package]\nname = \"demo\"\nversion = \"0.1.0\"\n":                         "toml",
+		"[server]\nhost = 127.0.0.1\nport = 8080\n":                                 "ini",
+		"diff --git a/main.go b/main.go\n--- a/main.go\n+++ b/main.go\n@@ -1 +1 @@\n": "diff",
+	}
+	for src, want := range cases {
+		if got := GuessLang(src); got != want {
+			t.Errorf("GuessLang(%q) = %q, want %q", src, got, want)
+		}
+	}
+}
+
+// TestGuessLangMagicMarkers verifies enry's built-in shebang / signature
+// handling that replaced the hand-rolled magic-marker pre-checks (#41).
+func TestGuessLangMagicMarkers(t *testing.T) {
+	cases := map[string]string{
+		"#!/usr/bin/env node\nconsole.log('hi');\n":             "javascript",
+		"#!/usr/bin/env python3\nimport sys\nprint(sys.argv)\n": "python",
+		"#!/usr/bin/python\nprint('x')\n":                       "python",
+		"#!/bin/bash\nset -euo pipefail\necho hi\n":             "bash",
+		"#!/bin/sh\necho hi\n":                                  "bash",
+		"<?php\necho 'x';\n":                                    "php",
+		"<!DOCTYPE html>\n<html><body></body></html>":           "html",
+		"FROM alpine:3.19\nCOPY app /app\n":                     "dockerfile",
+		"FROM ubuntu:24.04\nRUN apt-get update\n":               "dockerfile",
+		"diff --git a/x.txt b/x.txt\nindex 123..456 100644\n":   "diff",
+		"--- a/config.yml\n+++ b/config.yml\n@@ -1,2 +1,3 @@\n": "diff",
+	}
+	for src, want := range cases {
+		if got := GuessLang(src); got != want {
+			t.Errorf("GuessLang(%q) = %q, want %q", src, got, want)
+		}
+	}
+}
+
+// TestGuessLangCanonical verifies enry display names are mapped/lowercased to
+// our stored ids.
+func TestGuessLangCanonical(t *testing.T) {
+	if got := GuessLang("FROM debian:12\nCMD [\"sh\"]\n"); got != "dockerfile" {
+		t.Errorf("Dockerfile canonical mapping failed: got %q", got)
+	}
+	if got := GuessLang("#!/bin/sh\necho hi\n"); got != "bash" {
+		t.Errorf("Shell canonical mapping failed: got %q", got)
+	}
+	// uncurated languages still come back lowercase
+	if got := GuessLang("<h1>{{.Name}}</h1>\n"); got != strings.ToLower(got) {
+		t.Errorf("expected lowercase output, got %q", got)
+	}
+}
@@ -0,0 +1,150 @@
+package lang
+
+import (
+	"html"
+	"regexp"
+	"strings"
+)
+
+// Minimal regex-based syntax highlighter for the paste view (#1).
+// Server-side, no external dependencies. Tokens: comments, strings,
+// numbers, keywords. Output is HTML with span classes styled in app.css.
+// Highlighting is applied per line so the gutter stays line-aligned.
+
+type hlLang struct {
+	keywords  map[string]bool
+	lineComps []string // line comment prefixes
+	blockCom  [2]string
+}
+
+var hlLangs = map[string]hlLang{
+	"go": {
+		keywords: set("break case chan const continue default defer else fallthrough for func go goto if import interface map package range return select struct switch type var nil true false string int int64 int32 uint byte rune bool float64 float32 error make new len cap append panic recover"),
+		lineComps: []string{"//"},
+		blockCom: [2]string{"/*", "*/"},
+	},
+	"python": {
+		keywords: set("and as assert async await break class continue def del elif else except False finally for from global if import in is lambda None nonlocal not or pass raise return True try while with yield self print len range str int float list dict set tuple open"),
+		lineComps: []string{"#"},
+	},
+	"javascript": {
+		keywords: set("async await break case catch class const continue debugger default delete do else export extends finally for function if import in instanceof let new null of return static super switch this throw true false try typeof undefined var void while with yield console log document window Math JSON Array Object String Number Boolean Promise"),
+		lineComps: []string{"//"},
+		blockCom: [2]string{"/*", "*/"},
+	},
+	"json": {
+		keywords: set("true false null"),
+	},
+	"bash": {
+		keywords: set("if then else elif fi for while do done case esac function return exit local export echo cd ls grep awk sed cat curl sudo apt git make echo read shift set unset trap source alias printf test rm mv cp mkdir chmod chown"),
+		lineComps: []string{"#"},
+	},
+	"sql": {
+		keywords: set("SELECT FROM WHERE INSERT INTO VALUES UPDATE SET DELETE CREATE TABLE DROP ALTER INDEX JOIN LEFT RIGHT INNER OUTER ON GROUP BY ORDER HAVING LIMIT OFFSET AND OR NOT NULL IS IN AS DISTINCT UNION ALL PRIMARY KEY FOREIGN REFERENCES DEFAULT UNIQUE CHECK VIEW WITH RETURNING EXISTS CASE WHEN THEN ELSE END COUNT SUM AVG MIN MAX"),
+		lineComps: []string{"--"},
+		blockCom: [2]string{"/*", "*/"},
+	},
+}
+
+// aliases from the language dropdown / guesser
+var hlAliases = map[string]string{
+	"py": "python", "python3": "python",
+	"js": "javascript", "node": "javascript", "typescript": "javascript", "ts": "javascript",
+	"sh": "bash", "shell": "bash", "zsh": "bash",
+	"golang": "go",
+	"c": "go", "cpp": "go", "c++": "go", "java": "go", "rust": "go", "rs": "go",
+	// C-family shares the same token rules as Go for highlighting purposes
+}
+
+func set(words string) map[string]bool {
+	m := make(map[string]bool)
+	for _, w := range strings.Fields(words) {
+		m[w] = true
+	}
+	return m
+}
+
+func resolveLang(lang string) (string, hlLang, bool) {
+	l := strings.ToLower(strings.TrimSpace(lang))
+	if l == "" || l == "text" || l == "markdown" || l == "yaml" {
+		return "", hlLang{}, false
+	}
+	if l == "yml" {
+		return "", hlLang{}, false
+	}
+	if g, ok := hlAliases[l]; ok {
+		if h, ok2 := hlLangs[g]; ok2 {
+			return g, h, true
+		}
+		return "", hlLang{}, false
+	}
+	h, ok := hlLangs[l]
+	return l, h, ok
+}
+
+var hlTokenRe = regexp.MustCompile(`("(?:[^"\\]|\\.)*"?|'(?:[^'\\]|\\.)*'?|` + "`" + `[^` + "`" + `]*` + "`" + `?|//[^\n]*|--[^\n]*|#[^\n]*|/\*.*?(?:\*/|$)|\b(?:[0-9]+\.?[0-9]*|0x[0-9a-fA-F]+)\b|[A-Za-z_][A-Za-z0-9_]*)`)
+
+func highlightLine(line string, h hlLang, lang string) string {
+	var b strings.Builder
+	rest := line
+	// strip a trailing block-comment opener handled below; regex covers it
+	for {
+		loc := hlTokenRe.FindStringIndex(rest)
+		if loc == nil {
+			b.WriteString(html.EscapeString(rest))
+			break
+		}
+		b.WriteString(html.EscapeString(rest[:loc[0]]))
+		tok := rest[loc[0]:loc[1]]
+		cls := ""
+		switch {
+		case strings.HasPrefix(tok, "//") || strings.HasPrefix(tok, "#") ||
+			strings.HasPrefix(tok, "--") || strings.HasPrefix(tok, "/*"):
+			// '#/--' are comments only in langs that use them
+			if (strings.HasPrefix(tok, "#") && !containsStr(h.lineComps, "#")) ||
+				(strings.HasPrefix(tok, "--") && !containsStr(h.lineComps, "--")) {
+				cls = ""
+			} else {
+				cls = "tok-com"
+			}
+		case strings.HasPrefix(tok, "\"") || strings.HasPrefix(tok, "'") || strings.HasPrefix(tok, "`"):
+			cls = "tok-str"
+		case tok[0] >= '0' && tok[0] <= '9':
+			cls = "tok-num"
+		case h.keywords[tok]:
+			cls = "tok-kw"
+		}
+		if cls != "" {
+			b.WriteString(`<span class="` + cls + `">` + html.EscapeString(tok) + `</span>`)
+		} else {
+			b.WriteString(html.EscapeString(tok))
+		}
+		rest = rest[loc[1]:]
+	}
+	return b.String()
+}
+
+func containsStr(list []string, s string) bool {
+	for _, v := range list {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}
+
+// highlightCode returns HTML with highlighting spans; safe because all
+// non-token text is html-escaped.
+func HighlightCode(content, langID string) string {
+	l, h, ok := resolveLang(langID)
+	_ = l
+	if !ok {
+		return html.EscapeString(content)
+	}
+	lines := strings.Split(content, "\n")
+	out := make([]string, len(lines))
+	for i, line := range lines {
+		out[i] = highlightLine(line, h, langID)
+	}
+	return strings.Join(out, "\n")
+}