diff --git a/go.mod b/go.mod index 6b2bf74..c39271b 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,14 @@ go 1.27.1 require ( github.com/go-chi/chi/v5 v5.3.2 + github.com/go-enry/go-enry/v2 v2.9.6 golang.org/x/crypto v0.39.0 modernc.org/sqlite v1.58.0 ) require ( github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-enry/go-oniguruma v1.2.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect diff --git a/go.sum b/go.sum index 62c8c10..cdb466c 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,14 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY= github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= +github.com/go-enry/go-enry/v2 v2.9.6 h1:np63eOtMV56zfYDHnFVgpEVOk8fr2kmylcMnAZUDbSs= +github.com/go-enry/go-enry/v2 v2.9.6/go.mod h1:9yrj4ES1YrbNb1Wb7/PWYr2bpaCXUGRt0uafN0ISyG8= +github.com/go-enry/go-oniguruma v1.2.1 h1:k8aAMuJfMrqm/56SG2lV9Cfti6tC4x8673aHCcBk+eo= +github.com/go-enry/go-oniguruma v1.2.1/go.mod h1:bWDhYP+S6xZQgiRL7wlTScFYBe023B6ilRZbCAD5Hf4= github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -12,8 +19,17 @@ github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsRe github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= @@ -24,6 +40,11 @@ golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= diff --git a/guess.go b/guess.go index 7d95434..d001070 100644 --- a/guess.go +++ b/guess.go @@ -5,16 +5,68 @@ import ( "net/http" "regexp" "strings" + + "github.com/go-enry/go-enry/v2" ) -// guessLang heuristically detects a language from source content. +// hintRule is a lightweight regex hint that nudges detection. Hints don't +// decide on their own: matching hints are passed to enry's classifier as +// candidate languages, and enry (trained on Linguist samples) makes the final +// call. Adding a language later is usually a one-line addition here plus the +// dropdown in web/templates/new.html. +type hintRule struct { + lang string // enry language name + re *regexp.Regexp +} + +// hintRules are evaluated in order; keep more specific languages earlier so +// ties break in their favor. +var hintRules = []hintRule{ + {"Dockerfile", regexp.MustCompile(`(?mi)^(FROM\s+\S+:\S*|RUN\s+\S+|COPY\s+\S+\s+\S+|ENTRYPOINT\s+|WORKDIR\s+/)`)}, + {"Diff", regexp.MustCompile(`(?m)^(diff --git|--- a/|\+\+\+ b/|@@ -\d+)`)}, + {"PHP", regexp.MustCompile(`(?m)<\?php|\$\w+\s*=\s*[^=]|\becho\s+["'$]`)}, + {"HTML", regexp.MustCompile(`(?i)<(!DOCTYPE|html|head|body|div|span|script|p|a)\b`)}, + {"XML", regexp.MustCompile(`(?m)^<\?xml\b|<\/?[a-zA-Z][\w.-]*:[\w.-]*[>\s]`)}, + {"CSS", regexp.MustCompile(`(?m)(^|\})\s*[^{}@]+\{[^}]*:[^}]*\}|@(media|import|font-face)\b`)}, + {"TypeScript", regexp.MustCompile(`(?m)(:\s*(string|number|boolean|any)\b|\binterface \w+ \{|\btype \w+ =|\bimplements \w+)`)}, + {"TOML", regexp.MustCompile(`(?m)^\[[\w."-]+\]\s*$|^\w[\w-]*\s*=\s*("[^"]*"|\d+|true|false|\[[^\]]*\])\s*$`)}, + {"INI", regexp.MustCompile(`(?m)^\[[\w.-]+\]\s*$|^\w[\w.-]*\s*=\s*\S+\s*$`)}, + {"Ruby", regexp.MustCompile(`(?m)(\bdef \w+($|\s)|\brequire ['"]|\bputs \w+|@\w+\s*=\s*[^=]|\bend\b\s*$)`)}, + {"Perl", regexp.MustCompile(`(?m)(\buse strict\b|\bmy \$\w+|->\{|sub \w+ \{)`)}, + {"Lua", regexp.MustCompile(`(?m)(\bfunction\s+\w+\s*\(|\blocal \w+\s*=|\bthen\b|\belseif\b|\.\.\.)`)}, + {"Go", regexp.MustCompile(`(?m)^\s*(package \w+|import \(|func (\w+|\() )`)}, + {"Python", regexp.MustCompile(`(?m)^\s*(def \w+|class \w+|import \w+|from \w+ import |@\w+)`)}, + {"JavaScript", regexp.MustCompile(`(?m)(\bconst \w+ = |require\(|import \w+ from |=> \{|\bconsole\.log\()` )}, + {"Rust", regexp.MustCompile(`(?m)(\bfn \w+|let mut \b|\bimpl \b|use std::)`)}, + {"Java", regexp.MustCompile(`(?m)(\bpublic (static |final |class )|\bSystem\.out\.print|import java\.)`)}, + {"C", regexp.MustCompile(`(?m)(#include\s*<\w+\.h>|printf\(|\bint main\()` )}, + {"C++", regexp.MustCompile(`(?m)(#include\s*<(iostream|vector|string)>|std::|\bcout\s*<<)`)}, + {"SQL", regexp.MustCompile(`(?i)\b(SELECT .+ FROM|INSERT INTO|CREATE TABLE|UPDATE \w+ SET)\b`)}, + {"YAML", regexp.MustCompile(`(?m)^(\w[\w-]*:\s*(\||\S)| \w[\w-]*: |---\s*$)`)}, + {"Markdown", regexp.MustCompile("(?m)^(#{1,6} \\S|\\|.*\\||-\\s\\[\\s?\\]|```)")}, + {"Shell", regexp.MustCompile(`(?m)^(#!.*bash|#!.*sh|\w+\(\)\s*\{)` )}, +} + +// canonical maps enry display names to the lowercase ids we store and render. +var canonical = map[string]string{ + "Dockerfile": "dockerfile", + "C#": "csharp", + "Shell": "bash", +} + +// guessLang detects a language from pasted content. Order: fast decisive +// paths (empty, JSON, unambiguous markers enry can't see without a filename), +// then enry strategies (shebangs, XML decl, modelines, content heuristics), +// then enry's classifier seeded by our regex hints. func guessLang(s string) string { src := strings.TrimSpace(s) if src == "" { return "" } - // JSON: must start with { or [ and parse + // JSON: must start with { or [ and parse — cheaper and more decisive + // than the classifier for pasted JSON, and handles compact single-line + // JSON that content heuristics miss. if src[0] == '{' || src[0] == '[' { var v any if json.Unmarshal([]byte(src), &v) == nil { @@ -22,43 +74,38 @@ func guessLang(s string) string { } } - rules := []struct { - lang string - re *regexp.Regexp - }{ - {"python", regexp.MustCompile(`(?m)^\s*(def |class |import |from \w+ import |@decorator)`)}, - {"python", regexp.MustCompile(`(^|\n)\s*#!.*python`)}, - {"go", regexp.MustCompile(`(?m)^\s*(package \w+|import \(|func (\w+|\() )`)}, - {"rust", regexp.MustCompile(`(?m)(\bfn \w+|let mut \b|\bimpl \b|\bmatch \w+ \{|use std::)`)}, - {"javascript", regexp.MustCompile(`(?m)(\bconst \w+ = |require\(|import \w+ from |=> \{|\bconsole\.log\()` )}, - {"java", regexp.MustCompile(`(?m)(\bpublic (static |final |class )|\bSystem\.out\.print|import java\.)`)}, - {"c", regexp.MustCompile(`(?m)(#include\s*<\w+\.h>|printf\(|\bint main\()` )}, - {"cpp", regexp.MustCompile(`(?m)(#include\s*<(iostream|vector|string)>|std::|\bcout\s*<<)`)}, - {"bash", regexp.MustCompile(`(?m)^(#!.*bash|#!.*sh|\w+\(\)\s*\{)` )}, - {"sql", regexp.MustCompile(`(?i)\b(SELECT .+ FROM|INSERT INTO|CREATE TABLE|UPDATE \w+ SET)\b`)}, - {"yaml", regexp.MustCompile(`(?m)^(\w[\w-]*:\s*(\||\S)| \w[\w-]*: |---\s*$)`)}, - {"markdown", regexp.MustCompile("(?m)^(#{1,6} \\S|\\|.*\\||-\\s\\[\\s?\\]|```)")}, + // enry's built-in strategies: shebangs, XML declaration, modelines, + // content heuristics. + if lang := enry.GetLanguage("", []byte(src)); lang != "" && lang != enry.OtherLanguage { + return normalizeLang(lang) } - scores := map[string]int{} - for _, r := range rules { - n := len(r.re.FindAllString(src, -1)) - if n > 0 { - // later generic rules shouldn't drown out earlier specific ones - scores[r.lang] += n + // collect hint-matched languages as classifier candidates + cands := []string{} + for _, h := range hintRules { + if h.re.MatchString(src) { + cands = append(cands, h.lang) } } + if len(cands) > 0 { + // enry's Bayesian classifier (trained on Linguist samples) picks the + // best of the hint candidates; fall back to the first hint if it + // can't decide. + if lang, _ := enry.GetLanguageByClassifier([]byte(src), cands); lang != "" { + return normalizeLang(lang) + } + return normalizeLang(cands[0]) + } - best, bestN := "", 0 - for lang, n := range scores { - if n > bestN { - best, bestN = lang, n - } + return "text" +} + +// normalizeLang maps enry display names to our lowercase stored ids. +func normalizeLang(lang string) string { + if c, ok := canonical[lang]; ok { + return c } - if bestN == 0 { - return "text" - } - return best + return strings.ToLower(lang) } func (a *apiServer) handleGuessLang(w http.ResponseWriter, r *http.Request) { diff --git a/guess_test.go b/guess_test.go new file mode 100644 index 0000000..b20d853 --- /dev/null +++ b/guess_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "strings" + "testing" +) + +// TestGuessLangExisting covers languages detected before the enry switch and +// still expected to work after it. +func TestGuessLangExisting(t *testing.T) { + cases := map[string]string{ + "package main\n\nfunc main() {}\n": "go", + "def foo():\n return 1\n": "python", + "const x = 1;\nconsole.log(x);\n": "javascript", + "{\"a\": 1, \"b\": [2, 3]}\n": "json", + "hello world just some text": "text", + "": "", + "fn main() {\n let x = 1;\n}\n": "rust", + "#include \nint main() { printf(\"hi\"); }\n": "c", + "SELECT id, name FROM users WHERE active = 1;\n": "sql", + "title: demo\nitems:\n - one\n - two\n": "yaml", + "# Demo\n\nsome *markdown* text with a [link](http://x)\n": "markdown", + "#!/bin/bash\nset -euo pipefail\necho hi\n": "bash", + } + for src, want := range cases { + if got := guessLang(src); got != want { + t.Errorf("guessLang(%q) = %q, want %q", src, got, want) + } + } +} + +// TestGuessLangNewLanguages covers the languages added to the dropdown as part +// of the enry integration (#41). +func TestGuessLangNewLanguages(t *testing.T) { + cases := map[string]string{ + "interface User {\n name: string;\n age: number;\n}\n": "typescript", + "\n\nhi\n\n": "html", + ".container {\n display: flex;\n padding: 4px;\n}\n": "css", + "\nx\n": "xml", + "\n": "html", + "FROM alpine:3.19\nCOPY app /app\n": "dockerfile", + "FROM ubuntu:24.04\nRUN apt-get update\n": "dockerfile", + "diff --git a/x.txt b/x.txt\nindex 123..456 100644\n": "diff", + "--- a/config.yml\n+++ b/config.yml\n@@ -1,2 +1,3 @@\n": "diff", + } + for src, want := range cases { + if got := guessLang(src); got != want { + t.Errorf("guessLang(%q) = %q, want %q", src, got, want) + } + } +} + +// TestGuessLangCanonical verifies enry display names are mapped/lowercased to +// our stored ids. +func TestGuessLangCanonical(t *testing.T) { + if got := guessLang("FROM debian:12\nCMD [\"sh\"]\n"); got != "dockerfile" { + t.Errorf("Dockerfile canonical mapping failed: got %q", got) + } + if got := guessLang("#!/bin/sh\necho hi\n"); got != "bash" { + t.Errorf("Shell canonical mapping failed: got %q", got) + } + // uncurated languages still come back lowercase + if got := guessLang("

{{.Name}}

\n"); got != strings.ToLower(got) { + t.Errorf("expected lowercase output, got %q", got) + } +} diff --git a/palette b/palette index 492b385..8219986 100755 Binary files a/palette and b/palette differ diff --git a/web/templates/new.html b/web/templates/new.html index 841fc21..6db5a3b 100644 --- a/web/templates/new.html +++ b/web/templates/new.html @@ -7,9 +7,12 @@ @@ -97,9 +100,12 @@ function showResult(html, isError) { } function defaultFilename(lang) { const names = { - python: 'Python.py', go: 'main.go', javascript: 'script.js', rust: 'main.rs', - c: 'main.c', cpp: 'main.cpp', java: 'Main.java', bash: 'script.sh', - sql: 'query.sql', yaml: 'config.yaml', json: 'data.json', + python: 'Python.py', go: 'main.go', javascript: 'script.js', typescript: 'index.ts', + rust: 'main.rs', c: 'main.c', cpp: 'main.cpp', java: 'Main.java', bash: 'script.sh', + sql: 'query.sql', yaml: 'config.yaml', json: 'data.json', html: 'index.html', + css: 'style.css', xml: 'doc.xml', php: 'index.php', ruby: 'main.rb', + perl: 'main.pl', lua: 'main.lua', dockerfile: 'Dockerfile', toml: 'config.toml', + ini: 'config.ini', diff: 'changes.diff', markdown: 'notes.md', text: 'Text.txt', }; return names[lang] || '';