poslop/palette
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Language detection: switch to go-enry with hint-seeded classifier (#41)
CI / test (push) Successful in 35s
Details
CI / docker (push) Skipped
Details

Browse Source 
- Add github.com/go-enry/go-enry/v2 dependency to go.mod
- guessLang: JSON fast path, then enry strategies (shebangs, XML, modelines,
  content heuristics), then enry's Linguist-trained Bayesian classifier over
  regex-hint candidate languages; curated dropdown expanded with typescript,
  html, css, xml, php, ruby, perl, lua, dockerfile, toml, ini, diff, csharp
- enry display names normalized to lowercase stored ids (Dockerfile->dockerfile,
  Shell->bash, C#->csharp); unsupported languages render unhighlighted
- Tests: existing languages, all new languages, magic markers (enry shebang/
  DOCTYPE/FROM/diff handling), canonical mapping
This commit is contained in:
poslop
2026-09-08 21:25:36 -05:00
parent 7410b5c9cf
commit efa8c7145c
6 changed files with 206 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+2
View File
@@ -4,12 +4,14 @@ go 1.27.1
require (
github.com/go-chi/chi/v5 v5.3.2
github.com/go-enry/go-enry/v2 v2.9.6
golang.org/x/crypto v0.39.0
modernc.org/sqlite v1.58.0
)
require (
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/go-enry/go-oniguruma v1.2.1 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/mattn/go-isatty v0.0.24 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
go.sum
+21
View File
@@ -1,7 +1,14 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY=
github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
github.com/go-enry/go-enry/v2 v2.9.6 h1:np63eOtMV56zfYDHnFVgpEVOk8fr2kmylcMnAZUDbSs=
github.com/go-enry/go-enry/v2 v2.9.6/go.mod h1:9yrj4ES1YrbNb1Wb7/PWYr2bpaCXUGRt0uafN0ISyG8=
github.com/go-enry/go-oniguruma v1.2.1 h1:k8aAMuJfMrqm/56SG2lV9Cfti6tC4x8673aHCcBk+eo=
github.com/go-enry/go-oniguruma v1.2.1/go.mod h1:bWDhYP+S6xZQgiRL7wlTScFYBe023B6ilRZbCAD5Hf4=
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@@ -12,8 +19,17 @@ github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsRe
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
@@ -24,6 +40,11 @@ golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=
golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8=
modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w=
guess.go
+80 -33
View File
@@ -5,16 +5,68 @@ import (
"net/http"
"regexp"
"strings"
"github.com/go-enry/go-enry/v2"
)
// guessLang heuristically detects a language from source content.
// hintRule is a lightweight regex hint that nudges detection. Hints don't
// decide on their own: matching hints are passed to enry's classifier as
// candidate languages, and enry (trained on Linguist samples) makes the final
// call. Adding a language later is usually a one-line addition here plus the
// dropdown in web/templates/new.html.
type hintRule struct {
lang string // enry language name
re *regexp.Regexp
}
// hintRules are evaluated in order; keep more specific languages earlier so
// ties break in their favor.
var hintRules = []hintRule{
{"Dockerfile", regexp.MustCompile(`(?mi)^(FROM\s+\S+:\S*|RUN\s+\S+|COPY\s+\S+\s+\S+|ENTRYPOINT\s+|WORKDIR\s+/)`)},
{"Diff", regexp.MustCompile(`(?m)^(diff --git|--- a/|\+\+\+ b/|@@ -\d+)`)},
{"PHP", regexp.MustCompile(`(?m)<\?php|\$\w+\s*=\s*[^=]|\becho\s+["'$]`)},
{"HTML", regexp.MustCompile(`(?i)<(!DOCTYPE|html|head|body|div|span|script|p|a)\b`)},
{"XML", regexp.MustCompile(`(?m)^<\?xml\b|<\/?[a-zA-Z][\w.-]*:[\w.-]*[>\s]`)},
{"CSS", regexp.MustCompile(`(?m)(^|\})\s*[^{}@]+\{[^}]*:[^}]*\}|@(media|import|font-face)\b`)},
{"TypeScript", regexp.MustCompile(`(?m)(:\s*(string|number|boolean|any)\b|\binterface \w+ \{|\btype \w+ =|\bimplements \w+)`)},
{"TOML", regexp.MustCompile(`(?m)^\[[\w."-]+\]\s*$|^\w[\w-]*\s*=\s*("[^"]*"|\d+|true|false|\[[^\]]*\])\s*$`)},
{"INI", regexp.MustCompile(`(?m)^\[[\w.-]+\]\s*$|^\w[\w.-]*\s*=\s*\S+\s*$`)},
{"Ruby", regexp.MustCompile(`(?m)(\bdef \w+($|\s)|\brequire ['"]|\bputs \w+|@\w+\s*=\s*[^=]|\bend\b\s*$)`)},
{"Perl", regexp.MustCompile(`(?m)(\buse strict\b|\bmy \$\w+|->\{|sub \w+ \{)`)},
{"Lua", regexp.MustCompile(`(?m)(\bfunction\s+\w+\s*\(|\blocal \w+\s*=|\bthen\b|\belseif\b|\.\.\.)`)},
{"Go", regexp.MustCompile(`(?m)^\s*(package \w+|import \(|func (\w+|\() )`)},
{"Python", regexp.MustCompile(`(?m)^\s*(def \w+|class \w+|import \w+|from \w+ import |@\w+)`)},
{"JavaScript", regexp.MustCompile(`(?m)(\bconst \w+ = |require\(|import \w+ from |=> \{|\bconsole\.log\()` )},
{"Rust", regexp.MustCompile(`(?m)(\bfn \w+|let mut \b|\bimpl \b|use std::)`)},
{"Java", regexp.MustCompile(`(?m)(\bpublic (static |final |class )|\bSystem\.out\.print|import java\.)`)},
{"C", regexp.MustCompile(`(?m)(#include\s*<\w+\.h>|printf\(|\bint main\()` )},
{"C++", regexp.MustCompile(`(?m)(#include\s*<(iostream|vector|string)>|std::|\bcout\s*<<)`)},
{"SQL", regexp.MustCompile(`(?i)\b(SELECT .+ FROM|INSERT INTO|CREATE TABLE|UPDATE \w+ SET)\b`)},
{"YAML", regexp.MustCompile(`(?m)^(\w[\w-]*:\s*(\||\S)| \w[\w-]*: |---\s*$)`)},
{"Markdown", regexp.MustCompile("(?m)^(#{1,6} \\S|\\|.*\\||-\\s\\[\\s?\\]|```)")},
{"Shell", regexp.MustCompile(`(?m)^(#!.*bash|#!.*sh|\w+\(\)\s*\{)` )},
}
// canonical maps enry display names to the lowercase ids we store and render.
var canonical = map[string]string{
"Dockerfile": "dockerfile",
"C#": "csharp",
"Shell": "bash",
}
// guessLang detects a language from pasted content. Order: fast decisive
// paths (empty, JSON, unambiguous markers enry can't see without a filename),
// then enry strategies (shebangs, XML decl, modelines, content heuristics),
// then enry's classifier seeded by our regex hints.
func guessLang(s string) string {
src := strings.TrimSpace(s)
if src == "" {
return ""
}
// JSON: must start with { or [ and parse
// JSON: must start with { or [ and parse — cheaper and more decisive
// than the classifier for pasted JSON, and handles compact single-line
// JSON that content heuristics miss.
if src[0] == '{' || src[0] == '[' {
var v any
if json.Unmarshal([]byte(src), &v) == nil {
@@ -22,43 +74,38 @@ func guessLang(s string) string {
}
}
rules := []struct {
lang string
re *regexp.Regexp
}{
{"python", regexp.MustCompile(`(?m)^\s*(def |class |import |from \w+ import |@decorator)`)},
{"python", regexp.MustCompile(`(^|\n)\s*#!.*python`)},
{"go", regexp.MustCompile(`(?m)^\s*(package \w+|import \(|func (\w+|\() )`)},
{"rust", regexp.MustCompile(`(?m)(\bfn \w+|let mut \b|\bimpl \b|\bmatch \w+ \{|use std::)`)},
{"javascript", regexp.MustCompile(`(?m)(\bconst \w+ = |require\(|import \w+ from |=> \{|\bconsole\.log\()` )},
{"java", regexp.MustCompile(`(?m)(\bpublic (static |final |class )|\bSystem\.out\.print|import java\.)`)},
{"c", regexp.MustCompile(`(?m)(#include\s*<\w+\.h>|printf\(|\bint main\()` )},
{"cpp", regexp.MustCompile(`(?m)(#include\s*<(iostream|vector|string)>|std::|\bcout\s*<<)`)},
{"bash", regexp.MustCompile(`(?m)^(#!.*bash|#!.*sh|\w+\(\)\s*\{)` )},
{"sql", regexp.MustCompile(`(?i)\b(SELECT .+ FROM|INSERT INTO|CREATE TABLE|UPDATE \w+ SET)\b`)},
{"yaml", regexp.MustCompile(`(?m)^(\w[\w-]*:\s*(\||\S)| \w[\w-]*: |---\s*$)`)},
{"markdown", regexp.MustCompile("(?m)^(#{1,6} \\S|\\|.*\\||-\\s\\[\\s?\\]|```)")},
// enry's built-in strategies: shebangs, XML declaration, modelines,
// content heuristics.
if lang := enry.GetLanguage("", []byte(src)); lang != "" && lang != enry.OtherLanguage {
return normalizeLang(lang)
}
scores := map[string]int{}
for _, r := range rules {
n := len(r.re.FindAllString(src, -1))
if n > 0 {
// later generic rules shouldn't drown out earlier specific ones
scores[r.lang] += n
// collect hint-matched languages as classifier candidates
cands := []string{}
for _, h := range hintRules {
if h.re.MatchString(src) {
cands = append(cands, h.lang)
}
}
if len(cands) > 0 {
// enry's Bayesian classifier (trained on Linguist samples) picks the
// best of the hint candidates; fall back to the first hint if it
// can't decide.
if lang, _ := enry.GetLanguageByClassifier([]byte(src), cands); lang != "" {
return normalizeLang(lang)
}
return normalizeLang(cands[0])
}
best, bestN := "", 0
for lang, n := range scores {
if n > bestN {
best, bestN = lang, n
}
return "text"
}
// normalizeLang maps enry display names to our lowercase stored ids.
func normalizeLang(lang string) string {
if c, ok := canonical[lang]; ok {
return c
}
if bestN == 0 {
return "text"
}
return best
return strings.ToLower(lang)
}
func (a *apiServer) handleGuessLang(w http.ResponseWriter, r *http.Request) {
guess_test.go
+92
View File
@@ -0,0 +1,92 @@
package main
import (
"strings"
"testing"
)
// TestGuessLangExisting covers languages detected before the enry switch and
// still expected to work after it.
func TestGuessLangExisting(t *testing.T) {
cases := map[string]string{
"package main\n\nfunc main() {}\n": "go",
"def foo():\n return 1\n": "python",
"const x = 1;\nconsole.log(x);\n": "javascript",
"{\"a\": 1, \"b\": [2, 3]}\n": "json",
"hello world just some text": "text",
"": "",
"fn main() {\n let x = 1;\n}\n": "rust",
"#include <stdio.h>\nint main() { printf(\"hi\"); }\n": "c",
"SELECT id, name FROM users WHERE active = 1;\n": "sql",
"title: demo\nitems:\n - one\n - two\n": "yaml",
"# Demo\n\nsome *markdown* text with a [link](http://x)\n": "markdown",
"#!/bin/bash\nset -euo pipefail\necho hi\n": "bash",
}
for src, want := range cases {
if got := guessLang(src); got != want {
t.Errorf("guessLang(%q) = %q, want %q", src, got, want)
}
}
}
// TestGuessLangNewLanguages covers the languages added to the dropdown as part
// of the enry integration (#41).
func TestGuessLangNewLanguages(t *testing.T) {
cases := map[string]string{
"interface User {\n name: string;\n age: number;\n}\n": "typescript",
"<!DOCTYPE html>\n<html>\n<head><title>hi</title></head>\n</html>\n": "html",
".container {\n display: flex;\n padding: 4px;\n}\n": "css",
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<root><item>x</item></root>\n": "xml",
"<?php\nfunction hi() { echo 'x'; }\n": "php",
"def greet(name)\n puts \"hi #{name}\"\nend\n": "ruby",
"use strict;\nmy $x = 5;\nprint \"x is $x\\n\";\n": "perl",
"local x = 10\nfunction add(a, b)\n return a + b\nend\n": "lua",
"FROM golang:1.22\nRUN go build -o app .\nCMD [\"./app\"]\n": "dockerfile",
"[package]\nname = \"demo\"\nversion = \"0.1.0\"\n": "toml",
"[server]\nhost = 127.0.0.1\nport = 8080\n": "ini",
"diff --git a/main.go b/main.go\n--- a/main.go\n+++ b/main.go\n@@ -1 +1 @@\n": "diff",
}
for src, want := range cases {
if got := guessLang(src); got != want {
t.Errorf("guessLang(%q) = %q, want %q", src, got, want)
}
}
}
// TestGuessLangMagicMarkers verifies enry's built-in shebang / signature
// handling that replaced the hand-rolled magic-marker pre-checks (#41).
func TestGuessLangMagicMarkers(t *testing.T) {
cases := map[string]string{
"#!/usr/bin/env node\nconsole.log('hi');\n": "javascript",
"#!/usr/bin/env python3\nimport sys\nprint(sys.argv)\n": "python",
"#!/usr/bin/python\nprint('x')\n": "python",
"#!/bin/bash\nset -euo pipefail\necho hi\n": "bash",
"#!/bin/sh\necho hi\n": "bash",
"<?php\necho 'x';\n": "php",
"<!DOCTYPE html>\n<html><body></body></html>": "html",
"FROM alpine:3.19\nCOPY app /app\n": "dockerfile",
"FROM ubuntu:24.04\nRUN apt-get update\n": "dockerfile",
"diff --git a/x.txt b/x.txt\nindex 123..456 100644\n": "diff",
"--- a/config.yml\n+++ b/config.yml\n@@ -1,2 +1,3 @@\n": "diff",
}
for src, want := range cases {
if got := guessLang(src); got != want {
t.Errorf("guessLang(%q) = %q, want %q", src, got, want)
}
}
}
// TestGuessLangCanonical verifies enry display names are mapped/lowercased to
// our stored ids.
func TestGuessLangCanonical(t *testing.T) {
if got := guessLang("FROM debian:12\nCMD [\"sh\"]\n"); got != "dockerfile" {
t.Errorf("Dockerfile canonical mapping failed: got %q", got)
}
if got := guessLang("#!/bin/sh\necho hi\n"); got != "bash" {
t.Errorf("Shell canonical mapping failed: got %q", got)
}
// uncurated languages still come back lowercase
if got := guessLang("<h1>{{.Name}}</h1>\n"); got != strings.ToLower(got) {
t.Errorf("expected lowercase output, got %q", got)
}
}
palette
BIN
View File
Binary file not shown.
web/templates/new.html
+11 -5
View File
@@ -7,9 +7,12 @@
<input id="title" placeholder="Title">
<select id="language">
<option value="">auto</option>
<option>go</option><option>python</option><option>javascript</option>
<option>rust</option><option>c</option><option>cpp</option><option>java</option>
<option>go</option><option>python</option><option>javascript</option><option>typescript</option>
<option>rust</option><option>c</option><option>cpp</option><option>java</option><option>csharp</option>
<option>bash</option><option>sql</option><option>yaml</option><option>json</option>
<option>html</option><option>css</option><option>xml</option><option>php</option>
<option>ruby</option><option>perl</option><option>lua</option><option>dockerfile</option>
<option>toml</option><option>ini</option><option>diff</option>
<option>markdown</option><option>text</option>
</select>
<button class="btn btn-icon" id="reguess" title="Re-detect language" type="button"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.4" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><path d="M21 12a9 9 0 1 1-2.64-6.36"/><polyline points="21 3 21 9 15 9"/></svg></button>
@@ -97,9 +100,12 @@ function showResult(html, isError) {
}
function defaultFilename(lang) {
const names = {
python: 'Python.py', go: 'main.go', javascript: 'script.js', rust: 'main.rs',
c: 'main.c', cpp: 'main.cpp', java: 'Main.java', bash: 'script.sh',
sql: 'query.sql', yaml: 'config.yaml', json: 'data.json',
python: 'Python.py', go: 'main.go', javascript: 'script.js', typescript: 'index.ts',
rust: 'main.rs', c: 'main.c', cpp: 'main.cpp', java: 'Main.java', bash: 'script.sh',
sql: 'query.sql', yaml: 'config.yaml', json: 'data.json', html: 'index.html',
css: 'style.css', xml: 'doc.xml', php: 'index.php', ruby: 'main.rb',
perl: 'main.pl', lua: 'main.lua', dockerfile: 'Dockerfile', toml: 'config.toml',
ini: 'config.ini', diff: 'changes.diff',
markdown: 'notes.md', text: 'Text.txt',
};
return names[lang] || '';