From f4b6ffce91bcf9cd3c4ef85dd1a2b30b043c2d4d Mon Sep 17 00:00:00 2001
From: kj_sh604
Date: Sat, 18 Apr 2026 14:25:31 -0400
Subject: refactor: better concurrency and use base highlight options
---
src/index.html | 324 +------------------------------------------
src/main.js | 219 +++++++++++++++++++++++++++++
src/server.py | 430 +++++++++++++++++++++++++++++----------------------------
3 files changed, 437 insertions(+), 536 deletions(-)
(limited to 'src')
diff --git a/src/index.html b/src/index.html
index 8d89859..f5510a2 100644
--- a/src/index.html
+++ b/src/index.html
@@ -96,329 +96,7 @@
encrypted with mojicrypt
diff --git a/src/main.js b/src/main.js
index 1f1b755..e0a9600 100644
--- a/src/main.js
+++ b/src/main.js
@@ -1,5 +1,223 @@
const FORM_STATE_KEY = "kj-clipboard-form-state-v1";
+const HIGHLIGHTJS_LANGUAGES = [
+ "1c",
+ "abnf",
+ "accesslog",
+ "actionscript",
+ "ada",
+ "angelscript",
+ "apache",
+ "applescript",
+ "arcade",
+ "arduino",
+ "armasm",
+ "xml",
+ "asciidoc",
+ "aspectj",
+ "autohotkey",
+ "autoit",
+ "avrasm",
+ "awk",
+ "axapta",
+ "bash",
+ "basic",
+ "bnf",
+ "brainfuck",
+ "c",
+ "cal",
+ "capnproto",
+ "ceylon",
+ "clean",
+ "clojure",
+ "clojure-repl",
+ "cmake",
+ "coffeescript",
+ "coq",
+ "cos",
+ "cpp",
+ "crmsh",
+ "crystal",
+ "csharp",
+ "csp",
+ "css",
+ "d",
+ "markdown",
+ "dart",
+ "delphi",
+ "diff",
+ "django",
+ "dns",
+ "dockerfile",
+ "dos",
+ "dsconfig",
+ "dts",
+ "dust",
+ "ebnf",
+ "elixir",
+ "elm",
+ "ruby",
+ "erb",
+ "erlang-repl",
+ "erlang",
+ "excel",
+ "fix",
+ "flix",
+ "fortran",
+ "fsharp",
+ "gams",
+ "gauss",
+ "gcode",
+ "gherkin",
+ "glsl",
+ "gml",
+ "go",
+ "golo",
+ "gradle",
+ "graphql",
+ "groovy",
+ "haml",
+ "handlebars",
+ "haskell",
+ "haxe",
+ "hsp",
+ "http",
+ "hy",
+ "inform7",
+ "ini",
+ "irpf90",
+ "isbl",
+ "java",
+ "javascript",
+ "jboss-cli",
+ "json",
+ "julia",
+ "julia-repl",
+ "kotlin",
+ "lasso",
+ "latex",
+ "ldif",
+ "leaf",
+ "less",
+ "lisp",
+ "livecodeserver",
+ "livescript",
+ "llvm",
+ "lsl",
+ "lua",
+ "makefile",
+ "mathematica",
+ "matlab",
+ "maxima",
+ "mel",
+ "mercury",
+ "mipsasm",
+ "mizar",
+ "perl",
+ "mojolicious",
+ "monkey",
+ "moonscript",
+ "n1ql",
+ "nestedtext",
+ "nginx",
+ "nim",
+ "nix",
+ "node-repl",
+ "nsis",
+ "objectivec",
+ "ocaml",
+ "openscad",
+ "oxygene",
+ "parser3",
+ "pf",
+ "pgsql",
+ "php",
+ "php-template",
+ "plaintext",
+ "pony",
+ "powershell",
+ "processing",
+ "profile",
+ "prolog",
+ "properties",
+ "protobuf",
+ "puppet",
+ "purebasic",
+ "python",
+ "python-repl",
+ "q",
+ "qml",
+ "r",
+ "reasonml",
+ "rib",
+ "roboconf",
+ "routeros",
+ "rsl",
+ "ruleslanguage",
+ "rust",
+ "sas",
+ "scala",
+ "scheme",
+ "scilab",
+ "scss",
+ "shell",
+ "smali",
+ "smalltalk",
+ "sml",
+ "sqf",
+ "sql",
+ "stan",
+ "stata",
+ "step21",
+ "stylus",
+ "subunit",
+ "swift",
+ "taggerscript",
+ "yaml",
+ "tap",
+ "tcl",
+ "thrift",
+ "tp",
+ "twig",
+ "typescript",
+ "vala",
+ "vbnet",
+ "vbscript",
+ "vbscript-html",
+ "verilog",
+ "vhdl",
+ "vim",
+ "wasm",
+ "wren",
+ "x86asm",
+ "xl",
+ "xquery",
+ "zephir",
+];
+
+function syncLanguageOptions() {
+ const langSelect = document.getElementById("lang-select");
+ const selected = langSelect.value;
+
+ langSelect.innerHTML = "";
+
+ const autoOption = document.createElement("option");
+ autoOption.value = "";
+ autoOption.textContent = "(auto)";
+ langSelect.appendChild(autoOption);
+
+ for (const language of HIGHLIGHTJS_LANGUAGES) {
+ const option = document.createElement("option");
+ option.value = language;
+ option.textContent = language;
+ langSelect.appendChild(option);
+ }
+
+ if (selected && HIGHLIGHTJS_LANGUAGES.includes(selected)) {
+ langSelect.value = selected;
+ }
+}
+
function saveFormState() {
const state = {
content: document.getElementById("content").value,
@@ -110,6 +328,7 @@ function setStatus(msg) {
document.getElementById("status").textContent = msg;
}
+syncLanguageOptions();
restoreFormState();
document.getElementById("get-link-btn").addEventListener("click", createPaste);
diff --git a/src/server.py b/src/server.py
index bddee31..303c1bf 100644
--- a/src/server.py
+++ b/src/server.py
@@ -7,6 +7,7 @@
import http.server
import json
import ipaddress
+import queue
import re
import secrets
import signal
@@ -52,6 +53,14 @@ SQLITE_SYNCHRONOUS = "NORMAL" # OFF | NORMAL | FULL | EXTRA
# accept short bursts without immediately refusing tcp connections.
HTTP_REQUEST_QUEUE_SIZE = 64
+# single sqlite writer with queue + micro-batching reduces lock contention under load.
+WRITE_QUEUE_MAX_SIZE = 4096
+WRITE_QUEUE_WAIT_SECONDS = 8
+WRITE_QUEUE_ENQUEUE_TIMEOUT_SECONDS = 0.25
+WRITE_BATCH_SIZE = 32
+WRITE_BATCH_MAX_DELAY_MS = 12
+WRITE_WORKER_JOIN_TIMEOUT_SECONDS = 5
+
TRUST_PROXY = False
TRUSTED_PROXY_IPS = {"127.0.0.1", "::1"}
# hsts off by default to avoid breaking plain-http setups.
@@ -63,324 +72,194 @@ ALLOWED_LANGUAGES = {
"abnf",
"accesslog",
"actionscript",
- "as",
"ada",
"angelscript",
- "asc",
"apache",
- "apacheconf",
"applescript",
- "osascript",
"arcade",
"arduino",
- "ino",
"armasm",
- "arm",
+ "xml",
"asciidoc",
- "adoc",
"aspectj",
"autohotkey",
"autoit",
"avrasm",
"awk",
- "mawk",
- "nawk",
- "gawk",
- "ballerina",
- "bal",
+ "axapta",
"bash",
- "sh",
- "zsh",
"basic",
"bnf",
"brainfuck",
- "bf",
"c",
- "h",
- "csharp",
- "cpp",
- "hpp",
- "cc",
- "hh",
- "c++",
- "h++",
- "cxx",
- "hxx",
"cal",
- "cos",
- "cls",
"capnproto",
- "capnp",
+ "ceylon",
+ "clean",
"clojure",
- "clj",
+ "clojure-repl",
"cmake",
- "cmake.in",
"coffeescript",
- "coffee",
- "cson",
- "iced",
"coq",
+ "cos",
+ "cpp",
"crmsh",
- "crm",
- "pcmk",
"crystal",
- "cr",
+ "csharp",
"csp",
"css",
"d",
+ "markdown",
"dart",
- "dpr",
- "dfm",
- "pas",
- "pascal",
+ "delphi",
"diff",
- "patch",
"django",
- "jinja",
"dns",
- "zone",
- "bind",
"dockerfile",
- "docker",
"dos",
- "bat",
- "cmd",
"dsconfig",
"dts",
"dust",
- "dst",
"ebnf",
"elixir",
"elm",
+ "ruby",
+ "erb",
+ "erlang-repl",
"erlang",
- "erl",
"excel",
- "xls",
- "xlsx",
- "extempore",
- "xtlang",
- "xtm",
- "fsharp",
- "fs",
- "fsx",
- "fsi",
- "fsscript",
"fix",
+ "flix",
+ "fsharp",
"fortran",
- "f90",
- "f95",
"gcode",
- "nc",
"gams",
- "gms",
"gauss",
- "gss",
"gherkin",
+ "glsl",
+ "gml",
"go",
- "golang",
"golo",
- "gololang",
"gradle",
"graphql",
- "gql",
"groovy",
"haml",
"handlebars",
- "hbs",
- "html.hbs",
- "html.handlebars",
"haskell",
- "hs",
"haxe",
- "hx",
- "xml",
- "html",
- "xhtml",
- "rss",
- "atom",
- "xjb",
- "xsd",
- "xsl",
- "plist",
- "svg",
+ "hsp",
"http",
- "https",
"hy",
- "hylang",
"inform7",
- "i7",
"ini",
- "toml",
"irpf90",
+ "isbl",
"java",
- "jsp",
"javascript",
- "js",
- "jsx",
+ "jboss-cli",
"json",
- "jsonc",
- "json5",
"julia",
- "jl",
"julia-repl",
"kotlin",
- "kt",
"lasso",
- "lassoscript",
- "tex",
+ "latex",
"ldif",
"leaf",
"less",
"lisp",
"livecodeserver",
"livescript",
+ "llvm",
+ "lsl",
"lua",
- "pluto",
"makefile",
- "mk",
- "mak",
- "make",
- "markdown",
- "md",
- "mkdown",
- "mkd",
"mathematica",
- "mma",
- "wl",
"matlab",
"maxima",
"mel",
"mercury",
- "mips",
"mipsasm",
"mizar",
+ "perl",
"mojolicious",
"monkey",
"moonscript",
- "moon",
"n1ql",
+ "nestedtext",
"nginx",
- "nginxconf",
"nim",
- "nimrod",
"nix",
+ "node-repl",
"nsis",
"objectivec",
- "mm",
- "objc",
- "obj-c",
- "obj-c++",
- "objective-c++",
"ocaml",
- "glsl",
"openscad",
- "scad",
- "ruleslanguage",
"oxygene",
"parser3",
- "perl",
- "pl",
- "pm",
"pf",
- "pf.conf",
+ "pgsql",
"php",
+ "php-template",
"plaintext",
- "txt",
- "text",
"pony",
- "pgsql",
- "postgres",
- "postgresql",
"powershell",
- "ps",
- "ps1",
"processing",
"prolog",
"properties",
- "proto",
"protobuf",
"puppet",
- "pp",
+ "purebasic",
"python",
- "py",
- "gyp",
"profile",
"python-repl",
- "pycon",
- "k",
- "kdb",
+ "q",
"qml",
"r",
"reasonml",
"rib",
+ "roboconf",
+ "routeros",
"rsl",
- "graph",
- "instances",
- "ruby",
- "rb",
- "gemspec",
- "podspec",
- "thor",
- "irb",
+ "ruleslanguage",
"rust",
- "rs",
"sas",
"scala",
"scheme",
"scilab",
- "sci",
"scss",
"shell",
- "console",
"smali",
"smalltalk",
- "st",
"sml",
+ "sqf",
"sql",
"stan",
- "stanfuncs",
"stata",
- "p21",
- "step",
- "stp",
+ "step21",
"stylus",
- "styl",
"subunit",
"swift",
- "tcl",
- "tk",
+ "taggerscript",
+ "yaml",
"tap",
+ "tcl",
"thrift",
"tp",
"twig",
- "craftcms",
"typescript",
- "ts",
- "tsx",
- "mts",
- "cts",
"vala",
"vbnet",
- "vb",
"vbscript",
- "vbs",
+ "vbscript-html",
"verilog",
- "v",
"vhdl",
"vim",
- "axapta",
- "x++",
+ "wasm",
+ "wren",
"x86asm",
"xl",
- "tao",
"xquery",
- "xpath",
- "xq",
- "xqm",
- "yml",
- "yaml",
"zephir",
- "zep",
}
@@ -389,6 +268,10 @@ ALLOWED_LANGUAGES = {
_rate_lock = threading.Lock()
_rate_state = {}
+_write_queue = queue.Queue(maxsize=WRITE_QUEUE_MAX_SIZE)
+_write_worker_thread = None
+_write_worker_stop = threading.Event()
+
# database
@@ -408,6 +291,147 @@ def sqlite_retry_sleep(attempt):
time.sleep(min((delay_ms + jitter_ms) / 1000.0, 1.0))
+def build_write_job(content, language, is_code, is_encrypted):
+ return {
+ "content": content,
+ "language": language,
+ "is_code": int(is_code),
+ "is_encrypted": int(is_encrypted),
+ "created_at": int(time.time()),
+ "paste_id": None,
+ "error": None,
+ "done": threading.Event(),
+ }
+
+
+def execute_write_batch(conn, jobs):
+ for write_attempt in range(SQLITE_WRITE_RETRIES + 1):
+ try:
+ conn.execute("BEGIN IMMEDIATE")
+ for job in jobs:
+ for _ in range(5):
+ paste_id = generate_id()
+ try:
+ conn.execute(
+ "INSERT INTO pastes (id, content, language, is_code, is_encrypted, created_at) "
+ "VALUES (?, ?, ?, ?, ?, ?)",
+ (
+ paste_id,
+ job["content"],
+ job["language"],
+ job["is_code"],
+ job["is_encrypted"],
+ job["created_at"],
+ ),
+ )
+ job["paste_id"] = paste_id
+ break
+ except sqlite3.IntegrityError:
+ continue
+ if not job["paste_id"]:
+ raise RuntimeError("failed to generate unique paste id")
+
+ conn.commit()
+ return
+ except sqlite3.OperationalError as err:
+ try:
+ conn.rollback()
+ except sqlite3.DatabaseError:
+ pass
+
+ if is_sqlite_busy_error(err):
+ if write_attempt >= SQLITE_WRITE_RETRIES:
+ raise DatabaseBusyError("database is busy; retry shortly") from err
+ sqlite_retry_sleep(write_attempt)
+ continue
+
+ raise
+ except Exception:
+ try:
+ conn.rollback()
+ except sqlite3.DatabaseError:
+ pass
+ raise
+
+
+def flush_write_batch(conn, jobs):
+ try:
+ execute_write_batch(conn, jobs)
+ except Exception as err:
+ for job in jobs:
+ job["error"] = err
+ job["done"].set()
+ return
+
+ for job in jobs:
+ job["done"].set()
+
+
+def collect_write_batch(first_job):
+ jobs = [first_job]
+ deadline = time.monotonic() + (WRITE_BATCH_MAX_DELAY_MS / 1000.0)
+
+ while len(jobs) < WRITE_BATCH_SIZE:
+ remaining = deadline - time.monotonic()
+ if remaining <= 0:
+ break
+
+ try:
+ next_job = _write_queue.get(timeout=remaining)
+ except queue.Empty:
+ break
+
+ jobs.append(next_job)
+
+ return jobs
+
+
+def write_worker_loop():
+ conn = open_db()
+ try:
+ while True:
+ if _write_worker_stop.is_set() and _write_queue.empty():
+ break
+
+ try:
+ first_job = _write_queue.get(timeout=0.25)
+ except queue.Empty:
+ continue
+
+ jobs = collect_write_batch(first_job)
+ flush_write_batch(conn, jobs)
+ for _ in jobs:
+ _write_queue.task_done()
+ finally:
+ conn.close()
+
+
+def start_write_worker():
+ global _write_worker_thread
+
+ if _write_worker_thread and _write_worker_thread.is_alive():
+ return
+
+ _write_worker_stop.clear()
+ _write_worker_thread = threading.Thread(
+ target=write_worker_loop,
+ daemon=True,
+ name="sqlite-write-worker",
+ )
+ _write_worker_thread.start()
+
+
+def stop_write_worker():
+ global _write_worker_thread
+
+ if not _write_worker_thread:
+ return
+
+ _write_worker_stop.set()
+ _write_worker_thread.join(timeout=WRITE_WORKER_JOIN_TIMEOUT_SECONDS)
+ _write_worker_thread = None
+
+
def open_db():
conn = sqlite3.connect(
str(DB_PATH),
@@ -465,47 +489,25 @@ def is_valid_paste_id(paste_id):
def save_paste(content, language=None, is_code=False, is_encrypted=False):
"""store a paste in the database, return its id"""
- for write_attempt in range(SQLITE_WRITE_RETRIES + 1):
- conn = open_db()
- try:
- # reserve the write lock early to reduce lock thrash under bursty writes.
- conn.execute("BEGIN IMMEDIATE")
- for _ in range(5):
- paste_id = generate_id()
- try:
- conn.execute(
- "INSERT INTO pastes (id, content, language, is_code, is_encrypted, created_at) "
- "VALUES (?, ?, ?, ?, ?, ?)",
- (
- paste_id,
- content,
- language,
- int(is_code),
- int(is_encrypted),
- int(time.time()),
- ),
- )
- conn.commit()
- return paste_id
- except sqlite3.IntegrityError:
- continue
- conn.rollback()
- raise RuntimeError("failed to generate unique paste id")
- except sqlite3.OperationalError as err:
- try:
- conn.rollback()
- except sqlite3.DatabaseError:
- pass
- if is_sqlite_busy_error(err):
- if write_attempt >= SQLITE_WRITE_RETRIES:
- raise DatabaseBusyError("database is busy; retry shortly") from err
- sqlite_retry_sleep(write_attempt)
- continue
- raise
- finally:
- conn.close()
+ job = build_write_job(content, language, is_code, is_encrypted)
- raise DatabaseBusyError("database is busy; retry shortly")
+ try:
+ _write_queue.put(job, timeout=WRITE_QUEUE_ENQUEUE_TIMEOUT_SECONDS)
+ except queue.Full as err:
+ raise DatabaseBusyError("write queue is full; retry shortly") from err
+
+ if not job["done"].wait(WRITE_QUEUE_WAIT_SECONDS):
+ raise DatabaseBusyError("write queue timeout; retry shortly")
+
+ if job["error"]:
+ if isinstance(job["error"], Exception):
+ raise job["error"]
+ raise RuntimeError("write failed")
+
+ if not job["paste_id"]:
+ raise RuntimeError("write completed without paste id")
+
+ return job["paste_id"]
def get_paste(paste_id):
@@ -672,8 +674,8 @@ def paste_page(paste, csp_nonce):
highlight_css = ""
highlight_js = ""
if is_code:
- highlight_css = ''
- highlight_js = f"""
+ highlight_css = ''
+ highlight_js = f"""
hljs.highlightAll();"""
return f"""
@@ -1222,6 +1224,7 @@ class ClipboardHTTPServer(http.server.ThreadingHTTPServer):
def main():
init_db()
+ start_write_worker()
print(f"kj-clipboard - listening on {BIND}:{PORT}")
server = ClipboardHTTPServer((BIND, PORT), ClipboardHandler)
@@ -1245,6 +1248,7 @@ def main():
except KeyboardInterrupt:
print("\nreceived keyboard interrupt, shutting down.")
finally:
+ stop_write_worker()
server.server_close()
--
cgit v1.2.3