From 17c375a843e7eb0ec067a5f349b43eb50c8cd474 Mon Sep 17 00:00:00 2001 From: Harsh16gupta Date: Thu, 25 Jun 2026 20:02:53 +0530 Subject: [PATCH 1/3] feat: tag generation for the clusters --- src/commands/testEmbed.ts | 19 +- src/pipeline/clustering/postProcess.ts | 446 +++++++++++++++++++++++++ src/pipeline/runPipeline.ts | 13 + src/types/cluster.ts | 2 + src/webview/components/ClusterCard.tsx | 12 +- src/webview/pages/DashboardPage.tsx | 8 +- src/webview/panel.css | 31 +- 7 files changed, 524 insertions(+), 7 deletions(-) create mode 100644 src/pipeline/clustering/postProcess.ts diff --git a/src/commands/testEmbed.ts b/src/commands/testEmbed.ts index 744d9a9..28b1fba 100644 --- a/src/commands/testEmbed.ts +++ b/src/commands/testEmbed.ts @@ -7,6 +7,7 @@ import { isGenericTitle } from '../utils/titleFilter'; import { log, logErr } from '../utils/logger'; import { getEncoding } from 'js-tiktoken'; import { VectorCache } from '../pipeline/vectorCache'; +import { enrichResultsWithTags } from '../pipeline/clustering/postProcess'; // We use cl100k_base to approximate token counts for chunking. // The embedding model (all-MiniLM-L6-v2) uses a WordPiece tokenizer with a @@ -169,9 +170,20 @@ export const runTestEmbed = async (installDir: string) => { const vectors = noteVectors.map((nv) => nv.vector); const results = benchmark(vectors, clusterConfig); - // Log note titles per cluster for all strategies, in order (best to worst) + const notesMap = new Map(notes.map((n) => [n.id, n])); + const allPipelineDocuments = noteVectors.map((nv) => { + const originalNote = notesMap.get(nv.noteId); + return { + title: nv.title, + body: originalNote ? originalNote.body : '', + }; + }); + + enrichResultsWithTags(results, allPipelineDocuments); + + // Log note titles and tags per cluster for all strategies, in order (best to worst) for (const res of results) { - log(`\nCluster assignments (${res.strategyName}):`); + log(`\nCluster assignments & Analysis (${res.strategyName}):`); const clusterNotes = new Map(); for (let i = 0; i < noteVectors.length; i++) { const c = res.assignments[i]; @@ -180,7 +192,8 @@ export const runTestEmbed = async (installDir: string) => { } for (const [clusterId, titles] of clusterNotes) { const label = clusterId < 0 ? 'Noise/Outliers' : `Cluster ${clusterId}`; - log(` ${label} (${titles.length} notes):`); + const clusterTags = res.tags?.[clusterId] ? ` [Tags: ${res.tags[clusterId].join(', ')}]` : ''; + log(` ${label} (${titles.length} notes)${clusterTags}:`); for (const title of titles) { log(` - ${title}`); } diff --git a/src/pipeline/clustering/postProcess.ts b/src/pipeline/clustering/postProcess.ts new file mode 100644 index 0000000..ca366c5 --- /dev/null +++ b/src/pipeline/clustering/postProcess.ts @@ -0,0 +1,446 @@ +import { BenchmarkResult } from '../../types/cluster'; + +export const STOP_WORDS = new Set([ + // English articles, prepositions, conjunctions, pronouns (all length >= 3) + 'about', + 'above', + 'after', + 'again', + 'against', + 'all', + 'and', + 'any', + 'are', + 'arent', + 'because', + 'been', + 'before', + 'being', + 'below', + 'between', + 'both', + 'but', + 'cant', + 'cannot', + 'could', + 'couldnt', + 'did', + 'didnt', + 'does', + 'doesnt', + 'doing', + 'dont', + 'down', + 'during', + 'each', + 'few', + 'for', + 'from', + 'further', + 'had', + 'hadnt', + 'has', + 'hasnt', + 'have', + 'havent', + 'having', + 'hed', + 'hell', + 'hes', + 'her', + 'here', + 'heres', + 'hers', + 'herself', + 'him', + 'himself', + 'his', + 'how', + 'hows', + 'ill', + 'its', + 'itself', + 'lets', + 'more', + 'most', + 'mustnt', + 'myself', + 'nor', + 'not', + 'off', + 'once', + 'only', + 'other', + 'ought', + 'our', + 'ours', + 'ourselves', + 'out', + 'over', + 'own', + 'same', + 'shant', + 'she', + 'shed', + 'shell', + 'shes', + 'should', + 'shouldnt', + 'some', + 'such', + 'than', + 'that', + 'thats', + 'the', + 'their', + 'theirs', + 'them', + 'themselves', + 'then', + 'there', + 'theres', + 'these', + 'they', + 'theyd', + 'theyll', + 'theyre', + 'theyve', + 'this', + 'those', + 'through', + 'too', + 'under', + 'until', + 'very', + 'was', + 'wasnt', + 'wed', + 'well', + 'were', + 'weve', + 'werent', + 'what', + 'whats', + 'when', + 'whens', + 'where', + 'wheres', + 'which', + 'while', + 'who', + 'whos', + 'whom', + 'why', + 'whys', + 'with', + 'wont', + 'would', + 'wouldnt', + 'youd', + 'youll', + 'youre', + 'youve', + 'your', + 'yours', + 'yourself', + 'yourselves', + + // Markdown/HTML structure words or general noise words (all length >= 3) + 'http', + 'https', + 'www', + 'com', + 'org', + 'net', + 'html', + 'xml', + 'css', + 'img', + 'href', + 'src', + 'div', + 'span', + 'class', + 'get', + 'post', + 'put', + 'delete', + 'use', + 'using', + 'used', + 'make', + 'made', + 'take', + 'took', + 'see', + 'saw', + 'also', + 'like', + 'one', + 'two', + 'three', + 'four', + 'five', + 'six', + 'seven', + 'eight', + 'nine', + 'ten', + 'first', + 'second', + 'third', + + // Code keywords / programming syntax + 'const', + 'let', + 'var', + 'function', + 'return', + 'class', + 'interface', + 'type', + 'import', + 'export', + 'void', + 'string', + 'number', + 'boolean', + 'any', + 'public', + 'private', + 'protected', + 'async', + 'await', + 'null', + 'undefined', + 'true', + 'false', + 'switch', + 'case', + 'break', + + // Generic Joplin / note-taking fillers (often pollute tags in this context) + 'note', + 'notes', + 'joplin', + 'plugin', + 'folder', + 'folders', + 'notebook', + 'notebooks', + 'tag', + 'tags', + 'todo', + 'todos', + 'task', + 'tasks', + 'file', + 'files', + 'page', + 'pages', + 'data', + 'info', + 'information', +]); + +/** Words that look like plurals but should not be singularized. */ +const SINGULAR_EXCEPTIONS = new Set(['series', 'species', 'means', 'news', 'analysis', 'basis', 'crisis']); + +/** + * Strips code blocks, inline code, HTML tags, markdown links/images, and URLs + * from text to avoid polluting tag extraction. + */ +export function cleanText(text: string): string { + if (!text) return ''; + let cleaned = text; + // Strip triple-backtick markdown code blocks + cleaned = cleaned.replace(/```[\s\S]*?```/g, ' '); + // Strip inline code backticks + cleaned = cleaned.replace(/`[^`]*`/g, ' '); + // Strip markdown images ![alt](url) and links [text](url) — keep the text, remove syntax + cleaned = cleaned.replace(/!?\[([^\]]*)\]\([^)]*\)/g, '$1'); + // Strip HTML tags + cleaned = cleaned.replace(/<[^>]*>/g, ' '); + // Strip URLs + cleaned = cleaned.replace(/https?:\/\/\S+/gi, ' '); + return cleaned; +} + +/** + * A lightweight, dependency-free helper to stem basic English plural words to their singular form. + * Handles common cases like -ies -> -y, -es -> - (e.g. boxes -> box), and trailing -s (notes -> note). + */ +export function singularize(word: string): string { + if (word.length <= 3) return word; + if (SINGULAR_EXCEPTIONS.has(word)) return word; + if (word.endsWith('ss')) return word; // e.g. class, process + if (word.endsWith('ies')) return word.slice(0, -3) + 'y'; // e.g. categories -> category + if (word.endsWith('es')) { + const base = word.slice(0, -2); + if ( + base.endsWith('ss') || + base.endsWith('ch') || + base.endsWith('sh') || + base.endsWith('x') || + base.endsWith('z') + ) { + return base; // e.g. classes -> class, boxes -> box + } + return word.slice(0, -1); // e.g. databases -> database, lines -> line + } + if (word.endsWith('s') && !word.endsWith('us') && !word.endsWith('is') && !word.endsWith('as')) { + return word.slice(0, -1); // e.g. notes -> note, tasks -> task + } + return word; +} + +/** + * Lowercases text, cleans it, singularizes it, and tokenizes into alphabetic words + * of length >= 3 that are not in the stop words list. + */ +export function tokenize(text: string): string[] { + const cleaned = cleanText(text); + const matches = cleaned.toLowerCase().match(/[a-z]+/g) || []; + return matches.map(singularize).filter((w) => w.length >= 3 && !STOP_WORDS.has(w)); +} + +export interface DocumentText { + title: string; + body: string; +} + +/** + * Extracts descriptive tags from documents in a cluster using TF-IDF. + */ +export class TfidfExtractor { + private idfs: { [word: string]: number } = {}; + + constructor(allDocuments: DocumentText[]) { + const N = allDocuments.length; + if (N === 0) return; + + const docFreqs: { [word: string]: number } = {}; + + for (const doc of allDocuments) { + // For IDF, we only need unique words per document — no title weighting needed + const uniqueWords = this.getUniqueDocumentWords(doc); + for (const word of uniqueWords) { + docFreqs[word] = (docFreqs[word] || 0) + 1; + } + } + + for (const word of Object.keys(docFreqs)) { + const df = docFreqs[word]; + // Max DF rule: If a word appears in > 60% of all notes, it is too generic, ignore it. + if (df / N > 0.6) { + this.idfs[word] = 0; + } else { + this.idfs[word] = Math.log(N / df) + 1; + } + } + } + + /** + * Returns the unique set of words in a document (title + body), used for IDF counting. + * No title weighting — each document contributes at most 1 to each word's document frequency. + */ + private getUniqueDocumentWords(doc: DocumentText): Set { + const titleWords = tokenize(doc.title || ''); + const bodyWords = tokenize(doc.body || ''); + return new Set([...titleWords, ...bodyWords]); + } + + /** + * Returns words for TF scoring with title words weighted 3x higher. + * Uses push loops instead of spread to avoid excess intermediate array allocations. + */ + private getWeightedWords(doc: DocumentText): string[] { + const titleWords = tokenize(doc.title || ''); + const bodyWords = tokenize(doc.body || ''); + const result: string[] = []; + // Title words appear 3 times to boost their term frequency + for (let i = 0; i < 3; i++) { + for (const w of titleWords) { + result.push(w); + } + } + for (const w of bodyWords) { + result.push(w); + } + return result; + } + + /** + * Computes TF-IDF scores for words in the cluster documents and returns the top K. + */ + public extractClusterTags(clusterDocuments: DocumentText[], topK = 5): string[] { + if (clusterDocuments.length === 0) return []; + + const clusterWords: string[] = []; + for (const doc of clusterDocuments) { + const weighted = this.getWeightedWords(doc); + for (const w of weighted) { + clusterWords.push(w); + } + } + + if (clusterWords.length === 0) return []; + + const tfs: { [word: string]: number } = {}; + for (const word of clusterWords) { + tfs[word] = (tfs[word] || 0) + 1; + } + + const totalWords = clusterWords.length; + const scores: { word: string; score: number }[] = []; + + for (const word of Object.keys(tfs)) { + const tf = tfs[word] / totalWords; + const idf = this.idfs[word] || 0; // default to 0 if word is ignored/generic + if (idf > 0) { + scores.push({ word, score: tf * idf }); + } + } + + scores.sort((a, b) => b.score - a.score); + return scores.slice(0, topK).map((s) => s.word); + } +} + +/** + * Enriches benchmark results with extracted TF-IDF tags for each cluster. + * + * Builds the TF-IDF corpus from all pipeline documents once, then iterates + * over each strategy result to extract the top tags per cluster. + * + * @param results Benchmark results from the clustering pipeline + * @param documents All note documents used in the pipeline (same order as noteVectors) + * @param topK Number of tags to extract per cluster (default: 5) + */ +export function enrichResultsWithTags(results: BenchmarkResult[], documents: DocumentText[], topK = 5): void { + const tfidfExtractor = new TfidfExtractor(documents); + + for (const result of results) { + const tags: { [clusterId: number]: string[] } = {}; + + const clusterIndices: { [clusterId: number]: number[] } = {}; + result.assignments.forEach((clusterId, noteIdx) => { + if (clusterId !== -1) { + if (!clusterIndices[clusterId]) { + clusterIndices[clusterId] = []; + } + clusterIndices[clusterId].push(noteIdx); + } + }); + + for (const clusterIdStr of Object.keys(clusterIndices)) { + const clusterId = Number(clusterIdStr); + const indices = clusterIndices[clusterId]; + + const clusterDocuments = indices.map((idx) => documents[idx]); + tags[clusterId] = tfidfExtractor.extractClusterTags(clusterDocuments, topK); + } + + result.tags = tags; + } +} diff --git a/src/pipeline/runPipeline.ts b/src/pipeline/runPipeline.ts index 0a67fe4..dd964f8 100644 --- a/src/pipeline/runPipeline.ts +++ b/src/pipeline/runPipeline.ts @@ -8,6 +8,7 @@ import { isGenericTitle } from '../utils/titleFilter'; import { log, logErr } from '../utils/logger'; import { getEncoding } from 'js-tiktoken'; import { VectorCache } from './vectorCache'; +import { enrichResultsWithTags } from './clustering/postProcess'; // See testEmbed.ts for rationale on cl100k_base and the 200-token limit. const enc = getEncoding('cl100k_base'); @@ -178,6 +179,18 @@ export const runPipeline = async (installDir: string, callbacks: PipelineCallbac const vectors = noteVectors.map((nv) => nv.vector); const results = benchmark(vectors, DEFAULT_CONFIG); + // Post-process to extract tags/keywords for each cluster + const notesMap = new Map(notes.map((n) => [n.id, n])); + const allPipelineDocuments = noteVectors.map((nv) => { + const originalNote = notesMap.get(nv.noteId); + return { + title: nv.title, + body: originalNote ? originalNote.body : '', + }; + }); + + enrichResultsWithTags(results, allPipelineDocuments); + const panelNotes: PanelNote[] = noteVectors.map((nv) => ({ noteId: nv.noteId, title: nv.title, diff --git a/src/types/cluster.ts b/src/types/cluster.ts index 52028dd..6365583 100644 --- a/src/types/cluster.ts +++ b/src/types/cluster.ts @@ -41,4 +41,6 @@ export interface BenchmarkResult { /** Number of points classified as noise/outliers (HDBSCAN only) */ outlierCount: number; timeMs: number; + /** Extracted tags for each cluster, keyed by cluster ID. Outliers (-1) are excluded. */ + tags?: { [clusterId: number]: string[] }; } diff --git a/src/webview/components/ClusterCard.tsx b/src/webview/components/ClusterCard.tsx index 729e5db..8de3d66 100644 --- a/src/webview/components/ClusterCard.tsx +++ b/src/webview/components/ClusterCard.tsx @@ -6,9 +6,10 @@ interface ClusterCardProps { noteIndices: number[]; notes: PanelNote[]; isNoise?: boolean; + tags?: string[]; } -export const ClusterCard: React.FC = ({ title, noteIndices, notes, isNoise }) => { +export const ClusterCard: React.FC = ({ title, noteIndices, notes, isNoise, tags }) => { const [isExpanded, setIsExpanded] = React.useState(false); const handleHeaderClick = () => { @@ -27,6 +28,15 @@ export const ClusterCard: React.FC = ({ title, noteIndices, no
{title} + {tags && tags.length > 0 && ( +
+ {tags.map((tag, idx) => ( + + #{tag} + + ))} +
+ )}
{countLabel} diff --git a/src/webview/pages/DashboardPage.tsx b/src/webview/pages/DashboardPage.tsx index 3e84794..5dc3e35 100644 --- a/src/webview/pages/DashboardPage.tsx +++ b/src/webview/pages/DashboardPage.tsx @@ -41,7 +41,13 @@ export const DashboardPage: React.FC = () => {
{sortedClusterIds.map((id, idx) => ( - + ))} {noise.length > 0 && ( diff --git a/src/webview/panel.css b/src/webview/panel.css index 77de1f6..18b1020 100644 --- a/src/webview/panel.css +++ b/src/webview/panel.css @@ -219,9 +219,36 @@ body { .cluster-header-left { display: flex; - align-items: center; - gap: 8px; + flex-direction: column; + align-items: flex-start; + gap: 6px; min-width: 0; + flex: 1; +} + +.cluster-tags { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-top: 4px; +} + +.cluster-tag { + font-size: 0.8em; + font-weight: 600; + background: rgba(74, 144, 217, 0.14); /* fallback for older Chromium */ + background: color-mix(in srgb, var(--accent) 14%, transparent); + color: var(--accent); + padding: 2px 8px; + border-radius: 6px; + white-space: nowrap; + letter-spacing: 0.2px; + transition: all 0.15s ease; +} + +.cluster-tag:hover { + background: rgba(74, 144, 217, 0.22); /* fallback for older Chromium */ + background: color-mix(in srgb, var(--accent) 22%, transparent); } .cluster-title { From fd1210b4457e4070d9076463b1e98b77a4cb04e6 Mon Sep 17 00:00:00 2001 From: Harsh16gupta Date: Fri, 26 Jun 2026 09:09:52 +0530 Subject: [PATCH 2/3] fix: address Copilot review --- src/pipeline/clustering/postProcess.ts | 19 ++++++++----------- src/webview/components/ClusterCard.tsx | 2 +- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/src/pipeline/clustering/postProcess.ts b/src/pipeline/clustering/postProcess.ts index ca366c5..8a064c0 100644 --- a/src/pipeline/clustering/postProcess.ts +++ b/src/pipeline/clustering/postProcess.ts @@ -299,8 +299,8 @@ export function singularize(word: string): string { * of length >= 3 that are not in the stop words list. */ export function tokenize(text: string): string[] { - const cleaned = cleanText(text); - const matches = cleaned.toLowerCase().match(/[a-z]+/g) || []; + const cleaned = cleanText(text).toLowerCase().replace(/[’']/g, ''); + const matches = cleaned.match(/[a-z]+/g) || []; return matches.map(singularize).filter((w) => w.length >= 3 && !STOP_WORDS.has(w)); } @@ -376,22 +376,19 @@ export class TfidfExtractor { public extractClusterTags(clusterDocuments: DocumentText[], topK = 5): string[] { if (clusterDocuments.length === 0) return []; - const clusterWords: string[] = []; + const tfs: { [word: string]: number } = {}; + let totalWords = 0; + for (const doc of clusterDocuments) { const weighted = this.getWeightedWords(doc); for (const w of weighted) { - clusterWords.push(w); + tfs[w] = (tfs[w] || 0) + 1; + totalWords++; } } - if (clusterWords.length === 0) return []; - - const tfs: { [word: string]: number } = {}; - for (const word of clusterWords) { - tfs[word] = (tfs[word] || 0) + 1; - } + if (totalWords === 0) return []; - const totalWords = clusterWords.length; const scores: { word: string; score: number }[] = []; for (const word of Object.keys(tfs)) { diff --git a/src/webview/components/ClusterCard.tsx b/src/webview/components/ClusterCard.tsx index 8de3d66..334c0cf 100644 --- a/src/webview/components/ClusterCard.tsx +++ b/src/webview/components/ClusterCard.tsx @@ -31,7 +31,7 @@ export const ClusterCard: React.FC = ({ title, noteIndices, no {tags && tags.length > 0 && (
{tags.map((tag, idx) => ( - + #{tag} ))} From 119507486dc35c3a2e1220088978d54d93511b42 Mon Sep 17 00:00:00 2001 From: Harsh16gupta Date: Fri, 3 Jul 2026 18:12:09 +0530 Subject: [PATCH 3/3] feat: enhance TF-IDF tag extraction with n-grams and redundancy filtering --- src/pipeline/clustering/postProcess.ts | 68 +++++++++++++++++++++----- 1 file changed, 55 insertions(+), 13 deletions(-) diff --git a/src/pipeline/clustering/postProcess.ts b/src/pipeline/clustering/postProcess.ts index 8a064c0..7c88453 100644 --- a/src/pipeline/clustering/postProcess.ts +++ b/src/pipeline/clustering/postProcess.ts @@ -304,6 +304,27 @@ export function tokenize(text: string): string[] { return matches.map(singularize).filter((w) => w.length >= 3 && !STOP_WORDS.has(w)); } +/** + * Generates unigrams, bigrams, and trigrams from a sequence of tokens. + */ +export function getNgrams(tokens: string[]): string[] { + const ngrams: string[] = []; + const N = tokens.length; + for (let i = 0; i < N; i++) { + // Unigram + ngrams.push(tokens[i]); + // Bigram + if (i < N - 1) { + ngrams.push(`${tokens[i]} ${tokens[i + 1]}`); + } + // Trigram + if (i < N - 2) { + ngrams.push(`${tokens[i]} ${tokens[i + 1]} ${tokens[i + 2]}`); + } + } + return ngrams; +} + export interface DocumentText { title: string; body: string; @@ -322,7 +343,7 @@ export class TfidfExtractor { const docFreqs: { [word: string]: number } = {}; for (const doc of allDocuments) { - // For IDF, we only need unique words per document — no title weighting needed + // For IDF, we only need unique words/ngrams per document — no title weighting needed const uniqueWords = this.getUniqueDocumentWords(doc); for (const word of uniqueWords) { docFreqs[word] = (docFreqs[word] || 0) + 1; @@ -331,7 +352,7 @@ export class TfidfExtractor { for (const word of Object.keys(docFreqs)) { const df = docFreqs[word]; - // Max DF rule: If a word appears in > 60% of all notes, it is too generic, ignore it. + // Max DF rule: If a word/ngram appears in > 60% of all notes, it is too generic, ignore it. if (df / N > 0.6) { this.idfs[word] = 0; } else { @@ -341,37 +362,41 @@ export class TfidfExtractor { } /** - * Returns the unique set of words in a document (title + body), used for IDF counting. - * No title weighting — each document contributes at most 1 to each word's document frequency. + * Returns the unique set of words/ngrams in a document (title + body), used for IDF counting. + * No title weighting — each document contributes at most 1 to each ngram's document frequency. */ private getUniqueDocumentWords(doc: DocumentText): Set { const titleWords = tokenize(doc.title || ''); const bodyWords = tokenize(doc.body || ''); - return new Set([...titleWords, ...bodyWords]); + const titleNgrams = getNgrams(titleWords); + const bodyNgrams = getNgrams(bodyWords); + return new Set([...titleNgrams, ...bodyNgrams]); } /** - * Returns words for TF scoring with title words weighted 3x higher. + * Returns ngrams for TF scoring with title words weighted 3x higher. * Uses push loops instead of spread to avoid excess intermediate array allocations. */ private getWeightedWords(doc: DocumentText): string[] { const titleWords = tokenize(doc.title || ''); const bodyWords = tokenize(doc.body || ''); + const titleNgrams = getNgrams(titleWords); + const bodyNgrams = getNgrams(bodyWords); const result: string[] = []; - // Title words appear 3 times to boost their term frequency + // Title ngrams appear 3 times to boost their term frequency for (let i = 0; i < 3; i++) { - for (const w of titleWords) { - result.push(w); + for (const ng of titleNgrams) { + result.push(ng); } } - for (const w of bodyWords) { - result.push(w); + for (const ng of bodyNgrams) { + result.push(ng); } return result; } /** - * Computes TF-IDF scores for words in the cluster documents and returns the top K. + * Computes TF-IDF scores for ngrams in the cluster documents and returns the top K. */ public extractClusterTags(clusterDocuments: DocumentText[], topK = 5): string[] { if (clusterDocuments.length === 0) return []; @@ -400,7 +425,24 @@ export class TfidfExtractor { } scores.sort((a, b) => b.score - a.score); - return scores.slice(0, topK).map((s) => s.word); + + const selectedTags: string[] = []; + const usedWords = new Set(); + + for (const candidate of scores) { + if (selectedTags.length >= topK) break; + + const constituentWords = candidate.word.split(' '); + const allUsed = constituentWords.every((w) => usedWords.has(w)); + if (!allUsed) { + selectedTags.push(candidate.word); + for (const w of constituentWords) { + usedWords.add(w); + } + } + } + + return selectedTags; } }