abhishekbhakat/Prompter
1
0
Fork 0
Code Issues Pull Requests Actions Releases 2 Activity

modular code

Browse Source
This commit is contained in:
abhishekbhakat
2025-03-11 18:27:57 +00:00
parent 6bd50154f0
commit 3676136692
7 changed files with 427 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

311
src/extension.ts
View File

@@ -1,251 +1,22 @@
// The module 'vscode' contains the VS Code extensibility API
import * as vscode from 'vscode';
import * as path from 'path';
import * as fs from 'fs';
// Custom TreeItem for files and folders
class FileTreeItem extends vscode.TreeItem {
constructor(
public readonly resourceUri: vscode.Uri,
public readonly collapsibleState: vscode.TreeItemCollapsibleState,
public readonly isDirectory: boolean
) {
super(resourceUri, collapsibleState);
this.contextValue = isDirectory ? 'directory' : 'file';
this.tooltip = resourceUri.fsPath;
this.description = isDirectory ? 'folder' : path.extname(resourceUri.fsPath);
this.checkboxState = vscode.TreeItemCheckboxState.Unchecked;
}
}
// Import our modular components
import { PrompterSettings, DEFAULT_SETTINGS } from './models/settings';
import { FileTreeItem, SettingTreeItem } from './models/treeItems';
import { PrompterTreeProvider } from './providers/prompterTreeProvider';
import { PromptGenerator } from './utils/promptGenerator';
// Settings interface
interface PrompterSettings {
xmlEditsEnabled: boolean;
includeLineNumbers: boolean;
includeComments: boolean;
tokenCalculationEnabled: boolean;
}
// Custom TreeItem for settings
class SettingTreeItem extends vscode.TreeItem {
constructor(
public readonly label: string,
public readonly settingKey: keyof PrompterSettings,
public readonly checked: boolean
) {
super(label, vscode.TreeItemCollapsibleState.None);
this.checkboxState = checked ? vscode.TreeItemCheckboxState.Checked : vscode.TreeItemCheckboxState.Unchecked;
this.contextValue = 'setting';
}
}
// TreeView provider for the sidebar
class PrompterTreeProvider implements vscode.TreeDataProvider<FileTreeItem | SettingTreeItem> {
private _onDidChangeTreeData: vscode.EventEmitter<FileTreeItem | SettingTreeItem | undefined | null | void> = new vscode.EventEmitter<FileTreeItem | SettingTreeItem | undefined | null | void>();
readonly onDidChangeTreeData: vscode.Event<FileTreeItem | SettingTreeItem | undefined | null | void> = this._onDidChangeTreeData.event;
private selectedFiles: Set<string> = new Set();
private settings: PrompterSettings = {
xmlEditsEnabled: false,
includeLineNumbers: false,
includeComments: true,
tokenCalculationEnabled: true
};
private showingSettings: boolean = false;
constructor(private workspaceRoot: string | undefined) {}
// Toggle between file view and settings view
toggleSettingsView(): void {
this.showingSettings = !this.showingSettings;
this.refresh();
}
getTreeItem(element: FileTreeItem | SettingTreeItem): vscode.TreeItem {
// Return the element as is if it's a SettingTreeItem
if (element instanceof SettingTreeItem) {
return element;
}
// Handle FileTreeItem
const fileItem = element as FileTreeItem;
if (fileItem.resourceUri && this.selectedFiles.has(fileItem.resourceUri.fsPath)) {
fileItem.checkboxState = vscode.TreeItemCheckboxState.Checked;
} else {
fileItem.checkboxState = vscode.TreeItemCheckboxState.Unchecked;
}
return fileItem;
}
async getChildren(element?: FileTreeItem | SettingTreeItem): Promise<(FileTreeItem | SettingTreeItem)[]> {
// If we're showing settings, return settings items
if (this.showingSettings && !element) {
return this.getSettingsItems();
}
// Otherwise show files
if (!this.workspaceRoot) {
vscode.window.showInformationMessage('No workspace folder is opened');
return Promise.resolve([]);
}
if (element) {
// Make sure we're dealing with a FileTreeItem that has a resourceUri
if (!(element instanceof SettingTreeItem) && element.resourceUri) {
return this.getFilesInDirectory(element.resourceUri.fsPath);
}
return Promise.resolve([]);
} else {
return this.getFilesInDirectory(this.workspaceRoot);
}
}
private async getFilesInDirectory(dirPath: string): Promise<FileTreeItem[]> {
try {
const files = await vscode.workspace.fs.readDirectory(vscode.Uri.file(dirPath));
return Promise.all(files.map(async ([name, type]) => {
const filePath = path.join(dirPath, name);
const uri = vscode.Uri.file(filePath);
const isDirectory = (type & vscode.FileType.Directory) !== 0;
return new FileTreeItem(
uri,
isDirectory ? vscode.TreeItemCollapsibleState.Collapsed : vscode.TreeItemCollapsibleState.None,
isDirectory
);
}));
} catch (error) {
console.error(`Error reading directory ${dirPath}:`, error);
return [];
}
}
toggleSelection(item: FileTreeItem): void {
const filePath = item.resourceUri.fsPath;
console.log('Toggle selection called for:', filePath);
if (this.selectedFiles.has(filePath)) {
this.removeFromSelection(item);
} else {
this.addToSelection(item);
}
}
addToSelection(item: FileTreeItem): void {
const filePath = item.resourceUri.fsPath;
this.selectedFiles.add(filePath);
console.log('Added file to selection, count:', this.selectedFiles.size);
this._onDidChangeTreeData.fire();
}
removeFromSelection(item: FileTreeItem): void {
const filePath = item.resourceUri.fsPath;
this.selectedFiles.delete(filePath);
console.log('Removed file from selection, count:', this.selectedFiles.size);
this._onDidChangeTreeData.fire();
}
getSelectedFiles(): Set<string> {
console.log('getSelectedFiles called, count:', this.selectedFiles.size);
return this.selectedFiles;
}
// Get settings items for the tree view
private getSettingsItems(): SettingTreeItem[] {
return [
new SettingTreeItem('XML Edits', 'xmlEditsEnabled', this.settings.xmlEditsEnabled),
new SettingTreeItem('Include Line Numbers', 'includeLineNumbers', this.settings.includeLineNumbers),
new SettingTreeItem('Include Comments', 'includeComments', this.settings.includeComments),
new SettingTreeItem('Token Calculation', 'tokenCalculationEnabled', this.settings.tokenCalculationEnabled)
];
}
// Update a setting value
updateSetting(key: keyof PrompterSettings, value: boolean): void {
this.settings[key] = value;
this.refresh();
}
isShowingSettings(): boolean {
return this.showingSettings;
}
toggleXmlEdits(): void {
this.settings.xmlEditsEnabled = !this.settings.xmlEditsEnabled;
}
isXmlEditsEnabled(): boolean {
return this.settings.xmlEditsEnabled;
}
getSettings(): PrompterSettings {
return this.settings;
}
updateSettings(newSettings: PrompterSettings): void {
this.settings = { ...newSettings };
this.refresh();
}
refresh(): void {
this._onDidChangeTreeData.fire();
}
}
// Utility function to estimate tokens in text
function estimateTokens(text: string): number {
// Rough estimation: Split by whitespace and punctuation
// This is a simple approximation, actual token count may vary by model
const words = text.split(/[\s\p{P}]+/u).filter(Boolean);
return Math.ceil(words.length * 1.3); // Add 30% overhead for special tokens
}
// Function to read file content
async function readFileContent(filePath: string): Promise<string> {
try {
const readData = await vscode.workspace.fs.readFile(vscode.Uri.file(filePath));
return Buffer.from(readData).toString('utf8');
} catch (error) {
console.error(`Error reading file ${filePath}:`, error);
return '';
}
}
// Function to generate XML prompt
function generateXMLPrompt(files: Map<string, { content: string; tokens: number }>, settings: PrompterSettings): string {
const xmlParts = ['<?xml version="1.0" encoding="UTF-8"?>', '<prompt>'];
// Add files section
xmlParts.push(' <files>');
for (const [path, { content, tokens }] of files) {
xmlParts.push(' <file>');
xmlParts.push(` <path>${path}</path>`);
xmlParts.push(` <tokens>${tokens}</tokens>`);
xmlParts.push(` <content><![CDATA[${content}]]></content>`);
xmlParts.push(' </file>');
}
xmlParts.push(' </files>');
// Add options based on settings
xmlParts.push(' <options>');
if (settings.xmlEditsEnabled) {
xmlParts.push(' <xml_edits>true</xml_edits>');
}
if (settings.includeLineNumbers) {
xmlParts.push(' <include_line_numbers>true</include_line_numbers>');
}
if (!settings.includeComments) {
xmlParts.push(' <exclude_comments>true</exclude_comments>');
}
xmlParts.push(' </options>');
xmlParts.push('</prompt>');
return xmlParts.join('\n');
}
// This method is called when your extension is activated
/**
* This method is called when the extension is activated
*/
export function activate(context: vscode.ExtensionContext) {
console.log('Congratulations, your extension "prompter" is now active!');
console.log('Prompter extension is now active!');
// Get the workspace folder
const workspaceRoot = vscode.workspace.workspaceFolders?.[0]?.uri?.fsPath;
// Create the tree data provider
const prompterTreeProvider = new PrompterTreeProvider(workspaceRoot);
// Register the TreeView with checkbox support
@@ -293,14 +64,16 @@ export function activate(context: vscode.ExtensionContext) {
// Create copy button
const copyButton = vscode.window.createStatusBarItem(vscode.StatusBarAlignment.Right);
copyButton.text = "$(clippy) Copy";
copyButton.tooltip = "Generate and copy XML prompt";
copyButton.tooltip = "Generate and copy prompt";
copyButton.command = 'prompter.generatePrompt';
copyButton.show();
// Register command to toggle file selection
let toggleSelectionCommand = vscode.commands.registerCommand('prompter.toggleSelection', (item: FileTreeItem) => {
console.log('Toggle selection command triggered for:', item.resourceUri.fsPath);
prompterTreeProvider.toggleSelection(item);
if (item.resourceUri) {
console.log('Toggle selection command triggered for:', item.resourceUri.fsPath);
prompterTreeProvider.toggleSelection(item);
}
});
// Register command to toggle XML edits
@@ -317,9 +90,6 @@ export function activate(context: vscode.ExtensionContext) {
settingsButton.command = 'prompter.openSettings';
settingsButton.show();
// Settings panel state
let settingsPanel: vscode.WebviewPanel | undefined = undefined;
// Register command to open settings
let openSettingsCommand = vscode.commands.registerCommand('prompter.openSettings', () => {
prompterTreeProvider.toggleSettingsView();
@@ -335,40 +105,29 @@ export function activate(context: vscode.ExtensionContext) {
let generatePromptCommand = vscode.commands.registerCommand('prompter.generatePrompt', async () => {
const selectedFiles = prompterTreeProvider.getSelectedFiles();
console.log('Generate prompt command triggered, selected files:', [...selectedFiles]);
if (selectedFiles.size === 0) {
vscode.window.showInformationMessage('Please select files first');
return;
}
let totalTokens = 0;
const fileContents = new Map<string, { content: string; tokens: number }>();
// Process each selected file
for (const filePath of selectedFiles) {
const content = await readFileContent(filePath);
const tokens = estimateTokens(content);
totalTokens += tokens;
fileContents.set(filePath, { content, tokens });
// Show individual file token count
vscode.window.showInformationMessage(
`File: ${filePath}\nEstimated tokens: ${tokens}`
try {
// Generate the prompt using our utility class
const promptText = await PromptGenerator.generatePrompt(
selectedFiles,
prompterTreeProvider.getSettings()
);
// Copy to clipboard
await vscode.env.clipboard.writeText(promptText);
vscode.window.showInformationMessage('Prompt copied to clipboard!');
} catch (error) {
console.error('Error generating prompt:', error);
vscode.window.showErrorMessage('Error generating prompt');
}
// Show total token count
vscode.window.showInformationMessage(
`Total estimated tokens for ${selectedFiles.size} files: ${totalTokens}`
);
// Generate XML prompt
const xmlPrompt = generateXMLPrompt(fileContents, prompterTreeProvider.getSettings());
// Copy to clipboard
await vscode.env.clipboard.writeText(xmlPrompt);
vscode.window.showInformationMessage('XML prompt copied to clipboard!');
});
// Add all disposables to context subscriptions
context.subscriptions.push(
treeView,
xmlEditsButton,
@@ -381,5 +140,9 @@ export function activate(context: vscode.ExtensionContext) {
);
}
// This method is called when your extension is deactivated
export function deactivate() {}
/**
* This method is called when the extension is deactivated
*/
export function deactivate() {
// Clean up resources when the extension is deactivated
}

15
src/models/settings.ts Normal file
View File

@@ -0,0 +1,15 @@
// Settings interface for the Prompter extension
export interface PrompterSettings {
xmlEditsEnabled: boolean;
includeLineNumbers: boolean;
includeComments: boolean;
tokenCalculationEnabled: boolean;
}
// Default settings values
export const DEFAULT_SETTINGS: PrompterSettings = {
xmlEditsEnabled: false,
includeLineNumbers: false,
includeComments: true,
tokenCalculationEnabled: true
};

32
src/models/treeItems.ts Normal file
View File

@@ -0,0 +1,32 @@
import * as vscode from 'vscode';
import * as path from 'path';
import { PrompterSettings } from './settings';
// Tree item for files in the explorer
export class FileTreeItem extends vscode.TreeItem {
constructor(
public readonly resourceUri: vscode.Uri,
public readonly collapsibleState: vscode.TreeItemCollapsibleState
) {
super(resourceUri, collapsibleState);
this.tooltip = resourceUri.fsPath;
this.description = path.basename(resourceUri.fsPath);
this.contextValue = 'file';
// Set the checkbox state to unchecked by default
this.checkboxState = vscode.TreeItemCheckboxState.Unchecked;
}
}
// Tree item for settings in the settings view
export class SettingTreeItem extends vscode.TreeItem {
constructor(
public readonly label: string,
public readonly settingKey: keyof PrompterSettings,
public readonly checked: boolean
) {
super(label, vscode.TreeItemCollapsibleState.None);
this.checkboxState = checked ? vscode.TreeItemCheckboxState.Checked : vscode.TreeItemCheckboxState.Unchecked;
this.contextValue = 'setting';
}
}

171
src/providers/prompterTreeProvider.ts Normal file
View File

@@ -0,0 +1,171 @@
import * as vscode from 'vscode';
import * as path from 'path';
import { FileTreeItem, SettingTreeItem } from '../models/treeItems';
import { PrompterSettings, DEFAULT_SETTINGS } from '../models/settings';
/**
* Tree provider for the Prompter extension
* Handles both file browsing and settings views
*/
export class PrompterTreeProvider implements vscode.TreeDataProvider<FileTreeItem | SettingTreeItem> {
private _onDidChangeTreeData: vscode.EventEmitter<FileTreeItem | SettingTreeItem | undefined | null | void> =
new vscode.EventEmitter<FileTreeItem | SettingTreeItem | undefined | null | void>();
readonly onDidChangeTreeData: vscode.Event<FileTreeItem | SettingTreeItem | undefined | null | void> =
this._onDidChangeTreeData.event;
private selectedFiles: Set<string> = new Set();
private settings: PrompterSettings = { ...DEFAULT_SETTINGS };
private showingSettings: boolean = false;
constructor(private workspaceRoot: string | undefined) {}
// Toggle between file view and settings view
toggleSettingsView(): void {
this.showingSettings = !this.showingSettings;
this.refresh();
}
getTreeItem(element: FileTreeItem | SettingTreeItem): vscode.TreeItem {
// Return the element as is if it's a SettingTreeItem
if (element instanceof SettingTreeItem) {
return element;
}
// Handle FileTreeItem
const fileItem = element as FileTreeItem;
if (fileItem.resourceUri && this.selectedFiles.has(fileItem.resourceUri.fsPath)) {
fileItem.checkboxState = vscode.TreeItemCheckboxState.Checked;
} else {
fileItem.checkboxState = vscode.TreeItemCheckboxState.Unchecked;
}
return fileItem;
}
async getChildren(element?: FileTreeItem | SettingTreeItem): Promise<(FileTreeItem | SettingTreeItem)[]> {
// If we're showing settings, return settings items
if (this.showingSettings && !element) {
return this.getSettingsItems();
}
// Otherwise show files
if (!this.workspaceRoot) {
vscode.window.showInformationMessage('No workspace folder is opened');
return Promise.resolve([]);
}
if (element) {
// Make sure we're dealing with a FileTreeItem that has a resourceUri
if (!(element instanceof SettingTreeItem) && element.resourceUri) {
return this.getFilesInDirectory(element.resourceUri.fsPath);
}
return Promise.resolve([]);
} else {
return this.getFilesInDirectory(this.workspaceRoot);
}
}
private async getFilesInDirectory(dirPath: string): Promise<FileTreeItem[]> {
try {
const files = await vscode.workspace.fs.readDirectory(vscode.Uri.file(dirPath));
return files.map(([name, type]) => {
const filePath = path.join(dirPath, name);
const uri = vscode.Uri.file(filePath);
return new FileTreeItem(
uri,
type === vscode.FileType.Directory
? vscode.TreeItemCollapsibleState.Collapsed
: vscode.TreeItemCollapsibleState.None
);
});
} catch (error) {
console.error(`Error reading directory: ${dirPath}`, error);
return [];
}
}
// Refresh the tree view
refresh(): void {
this._onDidChangeTreeData.fire();
}
// Add a file to the selection
addToSelection(item: FileTreeItem): void {
if (item.resourceUri) {
this.selectedFiles.add(item.resourceUri.fsPath);
console.log(`Added ${item.resourceUri.fsPath} to selection`);
}
}
// Remove a file from the selection
removeFromSelection(item: FileTreeItem): void {
if (item.resourceUri) {
this.selectedFiles.delete(item.resourceUri.fsPath);
console.log(`Removed ${item.resourceUri.fsPath} from selection`);
}
}
// Toggle a file's selection status
toggleSelection(item: FileTreeItem): void {
if (item.resourceUri) {
const filePath = item.resourceUri.fsPath;
if (this.selectedFiles.has(filePath)) {
this.removeFromSelection(item);
} else {
this.addToSelection(item);
}
this.refresh();
}
}
// Get the selected files
getSelectedFiles(): Set<string> {
console.log('getSelectedFiles called, count:', this.selectedFiles.size);
return this.selectedFiles;
}
// Get settings items for the tree view
private getSettingsItems(): SettingTreeItem[] {
return [
new SettingTreeItem('XML Edits', 'xmlEditsEnabled', this.settings.xmlEditsEnabled),
new SettingTreeItem('Include Line Numbers', 'includeLineNumbers', this.settings.includeLineNumbers),
new SettingTreeItem('Include Comments', 'includeComments', this.settings.includeComments),
new SettingTreeItem('Token Calculation', 'tokenCalculationEnabled', this.settings.tokenCalculationEnabled)
];
}
// Update a setting value
updateSetting(key: keyof PrompterSettings, value: boolean): void {
this.settings[key] = value;
this.refresh();
}
// Check if showing settings view
isShowingSettings(): boolean {
return this.showingSettings;
}
// Toggle XML edits setting
toggleXmlEdits(): void {
this.settings.xmlEditsEnabled = !this.settings.xmlEditsEnabled;
this.refresh();
}
// Check if XML edits are enabled
isXmlEditsEnabled(): boolean {
return this.settings.xmlEditsEnabled;
}
// Get all settings
getSettings(): PrompterSettings {
return { ...this.settings };
}
// Update all settings at once
updateSettings(newSettings: PrompterSettings): void {
this.settings = { ...newSettings };
this.refresh();
}
}

21
src/utils/fileReader.ts Normal file
View File

@@ -0,0 +1,21 @@
import * as vscode from 'vscode';
/**
* Utility for reading file contents
*/
export class FileReader {
/**
* Read the content of a file
* @param filePath Path to the file to read
* @returns The file content as a string
*/
static async readFileContent(filePath: string): Promise<string> {
try {
const readData = await vscode.workspace.fs.readFile(vscode.Uri.file(filePath));
return Buffer.from(readData).toString('utf8');
} catch (error) {
console.error(`Error reading file ${filePath}:`, error);
return '';
}
}
}

135
src/utils/promptGenerator.ts Normal file
View File

@@ -0,0 +1,135 @@
import * as vscode from 'vscode';
import * as fs from 'fs';
import * as path from 'path';
import { PrompterSettings } from '../models/settings';
import { FileReader } from './fileReader';
import { TokenEstimator } from './tokenEstimator';
/**
* Utility class for generating prompts from selected files
*/
export class PromptGenerator {
/**
* Generate a prompt from the selected files
* @param selectedFiles Set of file paths to include in the prompt
* @param settings Settings to apply when generating the prompt
* @returns The generated prompt text
*/
static async generatePrompt(selectedFiles: Set<string>, settings: PrompterSettings): Promise<string> {
if (selectedFiles.size === 0) {
throw new Error('No files selected');
}
let totalTokens = 0;
const fileContents = new Map<string, { content: string; tokens: number }>();
// Process each selected file
for (const filePath of selectedFiles) {
const content = await FileReader.readFileContent(filePath);
const tokens = TokenEstimator.estimateTokens(content);
totalTokens += tokens;
fileContents.set(filePath, { content, tokens });
}
// Generate the prompt based on settings
if (settings.xmlEditsEnabled) {
return this.generateXMLPrompt(fileContents, settings);
} else {
return this.generatePlainPrompt(fileContents, settings);
}
}
/**
* Generate a plain text prompt
* @param files Map of file paths to content and token counts
* @param settings Settings to apply
* @returns Plain text prompt
*/
private static generatePlainPrompt(files: Map<string, { content: string; tokens: number }>, settings: PrompterSettings): string {
let promptText = '';
let totalTokenCount = 0;
for (const [filePath, { content, tokens }] of files) {
const fileName = path.basename(filePath);
totalTokenCount += tokens;
// Add the file to the prompt
promptText += `File: ${fileName}\n`;
// Add line numbers if enabled
if (settings.includeLineNumbers) {
const lines = content.split('\n');
for (let i = 0; i < lines.length; i++) {
promptText += `${i + 1}: ${lines[i]}\n`;
}
} else {
promptText += `${content}\n`;
}
promptText += '\n';
}
// Add token count if enabled
if (settings.tokenCalculationEnabled) {
promptText += `\nEstimated token count: ${totalTokenCount}`;
}
return promptText;
}
/**
* Generate an XML formatted prompt
* @param files Map of file paths to content and token counts
* @param settings Settings to apply
* @returns XML formatted prompt
*/
private static generateXMLPrompt(files: Map<string, { content: string; tokens: number }>, settings: PrompterSettings): string {
const xmlParts = ['<?xml version="1.0" encoding="UTF-8"?>', '<prompt>'];
// Add files section
xmlParts.push(' <files>');
for (const [filePath, { content, tokens }] of files) {
const extension = path.extname(filePath);
let language = extension.substring(1); // Remove the dot
// Handle special cases
if (extension === '.js' || extension === '.jsx') {
language = 'javascript';
} else if (extension === '.ts' || extension === '.tsx') {
language = 'typescript';
}
xmlParts.push(' <file>');
xmlParts.push(` <path>${path.basename(filePath)}</path>`);
xmlParts.push(` <type>${language}</type>`);
xmlParts.push(` <tokens>${tokens}</tokens>`);
// Format content based on settings
let formattedContent = content;
if (settings.includeLineNumbers) {
const lines = content.split('\n');
formattedContent = lines.map((line, i) => `${i + 1}: ${line}`).join('\n');
}
xmlParts.push(` <content><![CDATA[${formattedContent}]]></content>`);
xmlParts.push(' </file>');
}
xmlParts.push(' </files>');
// Add options based on settings
xmlParts.push(' <options>');
if (settings.includeLineNumbers) {
xmlParts.push(' <include_line_numbers>true</include_line_numbers>');
}
if (!settings.includeComments) {
xmlParts.push(' <exclude_comments>true</exclude_comments>');
}
if (settings.tokenCalculationEnabled) {
xmlParts.push(` <total_tokens>${Array.from(files.values()).reduce((sum, { tokens }) => sum + tokens, 0)}</total_tokens>`);
}
xmlParts.push(' </options>');
xmlParts.push('</prompt>');
return xmlParts.join('\n');
}
}

16
src/utils/tokenEstimator.ts Normal file
View File

@@ -0,0 +1,16 @@
/**
* Utility for estimating token counts in text
*/
export class TokenEstimator {
/**
* Estimate the number of tokens in a text string
* @param text The text to estimate tokens for
* @returns Estimated token count
*/
static estimateTokens(text: string): number {
// Rough estimation: Split by whitespace and punctuation
// This is a simple approximation, actual token count may vary by model
const words = text.split(/[\s\p{P}]+/u).filter(Boolean);
return Math.ceil(words.length * 1.3); // Add 30% overhead for special tokens
}
}