#!/usr/bin/env node
/**
* Token Count CLI
*
* Standalone CLI for token counting using the TokenMeasurementEngine.
* This provides direct access to the counting engine without requiring
* the MCP server to be running.
*
* Usage:
* token-count --text "Your text here"
* token-count --file path/to/file.txt
* token-count --files file1.txt file2.txt file3.txt
* echo "text" | token-count --stdin
*
* Options:
* --format json|simple Output format (default: json)
* --help Show help
*/
import fs from 'fs/promises';
import { TokenMeasurementEngine } from './TokenMeasurementEngine.js';
const engine = new TokenMeasurementEngine();
function showHelp() {
console.log(`
Token Count CLI - Direct access to token-analyzer engine
Usage:
token-count --text "Your text here"
token-count --file path/to/file.txt
token-count --files file1.txt file2.txt
echo "text" | token-count --stdin
Options:
--text <string> Count tokens in text string
--file <path> Count tokens in single file
--files <paths...> Count tokens in multiple files
--stdin Read from stdin
--format json|simple Output format (default: json)
--help Show this help
Examples:
token-count --text "Hello world" --format simple
token-count --file README.md
token-count --files src/*.ts --format json
cat document.md | token-count --stdin
`);
}
function analyzeText(text, label = 'text') {
const tokens = engine.countTokens(text);
const charCount = text.length;
const wordCount = text.split(/\s+/).filter(w => w.length > 0).length;
const lineCount = text.split('\n').length;
return {
label,
tokens,
characters: charCount,
words: wordCount,
lines: lineCount,
avg_chars_per_token: tokens > 0 ? Math.round((charCount / tokens) * 100) / 100 : 0
};
}
async function analyzeFile(filePath) {
try {
const content = await fs.readFile(filePath, 'utf-8');
const result = analyzeText(content, filePath);
result.file = filePath;
return result;
} catch (error) {
return {
file: filePath,
error: error.message
};
}
}
async function readStdin() {
const chunks = [];
for await (const chunk of process.stdin) {
chunks.push(chunk);
}
return Buffer.concat(chunks).toString('utf-8');
}
async function main() {
const args = process.argv.slice(2);
if (args.length === 0 || args.includes('--help') || args.includes('-h')) {
showHelp();
process.exit(0);
}
// Parse arguments
let mode = null;
let input = null;
let format = 'json';
const files = [];
for (let i = 0; i < args.length; i++) {
const arg = args[i];
switch (arg) {
case '--text':
mode = 'text';
input = args[++i];
break;
case '--file':
mode = 'file';
input = args[++i];
break;
case '--files':
mode = 'files';
// Collect all remaining args until next flag
i++;
while (i < args.length && !args[i].startsWith('--')) {
files.push(args[i]);
i++;
}
i--; // Back up one since loop will increment
break;
case '--stdin':
mode = 'stdin';
break;
case '--format':
format = args[++i];
break;
}
}
let output;
try {
switch (mode) {
case 'text':
if (!input) {
console.error('Error: No text provided');
process.exit(1);
}
output = { input_type: 'text', result: analyzeText(input) };
break;
case 'file':
if (!input) {
console.error('Error: No file path provided');
process.exit(1);
}
output = { input_type: 'file', result: await analyzeFile(input) };
break;
case 'files':
if (files.length === 0) {
console.error('Error: No files provided');
process.exit(1);
}
const results = await Promise.all(files.map(f => analyzeFile(f)));
const totalTokens = results.reduce((sum, r) => sum + (r.tokens || 0), 0);
const successful = results.filter(r => !r.error).length;
output = {
input_type: 'files',
results,
summary: {
total_files: files.length,
successful,
failed: files.length - successful,
total_tokens: totalTokens
}
};
break;
case 'stdin':
const stdinContent = await readStdin();
output = { input_type: 'stdin', result: analyzeText(stdinContent, 'stdin') };
break;
default:
console.error('Error: No input mode specified. Use --text, --file, --files, or --stdin');
process.exit(1);
}
// Output results
if (format === 'simple') {
if (output.result) {
console.log(output.result.tokens);
} else if (output.summary) {
console.log(output.summary.total_tokens);
}
} else {
console.log(JSON.stringify(output, null, 2));
}
} catch (error) {
console.error(`Error: ${error.message}`);
process.exit(1);
}
}
main();