spiderfoot-web-client.js•14.6 kB
// SpiderFoot Web Client - Interacts with the web interface
import axios from 'axios';
import * as cheerio from 'cheerio';
import { CookieJar } from 'tough-cookie';
import { wrapper as axiosCookieJarSupport } from 'axios-cookiejar-support';
import { createHash } from 'crypto';
const DEFAULT_SPIDERFOOT_URL = 'http://localhost:5001';
class SpiderFootWebClient {
constructor(baseUrl = DEFAULT_SPIDERFOOT_URL) {
this.baseUrl = baseUrl;
this.client = axios.create({
baseURL: baseUrl,
headers: {
'User-Agent': 'SpiderFoot-Web-Client/1.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Content-Type': 'application/x-www-form-urlencoded',
},
maxRedirects: 0,
validateStatus: status => status < 400 || status === 302
});
// Store cookies for session management
this.cookieJar = new CookieJar();
axiosCookieJarSupport(this.client);
this.client.defaults.jar = this.cookieJar;
}
// Update request headers with cookies
updateHeaders() {
if (this.cookieJar) {
this.client.defaults.headers.common['Cookie'] = this.cookieJar.getCookieString(this.baseUrl);
} else {
delete this.client.defaults.headers.common['Cookie'];
}
}
// Extract cookies from response headers
extractCookies(headers) {
if (headers['set-cookie']) {
const cookies = Array.isArray(headers['set-cookie'])
? headers['set-cookie']
: [headers['set-cookie']];
cookies.forEach(cookie => {
this.cookieJar.setCookie(cookie, this.baseUrl);
});
this.updateHeaders();
}
}
// Make a GET request and handle cookies
async get(url, params = {}) {
const response = await this.client.get(url, {
params,
headers: { ...this.client.defaults.headers.common }
});
if (response.headers) {
this.extractCookies(response.headers);
}
return response;
}
// Make a POST request with form data
async postForm(url, data) {
const formData = new URLSearchParams();
Object.entries(data).forEach(([key, value]) => {
formData.append(key, value);
});
const response = await this.client.post(url, formData.toString(), {
headers: {
...this.client.defaults.headers.common,
'Content-Type': 'application/x-www-form-urlencoded'
},
maxRedirects: 0,
validateStatus: null // Allow all status codes
});
if (response.headers) {
this.extractCookies(response.headers);
}
return response;
}
// Get CSRF token from a page
async getCsrfToken(pageUrl = '/newscan') {
try {
console.log(`Fetching CSRF token from ${pageUrl}...`);
const response = await this.get(pageUrl);
// Log the response for debugging
console.log('Response status:', response.status);
console.log('Response headers:', response.headers);
const $ = cheerio.load(response.data);
// Look for CSRF token in meta tags first
let csrfToken = $('meta[name="csrf-token"]').attr('content');
// If not found in meta tags, look for input field
if (!csrfToken) {
csrfToken = $('input[name="csrf_token"]').val();
}
// If still not found, try to extract from the page content
if (!csrfToken) {
const csrfMatch = response.data.match(/name="csrf_token"[^>]*value="([^"]*)"/);
if (csrfMatch && csrfMatch[1]) {
csrfToken = csrfMatch[1];
}
}
if (!csrfToken) {
console.error('CSRF token not found. Page content may have unexpected structure.');
console.error('First 500 chars of response:', response.data.substring(0, 500));
throw new Error('CSRF token not found on the page');
}
console.log('Found CSRF token:', csrfToken);
return csrfToken;
} catch (error) {
console.error('Error getting CSRF token:', error.message);
throw error;
}
}
// Start a new scan
async startScan(target, modules = ['type_DNS_TEXT'], scanType = 'domain', scanName = null) {
try {
console.log('Preparing to start a new scan...');
// First, get the new scan page to extract module and type lists
const newScanPage = await this.get('/newscan');
const $ = cheerio.load(newScanPage.data);
// Get the modulelist and typelist from the hidden inputs
const modulelist = $('input[name="modulelist"]').val() || '';
const typelist = $('input[name="typelist"]').val() || '';
// Get the form element
const form = $('form[action^="/startscan"]');
if (form.length === 0) {
throw new Error('Could not find scan form on the page');
}
const formAction = form.attr('action');
console.log(`Form action: ${formAction}`);
// Get all available modules from the checkboxes in the form
const availableModules = [];
$('input[type="checkbox"][id^="type_"]').each((i, el) => {
const moduleId = $(el).attr('id');
if (moduleId && moduleId.startsWith('type_')) {
availableModules.push(moduleId);
}
});
console.log(`Found ${availableModules.length} available modules`);
// If no modules provided, use the first available one
const modulesToUse = modules && modules.length > 0
? modules.filter(module => availableModules.includes(module) || availableModules.some(m => m.endsWith(`_${module}`)))
: [availableModules[0]];
if (modulesToUse.length === 0) {
console.warn('No valid modules found, using first available module');
modulesToUse.push(availableModules[0]);
}
// Prepare the form data
const formData = new URLSearchParams();
// Add standard form fields
formData.append('scanname', scanName || `scan-${Date.now()}`);
formData.append('scantarget', target);
formData.append('type', scanType);
formData.append('usecase', 'all');
formData.append('modulelist', modulelist);
formData.append('typelist', typelist);
// Add the btn_scan parameter which is required by the form
formData.append('btn_scan', 'Start Scan');
// Add selected modules to form data
// SpiderFoot expects the module IDs as direct parameters with value 'on'
modulesToUse.forEach(module => {
formData.append(module, 'on');
});
console.log(`Starting scan with ${modulesToUse.length} modules:`, modulesToUse);
console.log('Form data:', formData.toString());
// Submit the form with proper headers
const response = await this.client.post(formAction, formData.toString(), {
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Origin': this.baseUrl,
'Referer': `${this.baseUrl}/newscan`,
...this.client.defaults.headers.common
},
maxRedirects: 0,
validateStatus: null // Allow all status codes
});
console.log('Scan submission response status:', response.status);
// Check if the response is a redirect to the scans page
if (response.status === 302 || response.status === 200) {
// Extract scan ID from the response URL or body
let scanId = null;
// Try to get scan ID from Location header if it's a redirect
if (response.headers.location) {
const match = response.headers.location.match(/\/scanevent\?id=([^&]+)/);
if (match) {
scanId = match[1];
}
}
// If no scan ID from redirect, try to parse it from the response body
if (!scanId && response.data) {
const $ = cheerio.load(response.data);
const scanLink = $('a[href^="/scanevent?id="]').first();
if (scanLink.length) {
const href = scanLink.attr('href');
const match = href.match(/\/scanevent\?id=([^&]+)/);
if (match) {
scanId = match[1];
}
}
}
if (scanId) {
console.log(`Scan started successfully with ID: ${scanId}`);
return { success: true, scanId };
} else {
// If we can't find the scan ID, the scan might still have started
// Try to find it by listing all scans and getting the most recent one
try {
const scans = await this.listScans();
if (scans.length > 0) {
const latestScan = scans[0];
console.log(`Could not determine scan ID from response, using latest scan: ${latestScan.id}`);
return { success: true, scanId: latestScan.id };
}
} catch (err) {
console.warn('Could not list scans to find latest scan ID:', err.message);
}
console.warn('Scan might have started, but could not determine scan ID');
return { success: true, message: 'Scan started, but could not determine scan ID' };
}
} else {
// If we get here, the request failed
console.error('Failed to start scan. Response status:', response.status);
// Try to extract error message from response body if it's HTML
let errorMessage = `Server returned status ${response.status}`;
if (response.data && typeof response.data === 'string') {
const $ = cheerio.load(response.data);
const errorDiv = $('.alert.alert-danger');
if (errorDiv.length) {
const errorText = errorDiv.text().trim();
if (errorText) {
errorMessage = errorText.substring(0, 500);
}
}
}
console.error('Error starting scan:', errorMessage);
return {
success: false,
error: errorMessage,
response: {
status: response.status,
statusText: response.statusText,
headers: response.headers,
data: response.data
}
};
}
} catch (error) {
console.error('Error in startScan:', error.message);
console.error(error.stack);
return {
success: false,
error: error.message,
details: error.response ? {
status: error.response.status,
statusText: error.response.statusText,
headers: error.response.headers,
data: error.response.data
} : null
};
}
}
// Get scan status
async getScanStatus(scanId) {
const response = await this.get(`/scaninfo`, { id: scanId });
const $ = cheerio.load(response.data);
// Extract status from the page
const statusText = $('h2').text().trim();
const statusMatch = statusText.match(/Status: (\w+)/i);
const status = statusMatch ? statusMatch[1] : 'UNKNOWN';
// Extract other info
const info = {};
$('table tr').each((i, row) => {
const cols = $(row).find('td');
if (cols.length === 2) {
const key = $(cols[0]).text().trim().replace(':', '').toLowerCase();
const value = $(cols[1]).text().trim();
info[key] = value;
}
});
return { status, ...info };
}
// Get scan results
async getScanResults(scanId, resultType = 'summary') {
const response = await this.get('/scanresults', { id: scanId, type: resultType });
// For simplicity, return the raw HTML
// In a real implementation, you'd parse the HTML to extract the results
return response.data;
}
// List all scans
async listScans() {
const response = await this.get('/');
const $ = cheerio.load(response.data);
const scans = [];
$('table tbody tr').each((i, row) => {
const cols = $(row).find('td');
if (cols.length >= 6) { // Assuming at least 6 columns in the scans table
scans.push({
id: $(cols[0]).text().trim(),
name: $(cols[1]).text().trim(),
target: $(cols[2]).text().trim(),
type: $(cols[3]).text().trim(),
started: $(cols[4]).text().trim(),
status: $(cols[5]).text().trim()
});
}
});
return scans;
}
// Stop a scan
async stopScan(scanId) {
const response = await this.get('/scanstop', { id: scanId });
return response.status === 200 || response.status === 302;
}
// Delete a scan
async deleteScan(scanId) {
const response = await this.get('/scandelete', { id: scanId });
return response.status === 200 || response.status === 302;
}
}
// Example usage
async function testWebClient() {
const client = new SpiderFootWebClient();
try {
console.log('=== Testing SpiderFoot Web Client ===');
// List existing scans
console.log('\n1. Listing existing scans...');
const scans = await client.listScans();
console.log('Existing scans:', scans);
// Start a new scan
console.log('\n2. Starting a new scan...');
const target = 'example.com';
const scanResult = await client.startScan(target, ['sfp_dnsresolve', 'sfp_dnsbrute'], 'domain');
if (scanResult.success && scanResult.scanId) {
console.log(`Scan started with ID: ${scanResult.scanId}`);
// Check scan status
console.log('\n3. Checking scan status...');
const status = await client.getScanStatus(scanResult.scanId);
console.log('Scan status:', status);
// Get scan results after a short delay (in a real app, you'd poll until complete)
if (status.status === 'FINISHED') {
console.log('\n4. Getting scan results...');
const results = await client.getScanResults(scanResult.scanId, 'summary');
console.log('Scan results (first 500 chars):', results.substring(0, 500) + '...');
} else {
console.log('\n4. Scan is still running. In a real app, you would poll until complete.');
}
} else {
console.error('Failed to start scan:', scanResult.error || 'Unknown error');
}
console.log('\n✅ Web client test completed!');
} catch (error) {
console.error('\n❌ Test failed:', error.message);
console.error(error.stack);
process.exit(1);
}
}
// Run the test if this file is executed directly
if (process.argv[1] === new URL(import.meta.url).pathname) {
testWebClient();
}
export default SpiderFootWebClient;