/** * Multifetch, adapted from "@ntlab/sfetch" by BellezaEmporium. * Multiple concurrent fetch with a callback to process the result of each request. * The maximum number of concurrent workers can be configured with `setMaxWorker()`. * By default, the callback will only be called if the request is successful and returns a result. * This behavior can be changed with `setCheckResult()`. * A custom debug function can be set with `setDebugger()`. * Native mock support via `setMocks()` for testing without axios. */ import axios, { AxiosRequestConfig } from 'axios' import fs from 'fs' import path from 'path' interface QueueItem { url: string; method?: string; params?: Record; } type QueueEntry = string | QueueItem; type Callback = (queue: QueueEntry, data: unknown, headers?: unknown) => void; type DebugFn = (format: string, url: string, config: string) => void; type MockHandler = (url: string, config?: AxiosRequestConfig) => unknown; interface MockRoute { handler: MockHandler | string; dataDir?: string; } let nworker = 25 let checkResult = true let debug: DebugFn | undefined const mocks = new Map() /** * Process a mock route and return response */ const processMockRoute = (url: string, config: AxiosRequestConfig | undefined, route: MockRoute): unknown => { const { handler, dataDir } = route if (typeof handler === 'function') { return handler(url, config) } if (typeof handler === 'string') { // If it looks like a file path, read from disk if (handler.includes('.') || handler.includes('/')) { const filePath = dataDir ? path.join(dataDir, handler) : handler const content = fs.readFileSync(filePath, 'utf8') if (handler.endsWith('.json')) { try { return JSON.parse(content) } catch { return content } } return content } // Otherwise return as-is return handler } return handler } /** * Check if a URL matches any mock pattern */ const findMock = (url: string): MockRoute | undefined => { // Try exact match first if (mocks.has(url)) return mocks.get(url) // Try pattern matching (prefix match) for (const [pattern, route] of mocks) { if (url.startsWith(pattern)) return route } return undefined } async function doFetch(queues: QueueEntry[], cb: Callback) { if (!queues.length) return let resolveFinish: (() => void) | undefined const workers = new Set<() => void>() let activeWorkers = 0 const processQueue = () => { if (queues.length > 0 && activeWorkers < nworker) { const queue = queues.shift() if (queue === undefined) return activeWorkers++ const processRequest = async () => { try { const isQueueObject = typeof queue === 'object' && queue !== null const url = isQueueObject ? queue.url : (queue as string) const method = (isQueueObject && queue.method) ? queue.method : 'get' const params = (isQueueObject && queue.params) ? queue.params : {} const requestConfig: AxiosRequestConfig = method === 'request' ? { ...params, url } : { ...params, url, method: method as AxiosRequestConfig['method'] } if (debug) { debug('fetch %s with %s', url, JSON.stringify(requestConfig)) } // Check if there's a mock for this URL const mock = findMock(url) let response: unknown if (mock) { const mockResponse = processMockRoute(url, requestConfig, mock) const isObj = typeof mockResponse === 'object' && mockResponse !== null if (debug) { const hasData = isObj && 'data' in mockResponse const keys = isObj ? Object.keys(mockResponse).join(',') : '' debug(`mock response type: ${typeof mockResponse}, has data: ${hasData}, keys: ${keys}`, url, JSON.stringify(requestConfig)) } // Check if response looks like it's already formatted (has 'data' and optionally 'status'/'headers') const isFormatted = isObj && 'data' in mockResponse && 'status' in mockResponse response = isFormatted ? mockResponse : { data: mockResponse, status: 200, headers: {} } } else if (mocks.size > 0) { // If mocks are set up but this URL doesn't match, return 404 response = { data: '', status: 404, headers: {} } } else { // No mocks set up, use real axios const axMethod = (requestConfig.method || 'get').toLowerCase() if (axMethod === 'get' && typeof axios.get === 'function') { response = await axios.get(url, requestConfig) } else if (axMethod === 'post' && typeof axios.post === 'function') { response = await axios.post(url, requestConfig.data, requestConfig) } else { response = await axios.request(requestConfig) } } const res = response as { data?: unknown; headers?: unknown } | undefined if ((checkResult && res?.data) || !checkResult) { cb(queue, res?.data, res?.headers) } } catch (err: unknown) { const url = typeof queue === 'object' ? queue.url : queue const errorMessage = err instanceof Error ? err.message : String(err) console.error(`Unable to fetch ${url}: ${errorMessage}!`) if (!checkResult) { cb(queue, undefined) } } finally { activeWorkers-- workers.delete(processRequest) processQueue() if (activeWorkers === 0 && queues.length === 0 && resolveFinish) { resolveFinish() } } } workers.add(processRequest) processRequest() } } // Start initial workers const initialWorkers = Math.min(nworker, queues.length) for (let i = 0; i < initialWorkers; i++) { processQueue() } // Wait for all to complete if (workers.size > 0 || activeWorkers > 0) { await new Promise(resolve => { resolveFinish = resolve }) } } Object.assign(doFetch, { getMaxWorker() { return nworker }, setMaxWorker(n: number) { nworker = n return doFetch }, getCheckResult() { return checkResult }, setCheckResult(enabled: boolean) { checkResult = enabled return doFetch }, setDebugger(dbg: (arg0: string, arg1: unknown, arg2: string) => void) { debug = dbg return doFetch }, /** * Set mocks for URLs (for testing) * @param mockConfig - Object with URL patterns as keys and handlers as values * @param dataDir - Optional directory for resolving file paths in handlers * * Usage: * multifetch.setMocks({ * 'https://example.com/api': (url) => ({ data: 'response' }), * 'https://example.com/file': 'response.json' * }, __dirname) */ setMocks(mockConfig: Record, dataDir?: string) { mocks.clear() for (const [url, handler] of Object.entries(mockConfig)) { mocks.set(url, { handler, dataDir }) } return doFetch }, /** * Add a single mock route */ addMock(url: string, handler: MockHandler | string, dataDir?: string) { mocks.set(url, { handler, dataDir }) return doFetch }, /** * Clear all mocks */ clearMocks() { mocks.clear() return doFetch }, /** * Get registered mock URLs for debugging */ getMocks() { return Array.from(mocks.keys()) } }) export default doFetch module.exports = doFetch