diff --git a/scripts/core/multifetch.ts b/scripts/core/multifetch.ts index 02196d8c3..f3eddd7b8 100644 --- a/scripts/core/multifetch.ts +++ b/scripts/core/multifetch.ts @@ -1,243 +1,243 @@ -/** - * Multifetch, adapted from "@ntlab/sfetch" by BellezaEmporium. - * Multiple concurrent fetch with a callback to process the result of each request. - * The maximum number of concurrent workers can be configured with `setMaxWorker()`. - * By default, the callback will only be called if the request is successful and returns a result. - * This behavior can be changed with `setCheckResult()`. - * A custom debug function can be set with `setDebugger()`. - * Native mock support via `setMocks()` for testing without axios. - */ - -import axios, { AxiosRequestConfig } from 'axios' -import fs from 'fs' -import path from 'path' - -interface QueueItem { - url: string; - method?: string; - params?: Record; -} - -type QueueEntry = string | QueueItem; -type Callback = (queue: QueueEntry, data: unknown, headers?: unknown) => void; -type DebugFn = (format: string, url: string, config: string) => void; -type MockHandler = (url: string, config?: AxiosRequestConfig) => unknown; - -interface MockRoute { - handler: MockHandler | string; - dataDir?: string; -} - -let nworker = 25 -let checkResult = true -let debug: DebugFn | undefined -const mocks = new Map() - -/** - * Process a mock route and return response - */ -const processMockRoute = (url: string, config: AxiosRequestConfig | undefined, route: MockRoute): unknown => { - const { handler, dataDir } = route - - if (typeof handler === 'function') { - return handler(url, config) - } - - if (typeof handler === 'string') { - // If it looks like a file path, read from disk - if (handler.includes('.') || handler.includes('/')) { - const filePath = dataDir ? path.join(dataDir, handler) : handler - const content = fs.readFileSync(filePath, 'utf8') - if (handler.endsWith('.json')) { - try { - return JSON.parse(content) - } catch { - return content - } - } - return content - } - // Otherwise return as-is - return handler - } - - return handler -} - -/** - * Check if a URL matches any mock pattern - */ -const findMock = (url: string): MockRoute | undefined => { - // Try exact match first - if (mocks.has(url)) return mocks.get(url) - - // Try pattern matching (prefix match) - for (const [pattern, route] of mocks) { - if (url.startsWith(pattern)) return route - } - return undefined -} - -async function doFetch(queues: QueueEntry[], cb: Callback) { - if (!queues.length) return - - let resolveFinish: (() => void) | undefined - const workers = new Set<() => void>() - let activeWorkers = 0 - - const processQueue = () => { - if (queues.length > 0 && activeWorkers < nworker) { - const queue = queues.shift() - if (queue === undefined) return - - activeWorkers++ - - const processRequest = async () => { - try { - const isQueueObject = typeof queue === 'object' && queue !== null - const url = isQueueObject ? queue.url : (queue as string) - const method = (isQueueObject && queue.method) ? queue.method : 'get' - const params = (isQueueObject && queue.params) ? queue.params : {} - - const requestConfig: AxiosRequestConfig = method === 'request' - ? { ...params, url } - : { ...params, url, method: method as AxiosRequestConfig['method'] } - - if (debug) { - debug('fetch %s with %s', url, JSON.stringify(requestConfig)) - } - - // Check if there's a mock for this URL - const mock = findMock(url) - let response: unknown - - if (mock) { - const mockResponse = processMockRoute(url, requestConfig, mock) - - const isObj = typeof mockResponse === 'object' && mockResponse !== null - if (debug) { - const hasData = isObj && 'data' in mockResponse - const keys = isObj ? Object.keys(mockResponse).join(',') : '' - debug(`mock response type: ${typeof mockResponse}, has data: ${hasData}, keys: ${keys}`, url, JSON.stringify(requestConfig)) - } - - // Check if response looks like it's already formatted (has 'data' and optionally 'status'/'headers') - const isFormatted = isObj && 'data' in mockResponse && 'status' in mockResponse - response = isFormatted ? mockResponse : { data: mockResponse, status: 200, headers: {} } - } else if (mocks.size > 0) { - // If mocks are set up but this URL doesn't match, return 404 - response = { data: '', status: 404, headers: {} } - } else { - // No mocks set up, use real axios - const axMethod = (requestConfig.method || 'get').toLowerCase() - if (axMethod === 'get' && typeof axios.get === 'function') { - response = await axios.get(url, requestConfig) - } else if (axMethod === 'post' && typeof axios.post === 'function') { - response = await axios.post(url, requestConfig.data, requestConfig) - } else { - response = await axios.request(requestConfig) - } - } - - const res = response as { data?: unknown; headers?: unknown } | undefined - if ((checkResult && res?.data) || !checkResult) { - cb(queue, res?.data, res?.headers) - } - } catch (err: unknown) { - const url = typeof queue === 'object' ? queue.url : queue - const errorMessage = err instanceof Error ? err.message : String(err) - console.error(`Unable to fetch ${url}: ${errorMessage}!`) - if (!checkResult) { - cb(queue, undefined) - } - } finally { - activeWorkers-- - workers.delete(processRequest) - processQueue() - - if (activeWorkers === 0 && queues.length === 0 && resolveFinish) { - resolveFinish() - } - } - } - - workers.add(processRequest) - processRequest() - } - } - - // Start initial workers - const initialWorkers = Math.min(nworker, queues.length) - for (let i = 0; i < initialWorkers; i++) { - processQueue() - } - - // Wait for all to complete - if (workers.size > 0 || activeWorkers > 0) { - await new Promise(resolve => { - resolveFinish = resolve - }) - } -} - -Object.assign(doFetch, { - getMaxWorker() { - return nworker - }, - setMaxWorker(n: number) { - nworker = n - return doFetch - }, - getCheckResult() { - return checkResult - }, - setCheckResult(enabled: boolean) { - checkResult = enabled - return doFetch - }, - setDebugger(dbg: (arg0: string, arg1: unknown, arg2: string) => void) { - debug = dbg - return doFetch - }, - /** - * Set mocks for URLs (for testing) - * @param mockConfig - Object with URL patterns as keys and handlers as values - * @param dataDir - Optional directory for resolving file paths in handlers - * - * Usage: - * multifetch.setMocks({ - * 'https://example.com/api': (url) => ({ data: 'response' }), - * 'https://example.com/file': 'response.json' - * }, __dirname) - */ - setMocks(mockConfig: Record, dataDir?: string) { - mocks.clear() - for (const [url, handler] of Object.entries(mockConfig)) { - mocks.set(url, { handler, dataDir }) - } - return doFetch - }, - /** - * Add a single mock route - */ - addMock(url: string, handler: MockHandler | string, dataDir?: string) { - mocks.set(url, { handler, dataDir }) - return doFetch - }, - /** - * Clear all mocks - */ - clearMocks() { - mocks.clear() - return doFetch - }, - /** - * Get registered mock URLs for debugging - */ - getMocks() { - return Array.from(mocks.keys()) - } -}) - -export default doFetch +/** + * Multifetch, adapted from "@ntlab/sfetch" by BellezaEmporium. + * Multiple concurrent fetch with a callback to process the result of each request. + * The maximum number of concurrent workers can be configured with `setMaxWorker()`. + * By default, the callback will only be called if the request is successful and returns a result. + * This behavior can be changed with `setCheckResult()`. + * A custom debug function can be set with `setDebugger()`. + * Native mock support via `setMocks()` for testing without axios. + */ + +import axios, { AxiosRequestConfig } from 'axios' +import fs from 'fs' +import path from 'path' + +interface QueueItem { + url: string; + method?: string; + params?: Record; +} + +type QueueEntry = string | QueueItem; +type Callback = (queue: QueueEntry, data: unknown, headers?: unknown) => void; +type DebugFn = (format: string, url: string, config: string) => void; +type MockHandler = (url: string, config?: AxiosRequestConfig) => unknown; + +interface MockRoute { + handler: MockHandler | string; + dataDir?: string; +} + +let nworker = 25 +let checkResult = true +let debug: DebugFn | undefined +const mocks = new Map() + +/** + * Process a mock route and return response + */ +const processMockRoute = (url: string, config: AxiosRequestConfig | undefined, route: MockRoute): unknown => { + const { handler, dataDir } = route + + if (typeof handler === 'function') { + return handler(url, config) + } + + if (typeof handler === 'string') { + // If it looks like a file path, read from disk + if (handler.includes('.') || handler.includes('/')) { + const filePath = dataDir ? path.join(dataDir, handler) : handler + const content = fs.readFileSync(filePath, 'utf8') + if (handler.endsWith('.json')) { + try { + return JSON.parse(content) + } catch { + return content + } + } + return content + } + // Otherwise return as-is + return handler + } + + return handler +} + +/** + * Check if a URL matches any mock pattern + */ +const findMock = (url: string): MockRoute | undefined => { + // Try exact match first + if (mocks.has(url)) return mocks.get(url) + + // Try pattern matching (prefix match) + for (const [pattern, route] of mocks) { + if (url.startsWith(pattern)) return route + } + return undefined +} + +async function doFetch(queues: QueueEntry[], cb: Callback) { + if (!queues.length) return + + let resolveFinish: (() => void) | undefined + const workers = new Set<() => void>() + let activeWorkers = 0 + + const processQueue = () => { + if (queues.length > 0 && activeWorkers < nworker) { + const queue = queues.shift() + if (queue === undefined) return + + activeWorkers++ + + const processRequest = async () => { + try { + const isQueueObject = typeof queue === 'object' && queue !== null + const url = isQueueObject ? queue.url : (queue as string) + const method = (isQueueObject && queue.method) ? queue.method : 'get' + const params = (isQueueObject && queue.params) ? queue.params : {} + + const requestConfig: AxiosRequestConfig = method === 'request' + ? { ...params, url } + : { ...params, url, method: method as AxiosRequestConfig['method'] } + + if (debug) { + debug('fetch %s with %s', url, JSON.stringify(requestConfig)) + } + + // Check if there's a mock for this URL + const mock = findMock(url) + let response: unknown + + if (mock) { + const mockResponse = processMockRoute(url, requestConfig, mock) + + const isObj = typeof mockResponse === 'object' && mockResponse !== null + if (debug) { + const hasData = isObj && 'data' in mockResponse + const keys = isObj ? Object.keys(mockResponse).join(',') : '' + debug(`mock response type: ${typeof mockResponse}, has data: ${hasData}, keys: ${keys}`, url, JSON.stringify(requestConfig)) + } + + // Check if response looks like it's already formatted (has 'data' and optionally 'status'/'headers') + const isFormatted = isObj && 'data' in mockResponse && 'status' in mockResponse + response = isFormatted ? mockResponse : { data: mockResponse, status: 200, headers: {} } + } else if (mocks.size > 0) { + // If mocks are set up but this URL doesn't match, return 404 + response = { data: '', status: 404, headers: {} } + } else { + // No mocks set up, use real axios + const axMethod = (requestConfig.method || 'get').toLowerCase() + if (axMethod === 'get' && typeof axios.get === 'function') { + response = await axios.get(url, requestConfig) + } else if (axMethod === 'post' && typeof axios.post === 'function') { + response = await axios.post(url, requestConfig.data, requestConfig) + } else { + response = await axios.request(requestConfig) + } + } + + const res = response as { data?: unknown; headers?: unknown } | undefined + if ((checkResult && res?.data) || !checkResult) { + cb(queue, res?.data, res?.headers) + } + } catch (err: unknown) { + const url = typeof queue === 'object' ? queue.url : queue + const errorMessage = err instanceof Error ? err.message : String(err) + console.error(`Unable to fetch ${url}: ${errorMessage}!`) + if (!checkResult) { + cb(queue, undefined) + } + } finally { + activeWorkers-- + workers.delete(processRequest) + processQueue() + + if (activeWorkers === 0 && queues.length === 0 && resolveFinish) { + resolveFinish() + } + } + } + + workers.add(processRequest) + processRequest() + } + } + + // Start initial workers + const initialWorkers = Math.min(nworker, queues.length) + for (let i = 0; i < initialWorkers; i++) { + processQueue() + } + + // Wait for all to complete + if (workers.size > 0 || activeWorkers > 0) { + await new Promise(resolve => { + resolveFinish = resolve + }) + } +} + +Object.assign(doFetch, { + getMaxWorker() { + return nworker + }, + setMaxWorker(n: number) { + nworker = n + return doFetch + }, + getCheckResult() { + return checkResult + }, + setCheckResult(enabled: boolean) { + checkResult = enabled + return doFetch + }, + setDebugger(dbg: (arg0: string, arg1: unknown, arg2: string) => void) { + debug = dbg + return doFetch + }, + /** + * Set mocks for URLs (for testing) + * @param mockConfig - Object with URL patterns as keys and handlers as values + * @param dataDir - Optional directory for resolving file paths in handlers + * + * Usage: + * multifetch.setMocks({ + * 'https://example.com/api': (url) => ({ data: 'response' }), + * 'https://example.com/file': 'response.json' + * }, __dirname) + */ + setMocks(mockConfig: Record, dataDir?: string) { + mocks.clear() + for (const [url, handler] of Object.entries(mockConfig)) { + mocks.set(url, { handler, dataDir }) + } + return doFetch + }, + /** + * Add a single mock route + */ + addMock(url: string, handler: MockHandler | string, dataDir?: string) { + mocks.set(url, { handler, dataDir }) + return doFetch + }, + /** + * Clear all mocks + */ + clearMocks() { + mocks.clear() + return doFetch + }, + /** + * Get registered mock URLs for debugging + */ + getMocks() { + return Array.from(mocks.keys()) + } +}) + +export default doFetch module.exports = doFetch \ No newline at end of file