Change tokenk Axios wrapper to multifetch, config changes

scripts/core/index.ts

@@ -1,2 +1,3 @@
 export * from './htmlTable'
 export * from './utils'
+export { default as multifetch } from './multifetch'

scripts/core/multifetch.ts

@@ -0,0 +1,243 @@
+/**
+ * Multifetch, adapted from "@ntlab/sfetch" by BellezaEmporium.
+ * Multiple concurrent fetch with a callback to process the result of each request.
+ * The maximum number of concurrent workers can be configured with `setMaxWorker()`.
+ * By default, the callback will only be called if the request is successful and returns a result.
+ * This behavior can be changed with `setCheckResult()`.
+ * A custom debug function can be set with `setDebugger()`.
+ * Native mock support via `setMocks()` for testing without axios.
+ */
+
+import axios, { AxiosRequestConfig } from 'axios'
+import fs from 'fs'
+import path from 'path'
+
+interface QueueItem {
+    url: string;
+    method?: string;
+    params?: Record<string, unknown>;
+}
+
+type QueueEntry = string | QueueItem;
+type Callback = (queue: QueueEntry, data: unknown, headers?: unknown) => void;
+type DebugFn = (format: string, url: string, config: string) => void;
+type MockHandler = (url: string, config?: AxiosRequestConfig) => unknown;
+
+interface MockRoute {
+    handler: MockHandler | string;
+    dataDir?: string;
+}
+
+let nworker = 25
+let checkResult = true
+let debug: DebugFn | undefined
+const mocks = new Map<string, MockRoute>()
+
+/**
+ * Process a mock route and return response
+ */
+const processMockRoute = (url: string, config: AxiosRequestConfig | undefined, route: MockRoute): unknown => {
+    const { handler, dataDir } = route
+    
+    if (typeof handler === 'function') {
+        return handler(url, config)
+    }
+    
+    if (typeof handler === 'string') {
+        // If it looks like a file path, read from disk
+        if (handler.includes('.') || handler.includes('/')) {
+            const filePath = dataDir ? path.join(dataDir, handler) : handler
+            const content = fs.readFileSync(filePath, 'utf8')
+            if (handler.endsWith('.json')) {
+                try {
+                    return JSON.parse(content)
+                } catch {
+                    return content
+                }
+            }
+            return content
+        }
+        // Otherwise return as-is
+        return handler
+    }
+    
+    return handler
+}
+
+/**
+ * Check if a URL matches any mock pattern
+ */
+const findMock = (url: string): MockRoute | undefined => {
+    // Try exact match first
+    if (mocks.has(url)) return mocks.get(url)
+    
+    // Try pattern matching (prefix match)
+    for (const [pattern, route] of mocks) {
+        if (url.startsWith(pattern)) return route
+    }
+    return undefined
+}
+
+async function doFetch(queues: QueueEntry[], cb: Callback) {
+    if (!queues.length) return
+
+    let resolveFinish: (() => void) | undefined
+    const workers = new Set<() => void>()
+    let activeWorkers = 0
+
+    const processQueue = () => {
+        if (queues.length > 0 && activeWorkers < nworker) {
+            const queue = queues.shift()
+            if (queue === undefined) return
+            
+            activeWorkers++
+
+            const processRequest = async () => {
+                try {
+                    const isQueueObject = typeof queue === 'object' && queue !== null
+                    const url = isQueueObject ? queue.url : (queue as string)
+                    const method = (isQueueObject && queue.method) ? queue.method : 'get'
+                    const params = (isQueueObject && queue.params) ? queue.params : {}
+                    
+                    const requestConfig: AxiosRequestConfig = method === 'request'
+                        ? { ...params, url }
+                        : { ...params, url, method: method as AxiosRequestConfig['method'] }
+
+                    if (debug) {
+                        debug('fetch %s with %s', url, JSON.stringify(requestConfig))
+                    }
+
+                    // Check if there's a mock for this URL
+                    const mock = findMock(url)
+                    let response: unknown
+
+                    if (mock) {
+                        const mockResponse = processMockRoute(url, requestConfig, mock)
+                        
+                        const isObj = typeof mockResponse === 'object' && mockResponse !== null
+                        if (debug) {
+                            const hasData = isObj && 'data' in mockResponse
+                            const keys = isObj ? Object.keys(mockResponse).join(',') : ''
+                            debug(`mock response type: ${typeof mockResponse}, has data: ${hasData}, keys: ${keys}`, url, JSON.stringify(requestConfig))
+                        }
+
+                        // Check if response looks like it's already formatted (has 'data' and optionally 'status'/'headers')
+                        const isFormatted = isObj && 'data' in mockResponse && 'status' in mockResponse
+                        response = isFormatted ? mockResponse : { data: mockResponse, status: 200, headers: {} }
+                    } else if (mocks.size > 0) {
+                        // If mocks are set up but this URL doesn't match, return 404
+                        response = { data: '', status: 404, headers: {} }
+                    } else {
+                        // No mocks set up, use real axios
+                        const axMethod = (requestConfig.method || 'get').toLowerCase()
+                        if (axMethod === 'get' && typeof axios.get === 'function') {
+                            response = await axios.get(url, requestConfig)
+                        } else if (axMethod === 'post' && typeof axios.post === 'function') {
+                            response = await axios.post(url, requestConfig.data, requestConfig)
+                        } else {
+                            response = await axios.request(requestConfig)
+                        }
+                    }
+                    
+                    const res = response as { data?: unknown; headers?: unknown } | undefined
+                    if ((checkResult && res?.data) || !checkResult) {
+                        cb(queue, res?.data, res?.headers)
+                    }
+                } catch (err: unknown) {
+                    const url = typeof queue === 'object' ? queue.url : queue
+                    const errorMessage = err instanceof Error ? err.message : String(err)
+                    console.error(`Unable to fetch ${url}: ${errorMessage}!`)
+                    if (!checkResult) {
+                        cb(queue, undefined)
+                    }
+                } finally {
+                    activeWorkers--
+                    workers.delete(processRequest)
+                    processQueue()
+                    
+                    if (activeWorkers === 0 && queues.length === 0 && resolveFinish) {
+                        resolveFinish()
+                    }
+                }
+            }
+
+            workers.add(processRequest)
+            processRequest()
+        }
+    }
+
+    // Start initial workers
+    const initialWorkers = Math.min(nworker, queues.length)
+    for (let i = 0; i < initialWorkers; i++) {
+        processQueue()
+    }
+
+    // Wait for all to complete
+    if (workers.size > 0 || activeWorkers > 0) {
+        await new Promise<void>(resolve => {
+            resolveFinish = resolve
+        })
+    }
+}
+
+Object.assign(doFetch, {
+    getMaxWorker() {
+        return nworker
+    },
+    setMaxWorker(n: number) {
+        nworker = n
+        return doFetch
+    },
+    getCheckResult() {
+        return checkResult
+    },
+    setCheckResult(enabled: boolean) {
+        checkResult = enabled
+        return doFetch
+    },
+    setDebugger(dbg: (arg0: string, arg1: unknown, arg2: string) => void) {
+        debug = dbg
+        return doFetch
+    },
+    /**
+     * Set mocks for URLs (for testing)
+     * @param mockConfig - Object with URL patterns as keys and handlers as values
+     * @param dataDir - Optional directory for resolving file paths in handlers
+     * 
+     * Usage:
+     *   multifetch.setMocks({
+     *     'https://example.com/api': (url) => ({ data: 'response' }),
+     *     'https://example.com/file': 'response.json'
+     *   }, __dirname)
+     */
+    setMocks(mockConfig: Record<string, MockHandler | string>, dataDir?: string) {
+        mocks.clear()
+        for (const [url, handler] of Object.entries(mockConfig)) {
+            mocks.set(url, { handler, dataDir })
+        }
+        return doFetch
+    },
+    /**
+     * Add a single mock route
+     */
+    addMock(url: string, handler: MockHandler | string, dataDir?: string) {
+        mocks.set(url, { handler, dataDir })
+        return doFetch
+    },
+    /**
+     * Clear all mocks
+     */
+    clearMocks() {
+        mocks.clear()
+        return doFetch
+    },
+    /**
+     * Get registered mock URLs for debugging
+     */
+    getMocks() {
+        return Array.from(mocks.keys())
+    }
+})
+
+export default doFetch
+module.exports = doFetch

scripts/core/utils.ts

@@ -113,4 +113,4 @@ export function parseNumber(value: string): number {
 
 export function parseList(value: string): string[] {
   return value.split(',')
-}
+}