Use simultaneous fetch helper.

- Introduce fetch helper for site to fetch guide simultaneously.
- Clean-up unused requires.
- Updated tv.yandex.ru cookies.

Signed-off-by: Toha <tohenk@yahoo.com>
This commit is contained in:
Toha
2024-12-17 10:24:58 +07:00
parent f00d53cb7b
commit 916b5f5234
12 changed files with 273 additions and 429 deletions

View File

@@ -1,11 +1,12 @@
const cheerio = require('cheerio')
const dayjs = require('dayjs')
const utc = require('dayjs/plugin/utc')
const doFetch = require('../../scripts/core/fetch')
const debug = require('debug')('site:sky.com')
dayjs.extend(utc)
const nworker = 10
doFetch.setDebugger(debug)
module.exports = {
site: 'sky.com',
@@ -78,64 +79,3 @@ module.exports = {
return Object.values(channels)
}
}
async function doFetch(queues, cb) {
const axios = require('axios')
let n = Math.min(nworker, queues.length)
const workers = []
const adjustWorker = () => {
if (queues.length > workers.length && workers.length < nworker) {
let nw = Math.min(nworker, queues.length)
if (n < nw) {
n = nw
createWorker()
}
}
}
const createWorker = () => {
while (workers.length < n) {
startWorker()
}
}
const startWorker = () => {
const worker = () => {
if (queues.length) {
const queue = queues.shift()
const done = (res, headers) => {
if (res) {
cb(queue, res, headers)
adjustWorker()
}
worker()
}
const url = typeof queue === 'string' ? queue : queue.u
const params = typeof queue === 'object' && queue.params ? queue.params : {}
const method = typeof queue === 'object' && queue.m ? queue.m : 'get'
if (typeof debug === 'function') {
debug(`fetch %s with %s`, url, JSON.stringify(params))
}
axios[method](url, params)
.then(response => {
done(response.data, response.headers)
})
.catch(err => {
console.error(`Unable to fetch ${url}: ${err.message}!`)
done()
})
} else {
workers.splice(workers.indexOf(worker), 1)
}
}
workers.push(worker)
worker()
}
createWorker()
await new Promise(resolve => {
const interval = setInterval(() => {
if (workers.length === 0) {
clearInterval(interval)
resolve()
}
}, 500)
})
}