From 2d068686481920dc29ff161155c54f8e87f73ceb Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Sat, 1 Nov 2025 11:40:15 +0300 Subject: [PATCH 1/5] Update dependencies --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index f19d445be..583221253 100644 --- a/package-lock.json +++ b/package-lock.json @@ -51,7 +51,7 @@ "curl-generator": "^0.5.0", "cwait": "^1.1.2", "dayjs": "^1.11.13", - "epg-grabber": "^0.45.0", + "epg-grabber": "^0.46.0", "epg-parser": "^0.3.1", "eslint": "^9.32.0", "eslint-config-prettier": "^10.1.8", @@ -5477,9 +5477,9 @@ } }, "node_modules/epg-grabber": { - "version": "0.45.0", - "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.45.0.tgz", - "integrity": "sha512-GqjXRYOJcC3mX9OYdZHlpNEhwBbCF256uKGPXdUodUjkXTmraolIEXYRUeQJmPbL5/do766EewNwIBXqLzHkJA==", + "version": "0.46.0", + "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.46.0.tgz", + "integrity": "sha512-35VJZWV1tJLro5SNP9bobcVB4fWK9rTUYmtRJ1/wIOA/tktxlCvZ+n/6LoKWS89YtKujqq3jid3cVf7JSImKzA==", "dependencies": { "@freearhey/core": "^0.14.0", "@types/bluebird": "^3.5.42", @@ -14999,9 +14999,9 @@ "integrity": "sha512-oYp7156SP8LkeGD0GF85ad1X9Ai79WtRsZ2gxJqtBuzH+98YUV6jkHEKlZkMbcrjJjIVJNIDP/3WL9wQkoPbWA==" }, "epg-grabber": { - "version": "0.45.0", - "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.45.0.tgz", - "integrity": "sha512-GqjXRYOJcC3mX9OYdZHlpNEhwBbCF256uKGPXdUodUjkXTmraolIEXYRUeQJmPbL5/do766EewNwIBXqLzHkJA==", + "version": "0.46.0", + "resolved": "https://registry.npmjs.org/epg-grabber/-/epg-grabber-0.46.0.tgz", + "integrity": "sha512-35VJZWV1tJLro5SNP9bobcVB4fWK9rTUYmtRJ1/wIOA/tktxlCvZ+n/6LoKWS89YtKujqq3jid3cVf7JSImKzA==", "requires": { "@freearhey/core": "^0.14.0", "@types/bluebird": "^3.5.42", diff --git a/package.json b/package.json index e58062a9d..17b677beb 100644 --- a/package.json +++ b/package.json @@ -81,7 +81,7 @@ "curl-generator": "^0.5.0", "cwait": "^1.1.2", "dayjs": "^1.11.13", - "epg-grabber": "^0.45.0", + "epg-grabber": "^0.46.0", "epg-parser": "^0.3.1", "eslint": "^9.32.0", "eslint-config-prettier": "^10.1.8", From 1f99fbfffe77dbbc0e8f98bad8abe8353a3dcfa2 Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Sat, 1 Nov 2025 12:01:29 +0300 Subject: [PATCH 2/5] Update tests/__data__ --- .../epg_grab/custom_channels.guide.xml | 23 +++++++++------ .../epg_grab/multiple_channels.guide.xml | 29 +++++++++++-------- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/tests/__data__/expected/epg_grab/custom_channels.guide.xml b/tests/__data__/expected/epg_grab/custom_channels.guide.xml index 932a9e6a3..353b20e22 100644 --- a/tests/__data__/expected/epg_grab/custom_channels.guide.xml +++ b/tests/__data__/expected/epg_grab/custom_channels.guide.xml @@ -1,15 +1,20 @@ Custom Channel 1https://example.com -Custom Channel 2https://example.com -Channel 1https://example.com -Channel 3https://example2.com -Channel 4https://example2.com Channel 1https://example2.com -Programme1 (example.com) -Program1 (example.com) +Channel 4https://example2.com +Channel 3https://example2.com +Channel 1https://example.com +Custom Channel 2https://example.com +Program1 (example2.com) +Program1 (example2.com) +Programme1 (example2.com) Programme1 (example2.com) -Programme1 (example.com) -Program1 (example.com) -Program1 (example2.com) +Program1 (example2.com) Program1 (example2.com) +Program1 (example2.com) +Program1 (example2.com) +Programme1 (example2.com) +Programme1 (example2.com) +Program1 (example2.com) +Program1 (example2.com) \ No newline at end of file diff --git a/tests/__data__/expected/epg_grab/multiple_channels.guide.xml b/tests/__data__/expected/epg_grab/multiple_channels.guide.xml index 489914a70..5b0986152 100644 --- a/tests/__data__/expected/epg_grab/multiple_channels.guide.xml +++ b/tests/__data__/expected/epg_grab/multiple_channels.guide.xml @@ -1,18 +1,23 @@ Channel 1https://example.com -Channel 2https://example.com36 -Channel 1https://example.com -Channel 3https://example.com -Channel 3https://example2.com -Channel 4https://example2.com Channel 1https://example2.com -Programme1 (example.com) -Programme1 (example.com) +Channel 4https://example2.com +Channel 3https://example2.com +Channel 3https://example.com +Channel 1https://example.com +Channel 2https://example.com36 +Program1 (example2.com) +Program1 (example2.com) +Programme1 (example2.com) Programme1 (example2.com) -Program1 (example.com) -Program1 (example.com) -Program1 (example2.com) -Program1 (example.com) -Program1 (example.com) +Program1 (example2.com) Program1 (example2.com) +Program1 (example2.com) +Program1 (example2.com) +Program1 (example2.com) +Program1 (example2.com) +Programme1 (example2.com) +Programme1 (example2.com) +Program1 (example2.com) +Program1 (example2.com) \ No newline at end of file From 588bb8b4ff136b29b7eb1e60cac42b90f8d6a7f4 Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Sat, 1 Nov 2025 12:01:37 +0300 Subject: [PATCH 3/5] Update default.config.js --- scripts/default.config.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/default.config.js b/scripts/default.config.js index 2bc576bae..d5e49a641 100644 --- a/scripts/default.config.js +++ b/scripts/default.config.js @@ -1,6 +1,11 @@ export default { + output: 'guide.xml', days: 1, delay: 0, + maxConnections: 1, + curl: false, + gzip: false, + debug: false, request: { maxContentLength: 5242880, timeout: 30000, From 890da3fedb3d1685be2578aa651397e0b3ac6136 Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Sat, 1 Nov 2025 12:03:07 +0300 Subject: [PATCH 4/5] Update grab.ts --- scripts/commands/epg/grab.ts | 85 ++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 38 deletions(-) diff --git a/scripts/commands/epg/grab.ts b/scripts/commands/epg/grab.ts index f61d7de5e..cc858ae72 100644 --- a/scripts/commands/epg/grab.ts +++ b/scripts/commands/epg/grab.ts @@ -24,7 +24,7 @@ program 'Path to *.channels.xml file (required if the "--site" attribute is not specified)' ) ) - .addOption(new Option('-o, --output ', 'Path to output file').default('guide.xml')) + .addOption(new Option('-o, --output ', 'Path to output file')) .addOption(new Option('-l, --lang ', 'Filter channels by languages (ISO 639-1 codes)')) .addOption( new Option('-t, --timeout ', 'Override the default timeout for each request') @@ -47,27 +47,22 @@ program ) .addOption( new Option('--maxConnections ', 'Limit on the number of concurrent requests') - .default(1) .argParser(parseNumber) .env('MAX_CONNECTIONS') ) - .addOption( - new Option('--gzip', 'Create a compressed version of the guide as well') - .default(false) - .env('GZIP') - ) - .addOption(new Option('--curl', 'Display each request as CURL').default(false).env('CURL')) - .addOption(new Option('--debug', 'Enable debug mode').default(false).env('DEBUG')) + .addOption(new Option('--gzip', 'Create a compressed version of the guide as well').env('GZIP')) + .addOption(new Option('--curl', 'Display each request as CURL').env('CURL')) + .addOption(new Option('--debug', 'Enable debug mode').env('DEBUG')) .parse() interface GrabOptions { site?: string channels?: string - output: string - gzip: boolean - curl: boolean - debug: boolean - maxConnections: number + output?: string + gzip?: boolean + curl?: boolean + debug?: boolean + maxConnections?: number timeout?: number delay?: number lang?: string @@ -85,10 +80,10 @@ async function main() { const logger = new Logger({ level: options.debug ? LOG_LEVELS['debug'] : LOG_LEVELS['info'] }) logger.info('starting...') - let config: epgGrabber.Types.SiteConfig = defaultConfig + let globalConfig: epgGrabber.Types.SiteConfig = {} if (typeof options.timeout === 'number') - config = merge(config, { request: { timeout: options.timeout } }) + globalConfig = merge(globalConfig, { request: { timeout: options.timeout } }) if (options.proxy !== undefined) { const proxy = parseProxy(options.proxy) if ( @@ -96,31 +91,36 @@ async function main() { ['socks', 'socks5', 'socks5h', 'socks4', 'socks4a'].includes(String(proxy.protocol)) ) { const socksProxyAgent = new SocksProxyAgent(options.proxy) - config = merge(config, { + globalConfig = merge(globalConfig, { request: { httpAgent: socksProxyAgent, httpsAgent: socksProxyAgent } }) } else { - config = merge(config, { request: { proxy } }) + globalConfig = merge(globalConfig, { request: { proxy } }) } } - if (typeof options.output === 'string') config.output = options.output - if (typeof options.days === 'number') config.days = options.days - if (typeof options.delay === 'number') config.delay = options.delay - if (typeof options.maxConnections === 'number') config.maxConnections = options.maxConnections - if (typeof options.curl === 'boolean') config.curl = options.curl - if (typeof options.gzip === 'boolean') config.gzip = options.gzip - - const grabber = - process.env.NODE_ENV === 'test' ? new EPGGrabberMock(config) : new EPGGrabber(config) - - const globalConfig = grabber.globalConfig + if (typeof options.output === 'string') globalConfig.output = options.output + if (typeof options.days === 'number') globalConfig.days = options.days + if (typeof options.delay === 'number') globalConfig.delay = options.delay + if (typeof options.maxConnections === 'number') + globalConfig.maxConnections = options.maxConnections + if (typeof options.curl === 'boolean') globalConfig.curl = options.curl + if (typeof options.gzip === 'boolean') globalConfig.gzip = options.gzip + if (typeof options.debug === 'boolean') globalConfig.debug = options.debug logger.debug(`config: ${JSON.stringify(globalConfig, null, 2)}`) + const grabber = + process.env.NODE_ENV === 'test' + ? new EPGGrabberMock(globalConfig) + : new EPGGrabber(globalConfig) + grabber.client.instance.interceptors.request.use( request => { - if (globalConfig.curl) { + logger.debug(`request: ${JSON.stringify(request, null, 2)}`) + + const curl = globalConfig.curl || defaultConfig.curl + if (curl) { type AllowedMethods = | 'GET' | 'get' @@ -194,11 +194,12 @@ async function main() { channel.index = index++ if (!channel.site || !channel.site_id || !channel.name) continue - const config = await loadJs(channel.getConfigPath()) - const days: number = config.days || globalConfig.days + let config = await loadJs(channel.getConfigPath()) + config = merge(defaultConfig, config) if (!channel.xmltv_id) channel.xmltv_id = channel.site_id + const days = globalConfig.days || config.days const currDate = dayjs.utc(process.env.CURR_DATE || new Date().toISOString()) const dates = Array.from({ length: days }, (_, day) => currDate.add(day, 'd')) @@ -214,7 +215,9 @@ async function main() { }) } - const taskQueue = new TaskQueue(Promise as PromisyClass, options.maxConnections) + const maxConnections = globalConfig.maxConnections || defaultConfig.maxConnections + + const taskQueue = new TaskQueue(Promise as PromisyClass, maxConnections) const queueItems = queue.getItems() @@ -271,11 +274,15 @@ async function main() { await Promise.all(requests.all()) - const pathTemplate = new Template(options.output) + const output = globalConfig.output || defaultConfig.output - const channelsGroupedByKey = channels.groupBy((channel: Channel) => { - return pathTemplate.format({ lang: channel.lang || 'en', site: channel.site || '' }) - }) + const pathTemplate = new Template(output) + + const channelsGroupedByKey = channels + .uniqBy((channel: Channel) => `${channel.xmltv_id}:${channel.site}:${channel.lang}`) + .groupBy((channel: Channel) => { + return pathTemplate.format({ lang: channel.lang || 'en', site: channel.site || '' }) + }) const programsGroupedByKey = programs.groupBy((program: Program) => { const lang = @@ -286,12 +293,14 @@ async function main() { return pathTemplate.format({ lang, site: program.site || '' }) }) + const gzip = globalConfig.gzip || defaultConfig.gzip + for (const groupKey of channelsGroupedByKey.keys()) { const groupChannels = new Collection(channelsGroupedByKey.get(groupKey)) const groupPrograms = new Collection(programsGroupedByKey.get(groupKey)) const guide = new Guide({ filepath: groupKey, - gzip: options.gzip, + gzip, channels: groupChannels, programs: groupPrograms }) From 06a4cb66d48f95fc536b3e658209f6c91995e261 Mon Sep 17 00:00:00 2001 From: freearhey <7253922+freearhey@users.noreply.github.com> Date: Sun, 2 Nov 2025 00:49:17 +0300 Subject: [PATCH 5/5] Update format.yml --- .github/workflows/format.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index 7fe7e7acb..a227063c5 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -48,11 +48,11 @@ jobs: git config user.name "iptv-bot[bot]" git config user.email "84861620+iptv-bot[bot]@users.noreply.github.com" - name: Commit changed *.channels.xml files - if: steps.files_after.outputs.channels_any_changed == 'true' + if: steps.files.outputs.channels_any_changed == 'true' run: | git add streams git status git commit -m "[Bot] Format *.channels.xml files" -m "Committed by [iptv-bot](https://github.com/apps/iptv-bot) via [format](https://github.com/iptv-org/epg/actions/runs/${{ github.run_id }}) workflow." --no-verify - name: Push all changes to the repository - if: ${{ !env.ACT && github.ref == 'refs/heads/master' && steps.files_after.outputs.channels_any_changed == 'true' }} + if: ${{ !env.ACT && github.ref == 'refs/heads/master' && steps.files.outputs.channels_any_changed == 'true' }} run: git push \ No newline at end of file